diff --git a/.gitattributes b/.gitattributes index 9bed4fa42f48590ac03d6fec490772ba7b4c8efa..d89f6023c76d223f3ed08fac2e1c426fb0dcfbac 100644 --- a/.gitattributes +++ b/.gitattributes @@ -259,3 +259,4 @@ epfl_final/wandb/run-20260615_161335-aqqleyxv/run-aqqleyxv.wandb filter=lfs diff epfl_final/wandb/run-20260615_224051-7jul6zqc/run-7jul6zqc.wandb filter=lfs diff=lfs merge=lfs -text epfl_final/wandb/run-20260616_012002-apvt4901/run-apvt4901.wandb filter=lfs diff=lfs merge=lfs -text epfl_final/wandb/run-20260618_175952-33yw8345/run-33yw8345.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/log_files/eval_sparse_64.log filter=lfs diff=lfs merge=lfs -text diff --git a/epfl_final/log_files/1-2b_dense_cont_46k.log b/epfl_final/log_files/1-2b_dense_cont_46k.log new file mode 100644 index 0000000000000000000000000000000000000000..d08306107857bac6d1850bde85e4b23fb2c9ed6b --- /dev/null +++ b/epfl_final/log_files/1-2b_dense_cont_46k.log @@ -0,0 +1,351 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260529_183242 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260529_183242", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 48k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000 +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000... +Resumed from checkpoint at step 46000 (epoch 0, micro_step 460000) +Resuming training from step 46000 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 3yae6xjj +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260529_185135-3yae6xjj +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense continued 48k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/3yae6xjj +Starting training... +Validation at step 46000: Loss = 2.5221, Perplexity = 12.46 +Step 46010/50462 | Loss: 2.5136 | LR: 9.83e-06 | 82727.5ms/step | 0.7 samples/s +Step 46020/50462 | Loss: 2.4987 | LR: 9.79e-06 | 5073.9ms/step | 11.8 samples/s +Step 46030/50462 | Loss: 2.5210 | LR: 9.74e-06 | 5090.5ms/step | 11.8 samples/s +Step 46040/50462 | Loss: 2.5202 | LR: 9.70e-06 | 5078.9ms/step | 11.8 samples/s +Step 46050/50462 | Loss: 2.5031 | LR: 9.65e-06 | 5081.8ms/step | 11.8 samples/s +Step 46060/50462 | Loss: 2.4972 | LR: 9.61e-06 | 5097.5ms/step | 11.8 samples/s +Step 46070/50462 | Loss: 2.5165 | LR: 9.57e-06 | 5084.9ms/step | 11.8 samples/s +Step 46080/50462 | Loss: 2.5113 | LR: 9.52e-06 | 5091.8ms/step | 11.8 samples/s +Step 46090/50462 | Loss: 2.4799 | LR: 9.48e-06 | 5234.7ms/step | 11.5 samples/s +Step 46100/50462 | Loss: 2.5059 | LR: 9.44e-06 | 5098.0ms/step | 11.8 samples/s +Step 46110/50462 | Loss: 2.5016 | LR: 9.40e-06 | 5101.6ms/step | 11.8 samples/s +Step 46120/50462 | Loss: 2.4968 | LR: 9.35e-06 | 5074.7ms/step | 11.8 samples/s +Step 46130/50462 | Loss: 2.4982 | LR: 9.31e-06 | 5087.7ms/step | 11.8 samples/s +Step 46140/50462 | Loss: 2.4984 | LR: 9.27e-06 | 5089.6ms/step | 11.8 samples/s +Step 46150/50462 | Loss: 2.4967 | LR: 9.22e-06 | 5108.0ms/step | 11.7 samples/s +Step 46160/50462 | Loss: 2.5143 | LR: 9.18e-06 | 5084.8ms/step | 11.8 samples/s +Step 46170/50462 | Loss: 2.5149 | LR: 9.14e-06 | 5064.3ms/step | 11.8 samples/s +Step 46180/50462 | Loss: 2.4845 | LR: 9.10e-06 | 5102.9ms/step | 11.8 samples/s +Step 46190/50462 | Loss: 2.4850 | LR: 9.06e-06 | 5080.9ms/step | 11.8 samples/s +Step 46200/50462 | Loss: 2.4926 | LR: 9.01e-06 | 5067.8ms/step | 11.8 samples/s +Step 46210/50462 | Loss: 2.5091 | LR: 8.97e-06 | 5069.3ms/step | 11.8 samples/s +Step 46220/50462 | Loss: 2.5119 | LR: 8.93e-06 | 5073.3ms/step | 11.8 samples/s +Step 46230/50462 | Loss: 2.5180 | LR: 8.89e-06 | 5082.2ms/step | 11.8 samples/s +Step 46240/50462 | Loss: 2.5267 | LR: 8.85e-06 | 5088.3ms/step | 11.8 samples/s +Step 46250/50462 | Loss: 2.5245 | LR: 8.80e-06 | 5070.0ms/step | 11.8 samples/s +Validation at step 46250: Loss = 2.5160, Perplexity = 12.38 +Step 46260/50462 | Loss: 2.5244 | LR: 8.76e-06 | 7621.7ms/step | 7.9 samples/s +Step 46270/50462 | Loss: 2.5108 | LR: 8.72e-06 | 5100.0ms/step | 11.8 samples/s +Step 46280/50462 | Loss: 2.5424 | LR: 8.68e-06 | 5149.3ms/step | 11.7 samples/s +Step 46290/50462 | Loss: 2.5291 | LR: 8.64e-06 | 5099.2ms/step | 11.8 samples/s +Step 46300/50462 | Loss: 2.4995 | LR: 8.60e-06 | 5088.5ms/step | 11.8 samples/s +Step 46310/50462 | Loss: 2.5071 | LR: 8.56e-06 | 5094.1ms/step | 11.8 samples/s +Step 46320/50462 | Loss: 2.5327 | LR: 8.52e-06 | 5095.4ms/step | 11.8 samples/s +Step 46330/50462 | Loss: 2.5121 | LR: 8.47e-06 | 5088.7ms/step | 11.8 samples/s +Step 46340/50462 | Loss: 2.5119 | LR: 8.43e-06 | 5093.9ms/step | 11.8 samples/s +Step 46350/50462 | Loss: 2.5328 | LR: 8.39e-06 | 5093.1ms/step | 11.8 samples/s +Step 46360/50462 | Loss: 2.4956 | LR: 8.35e-06 | 5093.4ms/step | 11.8 samples/s +Step 46370/50462 | Loss: 2.4811 | LR: 8.31e-06 | 5080.5ms/step | 11.8 samples/s +Step 46380/50462 | Loss: 2.5062 | LR: 8.27e-06 | 5202.4ms/step | 11.5 samples/s +Step 46390/50462 | Loss: 2.4951 | LR: 8.23e-06 | 5087.3ms/step | 11.8 samples/s +Step 46400/50462 | Loss: 2.4869 | LR: 8.19e-06 | 5076.9ms/step | 11.8 samples/s +Step 46410/50462 | Loss: 2.4994 | LR: 8.15e-06 | 5061.7ms/step | 11.9 samples/s +Step 46420/50462 | Loss: 2.4990 | LR: 8.11e-06 | 5073.7ms/step | 11.8 samples/s +Step 46430/50462 | Loss: 2.5198 | LR: 8.07e-06 | 5074.7ms/step | 11.8 samples/s +Step 46440/50462 | Loss: 2.5022 | LR: 8.03e-06 | 5069.0ms/step | 11.8 samples/s +Step 46450/50462 | Loss: 2.5283 | LR: 7.99e-06 | 5074.0ms/step | 11.8 samples/s +Step 46460/50462 | Loss: 2.5065 | LR: 7.95e-06 | 5089.8ms/step | 11.8 samples/s +Step 46470/50462 | Loss: 2.4978 | LR: 7.91e-06 | 5072.7ms/step | 11.8 samples/s +Step 46480/50462 | Loss: 2.4927 | LR: 7.87e-06 | 5082.2ms/step | 11.8 samples/s +Step 46490/50462 | Loss: 2.4891 | LR: 7.83e-06 | 5074.3ms/step | 11.8 samples/s +Step 46500/50462 | Loss: 2.5059 | LR: 7.80e-06 | 5069.6ms/step | 11.8 samples/s +Validation at step 46500: Loss = 2.5156, Perplexity = 12.37 +Step 46510/50462 | Loss: 2.5005 | LR: 7.76e-06 | 7630.4ms/step | 7.9 samples/s +Step 46520/50462 | Loss: 2.5238 | LR: 7.72e-06 | 5095.5ms/step | 11.8 samples/s +Step 46530/50462 | Loss: 2.5039 | LR: 7.68e-06 | 5091.5ms/step | 11.8 samples/s +Step 46540/50462 | Loss: 2.5166 | LR: 7.64e-06 | 5093.1ms/step | 11.8 samples/s +Step 46550/50462 | Loss: 2.4782 | LR: 7.60e-06 | 5097.4ms/step | 11.8 samples/s +Step 46560/50462 | Loss: 2.4951 | LR: 7.56e-06 | 5145.5ms/step | 11.7 samples/s +Step 46570/50462 | Loss: 2.5057 | LR: 7.52e-06 | 5095.2ms/step | 11.8 samples/s +Step 46580/50462 | Loss: 2.4974 | LR: 7.49e-06 | 5092.4ms/step | 11.8 samples/s +Step 46590/50462 | Loss: 2.4902 | LR: 7.45e-06 | 5109.4ms/step | 11.7 samples/s +Step 46600/50462 | Loss: 2.5052 | LR: 7.41e-06 | 5093.6ms/step | 11.8 samples/s +Step 46610/50462 | Loss: 2.5036 | LR: 7.37e-06 | 5092.3ms/step | 11.8 samples/s +Step 46620/50462 | Loss: 2.4863 | LR: 7.33e-06 | 5083.4ms/step | 11.8 samples/s +Step 46630/50462 | Loss: 2.5004 | LR: 7.29e-06 | 5092.8ms/step | 11.8 samples/s +Step 46640/50462 | Loss: 2.5125 | LR: 7.26e-06 | 5085.1ms/step | 11.8 samples/s +Step 46650/50462 | Loss: 2.5046 | LR: 7.22e-06 | 5094.1ms/step | 11.8 samples/s +Step 46660/50462 | Loss: 2.4945 | LR: 7.18e-06 | 5086.9ms/step | 11.8 samples/s +Step 46670/50462 | Loss: 2.5093 | LR: 7.14e-06 | 5197.9ms/step | 11.5 samples/s +Step 46680/50462 | Loss: 2.5178 | LR: 7.11e-06 | 5093.8ms/step | 11.8 samples/s +Step 46690/50462 | Loss: 2.5347 | LR: 7.07e-06 | 5074.4ms/step | 11.8 samples/s +Step 46700/50462 | Loss: 2.4950 | LR: 7.03e-06 | 5099.1ms/step | 11.8 samples/s +Step 46710/50462 | Loss: 2.5279 | LR: 6.99e-06 | 5099.5ms/step | 11.8 samples/s +Step 46720/50462 | Loss: 2.5116 | LR: 6.96e-06 | 5085.8ms/step | 11.8 samples/s +Step 46730/50462 | Loss: 2.4914 | LR: 6.92e-06 | 5097.2ms/step | 11.8 samples/s +Step 46740/50462 | Loss: 2.5180 | LR: 6.88e-06 | 5074.8ms/step | 11.8 samples/s +Step 46750/50462 | Loss: 2.5009 | LR: 6.85e-06 | 5085.5ms/step | 11.8 samples/s +Validation at step 46750: Loss = 2.5154, Perplexity = 12.37 +Step 46760/50462 | Loss: 2.5153 | LR: 6.81e-06 | 7626.9ms/step | 7.9 samples/s +Step 46770/50462 | Loss: 2.5049 | LR: 6.77e-06 | 5095.0ms/step | 11.8 samples/s +Step 46780/50462 | Loss: 2.5303 | LR: 6.74e-06 | 5084.6ms/step | 11.8 samples/s +Step 46790/50462 | Loss: 2.4876 | LR: 6.70e-06 | 5082.0ms/step | 11.8 samples/s +Step 46800/50462 | Loss: 2.5255 | LR: 6.66e-06 | 5090.2ms/step | 11.8 samples/s +Step 46810/50462 | Loss: 2.5217 | LR: 6.63e-06 | 5101.5ms/step | 11.8 samples/s +Step 46820/50462 | Loss: 2.4851 | LR: 6.59e-06 | 5087.0ms/step | 11.8 samples/s +Step 46830/50462 | Loss: 2.4832 | LR: 6.56e-06 | 5085.6ms/step | 11.8 samples/s +Step 46840/50462 | Loss: 2.5086 | LR: 6.52e-06 | 5138.2ms/step | 11.7 samples/s +Step 46850/50462 | Loss: 2.5047 | LR: 6.48e-06 | 5085.6ms/step | 11.8 samples/s +Step 46860/50462 | Loss: 2.5043 | LR: 6.45e-06 | 5081.3ms/step | 11.8 samples/s +Step 46870/50462 | Loss: 2.4625 | LR: 6.41e-06 | 5097.4ms/step | 11.8 samples/s +Step 46880/50462 | Loss: 2.4979 | LR: 6.38e-06 | 5093.3ms/step | 11.8 samples/s +Step 46890/50462 | Loss: 2.5039 | LR: 6.34e-06 | 5102.0ms/step | 11.8 samples/s +Step 46900/50462 | Loss: 2.4961 | LR: 6.31e-06 | 5105.7ms/step | 11.8 samples/s +Step 46910/50462 | Loss: 2.4893 | LR: 6.27e-06 | 5087.8ms/step | 11.8 samples/s +Step 46920/50462 | Loss: 2.5272 | LR: 6.24e-06 | 5093.6ms/step | 11.8 samples/s +Step 46930/50462 | Loss: 2.5035 | LR: 6.20e-06 | 5082.2ms/step | 11.8 samples/s +Step 46940/50462 | Loss: 2.5054 | LR: 6.17e-06 | 5093.6ms/step | 11.8 samples/s +Step 46950/50462 | Loss: 2.4885 | LR: 6.13e-06 | 5082.0ms/step | 11.8 samples/s +Step 46960/50462 | Loss: 2.5119 | LR: 6.10e-06 | 5136.6ms/step | 11.7 samples/s +Step 46970/50462 | Loss: 2.5154 | LR: 6.06e-06 | 5155.0ms/step | 11.6 samples/s +Step 46980/50462 | Loss: 2.5147 | LR: 6.03e-06 | 5107.1ms/step | 11.7 samples/s +Step 46990/50462 | Loss: 2.5355 | LR: 5.99e-06 | 5113.6ms/step | 11.7 samples/s +Step 47000/50462 | Loss: 2.5152 | LR: 5.96e-06 | 5107.9ms/step | 11.7 samples/s +Validation at step 47000: Loss = 2.5153, Perplexity = 12.37 +Saved checkpoint to checkpoints/run_20260529_183242/checkpoint-47000 +Step 47010/50462 | Loss: 2.5271 | LR: 5.93e-06 | 8643.4ms/step | 6.9 samples/s +Step 47020/50462 | Loss: 2.5038 | LR: 5.89e-06 | 5083.1ms/step | 11.8 samples/s +Step 47030/50462 | Loss: 2.5237 | LR: 5.86e-06 | 5090.5ms/step | 11.8 samples/s +Step 47040/50462 | Loss: 2.4690 | LR: 5.82e-06 | 5082.3ms/step | 11.8 samples/s +Step 47050/50462 | Loss: 2.5055 | LR: 5.79e-06 | 5078.2ms/step | 11.8 samples/s +Step 47060/50462 | Loss: 2.5038 | LR: 5.76e-06 | 5087.9ms/step | 11.8 samples/s +Step 47070/50462 | Loss: 2.4966 | LR: 5.72e-06 | 5079.2ms/step | 11.8 samples/s +Step 47080/50462 | Loss: 2.5046 | LR: 5.69e-06 | 5072.9ms/step | 11.8 samples/s +Step 47090/50462 | Loss: 2.4994 | LR: 5.65e-06 | 5088.7ms/step | 11.8 samples/s +Step 47100/50462 | Loss: 2.5033 | LR: 5.62e-06 | 5077.8ms/step | 11.8 samples/s +Step 47110/50462 | Loss: 2.5071 | LR: 5.59e-06 | 5093.1ms/step | 11.8 samples/s +Step 47120/50462 | Loss: 2.4575 | LR: 5.56e-06 | 5138.0ms/step | 11.7 samples/s +Step 47130/50462 | Loss: 2.5043 | LR: 5.52e-06 | 5102.5ms/step | 11.8 samples/s +Step 47140/50462 | Loss: 2.5167 | LR: 5.49e-06 | 5110.2ms/step | 11.7 samples/s +Step 47150/50462 | Loss: 2.4729 | LR: 5.46e-06 | 5103.3ms/step | 11.8 samples/s +Step 47160/50462 | Loss: 2.5147 | LR: 5.42e-06 | 5100.1ms/step | 11.8 samples/s +Step 47170/50462 | Loss: 2.4977 | LR: 5.39e-06 | 5108.1ms/step | 11.7 samples/s +Step 47180/50462 | Loss: 2.5073 | LR: 5.36e-06 | 5101.7ms/step | 11.8 samples/s +Step 47190/50462 | Loss: 2.5018 | LR: 5.33e-06 | 5114.6ms/step | 11.7 samples/s +Step 47200/50462 | Loss: 2.4998 | LR: 5.29e-06 | 5123.2ms/step | 11.7 samples/s +Step 47210/50462 | Loss: 2.4737 | LR: 5.26e-06 | 5122.3ms/step | 11.7 samples/s +Step 47220/50462 | Loss: 2.5144 | LR: 5.23e-06 | 5109.2ms/step | 11.7 samples/s +Step 47230/50462 | Loss: 2.5125 | LR: 5.20e-06 | 5098.5ms/step | 11.8 samples/s +Step 47240/50462 | Loss: 2.4808 | LR: 5.16e-06 | 5102.6ms/step | 11.8 samples/s +Step 47250/50462 | Loss: 2.5115 | LR: 5.13e-06 | 5117.1ms/step | 11.7 samples/s +Validation at step 47250: Loss = 2.5152, Perplexity = 12.37 +Step 47260/50462 | Loss: 2.5060 | LR: 5.10e-06 | 7731.6ms/step | 7.8 samples/s +Step 47270/50462 | Loss: 2.5177 | LR: 5.07e-06 | 5077.9ms/step | 11.8 samples/s +Step 47280/50462 | Loss: 2.4915 | LR: 5.04e-06 | 5083.4ms/step | 11.8 samples/s +Step 47290/50462 | Loss: 2.5194 | LR: 5.01e-06 | 5090.9ms/step | 11.8 samples/s +Step 47300/50462 | Loss: 2.5177 | LR: 4.97e-06 | 5064.7ms/step | 11.8 samples/s +Step 47310/50462 | Loss: 2.4795 | LR: 4.94e-06 | 5069.3ms/step | 11.8 samples/s +Step 47320/50462 | Loss: 2.4967 | LR: 4.91e-06 | 5072.6ms/step | 11.8 samples/s +Step 47330/50462 | Loss: 2.4998 | LR: 4.88e-06 | 5065.2ms/step | 11.8 samples/s +Step 47340/50462 | Loss: 2.4647 | LR: 4.85e-06 | 5068.4ms/step | 11.8 samples/s +Step 47350/50462 | Loss: 2.5123 | LR: 4.82e-06 | 5072.7ms/step | 11.8 samples/s +Step 47360/50462 | Loss: 2.4828 | LR: 4.79e-06 | 5087.6ms/step | 11.8 samples/s +Step 47370/50462 | Loss: 2.5210 | LR: 4.76e-06 | 5065.7ms/step | 11.8 samples/s +Step 47380/50462 | Loss: 2.5134 | LR: 4.73e-06 | 5062.9ms/step | 11.9 samples/s +Step 47390/50462 | Loss: 2.4962 | LR: 4.70e-06 | 5072.8ms/step | 11.8 samples/s +Step 47400/50462 | Loss: 2.4993 | LR: 4.67e-06 | 5133.0ms/step | 11.7 samples/s +Step 47410/50462 | Loss: 2.5106 | LR: 4.64e-06 | 5110.1ms/step | 11.7 samples/s +Step 47420/50462 | Loss: 2.4624 | LR: 4.61e-06 | 5112.3ms/step | 11.7 samples/s +Step 47430/50462 | Loss: 2.4903 | LR: 4.58e-06 | 5106.6ms/step | 11.7 samples/s +Step 47440/50462 | Loss: 2.4991 | LR: 4.55e-06 | 5109.8ms/step | 11.7 samples/s +Step 47450/50462 | Loss: 2.4738 | LR: 4.52e-06 | 5118.8ms/step | 11.7 samples/s +Step 47460/50462 | Loss: 2.5186 | LR: 4.49e-06 | 5111.5ms/step | 11.7 samples/s +Step 47470/50462 | Loss: 2.5164 | LR: 4.46e-06 | 5107.9ms/step | 11.7 samples/s +Step 47480/50462 | Loss: 2.5151 | LR: 4.43e-06 | 5114.2ms/step | 11.7 samples/s +Step 47490/50462 | Loss: 2.5078 | LR: 4.40e-06 | 5105.7ms/step | 11.8 samples/s +Step 47500/50462 | Loss: 2.4992 | LR: 4.37e-06 | 5118.2ms/step | 11.7 samples/s +Validation at step 47500: Loss = 2.5152, Perplexity = 12.37 +Step 47510/50462 | Loss: 2.4700 | LR: 4.34e-06 | 7613.1ms/step | 7.9 samples/s +Step 47520/50462 | Loss: 2.5181 | LR: 4.31e-06 | 5109.3ms/step | 11.7 samples/s +Step 47530/50462 | Loss: 2.5134 | LR: 4.28e-06 | 5083.8ms/step | 11.8 samples/s +Step 47540/50462 | Loss: 2.4818 | LR: 4.25e-06 | 5094.2ms/step | 11.8 samples/s +Step 47550/50462 | Loss: 2.5112 | LR: 4.22e-06 | 5192.0ms/step | 11.6 samples/s +Step 47560/50462 | Loss: 2.4964 | LR: 4.19e-06 | 5078.0ms/step | 11.8 samples/s +Step 47570/50462 | Loss: 2.5066 | LR: 4.16e-06 | 5091.1ms/step | 11.8 samples/s +Step 47580/50462 | Loss: 2.5418 | LR: 4.13e-06 | 5111.1ms/step | 11.7 samples/s +Step 47590/50462 | Loss: 2.5126 | LR: 4.11e-06 | 5099.3ms/step | 11.8 samples/s +Step 47600/50462 | Loss: 2.5036 | LR: 4.08e-06 | 5093.2ms/step | 11.8 samples/s +Step 47610/50462 | Loss: 2.4834 | LR: 4.05e-06 | 5096.8ms/step | 11.8 samples/s +Step 47620/50462 | Loss: 2.4971 | LR: 4.02e-06 | 5100.0ms/step | 11.8 samples/s +Step 47630/50462 | Loss: 2.5243 | LR: 3.99e-06 | 5092.0ms/step | 11.8 samples/s +Step 47640/50462 | Loss: 2.4794 | LR: 3.97e-06 | 5100.3ms/step | 11.8 samples/s diff --git a/epfl_final/log_files/1-2b_dense_cont_48k.log b/epfl_final/log_files/1-2b_dense_cont_48k.log new file mode 100644 index 0000000000000000000000000000000000000000..d0728de64daf02d5481945a7c8c1ea3569ed0fae --- /dev/null +++ b/epfl_final/log_files/1-2b_dense_cont_48k.log @@ -0,0 +1,470 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260529_182827 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260529_182827", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-48000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 48k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-48000 +Checkpoint activations: FalseCheckpoint activations: False + +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) + +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-48000... +Resumed from checkpoint at step 48000 (epoch 0, micro_step 480000) +Resuming training from step 48000 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run wbfo0o07 +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260529_184637-wbfo0o07 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense continued 48k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/wbfo0o07 +Starting training... +Validation at step 48000: Loss = 2.5221, Perplexity = 12.45 +Step 48010/50462 | Loss: 2.5176 | LR: 3.00e-06 | 79833.7ms/step | 0.8 samples/s +Step 48020/50462 | Loss: 2.5457 | LR: 2.97e-06 | 5205.0ms/step | 11.5 samples/s +Step 48030/50462 | Loss: 2.4873 | LR: 2.95e-06 | 5232.3ms/step | 11.5 samples/s +Step 48040/50462 | Loss: 2.5006 | LR: 2.92e-06 | 5301.5ms/step | 11.3 samples/s +Step 48050/50462 | Loss: 2.4934 | LR: 2.90e-06 | 5246.8ms/step | 11.4 samples/s +Step 48060/50462 | Loss: 2.5102 | LR: 2.87e-06 | 5251.1ms/step | 11.4 samples/s +Step 48070/50462 | Loss: 2.5122 | LR: 2.85e-06 | 5248.6ms/step | 11.4 samples/s +Step 48080/50462 | Loss: 2.5359 | LR: 2.83e-06 | 5249.9ms/step | 11.4 samples/s +Step 48090/50462 | Loss: 2.5121 | LR: 2.80e-06 | 5347.1ms/step | 11.2 samples/s +Step 48100/50462 | Loss: 2.5147 | LR: 2.78e-06 | 5242.9ms/step | 11.4 samples/s +Step 48110/50462 | Loss: 2.5043 | LR: 2.76e-06 | 5250.5ms/step | 11.4 samples/s +Step 48120/50462 | Loss: 2.5025 | LR: 2.73e-06 | 5252.1ms/step | 11.4 samples/s +Step 48130/50462 | Loss: 2.5032 | LR: 2.71e-06 | 5251.2ms/step | 11.4 samples/s +Step 48140/50462 | Loss: 2.4849 | LR: 2.69e-06 | 5251.6ms/step | 11.4 samples/s +Step 48150/50462 | Loss: 2.4982 | LR: 2.66e-06 | 5254.7ms/step | 11.4 samples/s +Step 48160/50462 | Loss: 2.5177 | LR: 2.64e-06 | 5252.6ms/step | 11.4 samples/s +Step 48170/50462 | Loss: 2.4973 | LR: 2.62e-06 | 5242.5ms/step | 11.4 samples/s +Step 48180/50462 | Loss: 2.5098 | LR: 2.60e-06 | 5247.1ms/step | 11.4 samples/s +Step 48190/50462 | Loss: 2.5059 | LR: 2.57e-06 | 5245.1ms/step | 11.4 samples/s +Step 48200/50462 | Loss: 2.5151 | LR: 2.55e-06 | 5253.5ms/step | 11.4 samples/s +Step 48210/50462 | Loss: 2.4910 | LR: 2.53e-06 | 5254.3ms/step | 11.4 samples/s +Step 48220/50462 | Loss: 2.4970 | LR: 2.51e-06 | 5251.0ms/step | 11.4 samples/s +Step 48230/50462 | Loss: 2.5044 | LR: 2.48e-06 | 5256.5ms/step | 11.4 samples/s +Step 48240/50462 | Loss: 2.5022 | LR: 2.46e-06 | 5252.8ms/step | 11.4 samples/s +Step 48250/50462 | Loss: 2.5411 | LR: 2.44e-06 | 5256.3ms/step | 11.4 samples/s +Validation at step 48250: Loss = 2.5191, Perplexity = 12.42 +Step 48260/50462 | Loss: 2.4783 | LR: 2.42e-06 | 7822.7ms/step | 7.7 samples/s +Step 48270/50462 | Loss: 2.5044 | LR: 2.39e-06 | 5253.8ms/step | 11.4 samples/s +Step 48280/50462 | Loss: 2.5126 | LR: 2.37e-06 | 5252.6ms/step | 11.4 samples/s +Step 48290/50462 | Loss: 2.5001 | LR: 2.35e-06 | 5254.4ms/step | 11.4 samples/s +Step 48300/50462 | Loss: 2.4932 | LR: 2.33e-06 | 5250.0ms/step | 11.4 samples/s +Step 48310/50462 | Loss: 2.5132 | LR: 2.31e-06 | 5253.4ms/step | 11.4 samples/s +Step 48320/50462 | Loss: 2.5122 | LR: 2.29e-06 | 5303.0ms/step | 11.3 samples/s +Step 48330/50462 | Loss: 2.4952 | LR: 2.27e-06 | 5251.1ms/step | 11.4 samples/s +Step 48340/50462 | Loss: 2.5184 | LR: 2.24e-06 | 5249.1ms/step | 11.4 samples/s +Step 48350/50462 | Loss: 2.5319 | LR: 2.22e-06 | 5250.9ms/step | 11.4 samples/s +Step 48360/50462 | Loss: 2.4888 | LR: 2.20e-06 | 5243.3ms/step | 11.4 samples/s +Step 48370/50462 | Loss: 2.5259 | LR: 2.18e-06 | 5252.7ms/step | 11.4 samples/s +Step 48380/50462 | Loss: 2.4924 | LR: 2.16e-06 | 5336.2ms/step | 11.2 samples/s +Step 48390/50462 | Loss: 2.5401 | LR: 2.14e-06 | 5256.4ms/step | 11.4 samples/s +Step 48400/50462 | Loss: 2.4950 | LR: 2.12e-06 | 5248.6ms/step | 11.4 samples/s +Step 48410/50462 | Loss: 2.5232 | LR: 2.10e-06 | 5250.5ms/step | 11.4 samples/s +Step 48420/50462 | Loss: 2.5001 | LR: 2.08e-06 | 5251.9ms/step | 11.4 samples/s +Step 48430/50462 | Loss: 2.4958 | LR: 2.06e-06 | 5248.6ms/step | 11.4 samples/s +Step 48440/50462 | Loss: 2.5134 | LR: 2.04e-06 | 5252.7ms/step | 11.4 samples/s +Step 48450/50462 | Loss: 2.4793 | LR: 2.02e-06 | 5250.9ms/step | 11.4 samples/s +Step 48460/50462 | Loss: 2.5265 | LR: 2.00e-06 | 5259.1ms/step | 11.4 samples/s +Step 48470/50462 | Loss: 2.5147 | LR: 1.98e-06 | 5254.2ms/step | 11.4 samples/s +Step 48480/50462 | Loss: 2.4980 | LR: 1.96e-06 | 5255.0ms/step | 11.4 samples/s +Step 48490/50462 | Loss: 2.5062 | LR: 1.94e-06 | 5248.3ms/step | 11.4 samples/s +Step 48500/50462 | Loss: 2.4746 | LR: 1.92e-06 | 5244.1ms/step | 11.4 samples/s +Validation at step 48500: Loss = 2.5184, Perplexity = 12.41 +Step 48510/50462 | Loss: 2.5078 | LR: 1.90e-06 | 7816.9ms/step | 7.7 samples/s +Step 48520/50462 | Loss: 2.5070 | LR: 1.88e-06 | 5253.1ms/step | 11.4 samples/s +Step 48530/50462 | Loss: 2.5245 | LR: 1.86e-06 | 5251.9ms/step | 11.4 samples/s +Step 48540/50462 | Loss: 2.4813 | LR: 1.84e-06 | 5252.7ms/step | 11.4 samples/s +Step 48550/50462 | Loss: 2.4915 | LR: 1.82e-06 | 5254.9ms/step | 11.4 samples/s +Step 48560/50462 | Loss: 2.5018 | LR: 1.80e-06 | 5250.9ms/step | 11.4 samples/s +Step 48570/50462 | Loss: 2.5031 | LR: 1.78e-06 | 5254.8ms/step | 11.4 samples/s +Step 48580/50462 | Loss: 2.4960 | LR: 1.77e-06 | 5247.7ms/step | 11.4 samples/s +Step 48590/50462 | Loss: 2.5204 | LR: 1.75e-06 | 5248.2ms/step | 11.4 samples/s +Step 48600/50462 | Loss: 2.5116 | LR: 1.73e-06 | 5243.9ms/step | 11.4 samples/s +Step 48610/50462 | Loss: 2.4970 | LR: 1.71e-06 | 5295.3ms/step | 11.3 samples/s +Step 48620/50462 | Loss: 2.5031 | LR: 1.69e-06 | 5250.8ms/step | 11.4 samples/s +Step 48630/50462 | Loss: 2.5250 | LR: 1.67e-06 | 5243.3ms/step | 11.4 samples/s +Step 48640/50462 | Loss: 2.5075 | LR: 1.66e-06 | 5253.5ms/step | 11.4 samples/s +Step 48650/50462 | Loss: 2.5240 | LR: 1.64e-06 | 5249.8ms/step | 11.4 samples/s +Step 48660/50462 | Loss: 2.5194 | LR: 1.62e-06 | 5237.6ms/step | 11.5 samples/s +Step 48670/50462 | Loss: 2.5209 | LR: 1.60e-06 | 5251.4ms/step | 11.4 samples/s +Step 48680/50462 | Loss: 2.5028 | LR: 1.58e-06 | 5330.6ms/step | 11.3 samples/s +Step 48690/50462 | Loss: 2.5167 | LR: 1.57e-06 | 5244.3ms/step | 11.4 samples/s +Step 48700/50462 | Loss: 2.4763 | LR: 1.55e-06 | 5253.6ms/step | 11.4 samples/s +Step 48710/50462 | Loss: 2.4995 | LR: 1.53e-06 | 5244.0ms/step | 11.4 samples/s +Step 48720/50462 | Loss: 2.5213 | LR: 1.51e-06 | 5243.9ms/step | 11.4 samples/s +Step 48730/50462 | Loss: 2.5067 | LR: 1.50e-06 | 5247.1ms/step | 11.4 samples/s +Step 48740/50462 | Loss: 2.4841 | LR: 1.48e-06 | 5242.6ms/step | 11.4 samples/s +Step 48750/50462 | Loss: 2.5275 | LR: 1.46e-06 | 5243.3ms/step | 11.4 samples/s +Validation at step 48750: Loss = 2.5183, Perplexity = 12.41 +Step 48760/50462 | Loss: 2.5379 | LR: 1.44e-06 | 7808.8ms/step | 7.7 samples/s +Step 48770/50462 | Loss: 2.4784 | LR: 1.43e-06 | 5237.1ms/step | 11.5 samples/s +Step 48780/50462 | Loss: 2.5007 | LR: 1.41e-06 | 5242.7ms/step | 11.4 samples/s +Step 48790/50462 | Loss: 2.5085 | LR: 1.39e-06 | 5254.2ms/step | 11.4 samples/s +Step 48800/50462 | Loss: 2.4880 | LR: 1.38e-06 | 5251.7ms/step | 11.4 samples/s +Step 48810/50462 | Loss: 2.4975 | LR: 1.36e-06 | 5249.1ms/step | 11.4 samples/s +Step 48820/50462 | Loss: 2.5027 | LR: 1.34e-06 | 5247.9ms/step | 11.4 samples/s +Step 48830/50462 | Loss: 2.5194 | LR: 1.33e-06 | 5232.1ms/step | 11.5 samples/s +Step 48840/50462 | Loss: 2.5320 | LR: 1.31e-06 | 5250.8ms/step | 11.4 samples/s +Step 48850/50462 | Loss: 2.5044 | LR: 1.30e-06 | 5241.8ms/step | 11.4 samples/s +Step 48860/50462 | Loss: 2.4701 | LR: 1.28e-06 | 5235.5ms/step | 11.5 samples/s +Step 48870/50462 | Loss: 2.5130 | LR: 1.26e-06 | 5245.6ms/step | 11.4 samples/s +Step 48880/50462 | Loss: 2.4950 | LR: 1.25e-06 | 5245.0ms/step | 11.4 samples/s +Step 48890/50462 | Loss: 2.5261 | LR: 1.23e-06 | 5290.0ms/step | 11.3 samples/s +Step 48900/50462 | Loss: 2.5042 | LR: 1.22e-06 | 5251.0ms/step | 11.4 samples/s +Step 48910/50462 | Loss: 2.5180 | LR: 1.20e-06 | 5256.7ms/step | 11.4 samples/s +Step 48920/50462 | Loss: 2.5050 | LR: 1.19e-06 | 5241.5ms/step | 11.4 samples/s +Step 48930/50462 | Loss: 2.5010 | LR: 1.17e-06 | 5246.4ms/step | 11.4 samples/s +Step 48940/50462 | Loss: 2.4813 | LR: 1.16e-06 | 5248.6ms/step | 11.4 samples/s +Step 48950/50462 | Loss: 2.4816 | LR: 1.14e-06 | 5250.7ms/step | 11.4 samples/s +Step 48960/50462 | Loss: 2.5128 | LR: 1.13e-06 | 5250.3ms/step | 11.4 samples/s +Step 48970/50462 | Loss: 2.4803 | LR: 1.11e-06 | 5330.7ms/step | 11.3 samples/s +Step 48980/50462 | Loss: 2.5484 | LR: 1.10e-06 | 5250.8ms/step | 11.4 samples/s +Step 48990/50462 | Loss: 2.5327 | LR: 1.08e-06 | 5250.3ms/step | 11.4 samples/s +Step 49000/50462 | Loss: 2.5145 | LR: 1.07e-06 | 5250.4ms/step | 11.4 samples/s +Validation at step 49000: Loss = 2.5182, Perplexity = 12.41 +Saved checkpoint to checkpoints/run_20260529_182827/checkpoint-49000 +Step 49010/50462 | Loss: 2.5031 | LR: 1.05e-06 | 8666.1ms/step | 6.9 samples/s +Step 49020/50462 | Loss: 2.5109 | LR: 1.04e-06 | 5243.3ms/step | 11.4 samples/s +Step 49030/50462 | Loss: 2.5119 | LR: 1.02e-06 | 5247.9ms/step | 11.4 samples/s +Step 49040/50462 | Loss: 2.5084 | LR: 1.01e-06 | 5249.5ms/step | 11.4 samples/s +Step 49050/50462 | Loss: 2.5082 | LR: 9.95e-07 | 5249.7ms/step | 11.4 samples/s +Step 49060/50462 | Loss: 2.5291 | LR: 9.81e-07 | 5251.6ms/step | 11.4 samples/s +Step 49070/50462 | Loss: 2.5260 | LR: 9.67e-07 | 5253.8ms/step | 11.4 samples/s +Step 49080/50462 | Loss: 2.5281 | LR: 9.53e-07 | 5240.5ms/step | 11.4 samples/s +Step 49090/50462 | Loss: 2.4817 | LR: 9.39e-07 | 5250.7ms/step | 11.4 samples/s +Step 49100/50462 | Loss: 2.5309 | LR: 9.25e-07 | 5246.9ms/step | 11.4 samples/s +Step 49110/50462 | Loss: 2.5160 | LR: 9.12e-07 | 5252.1ms/step | 11.4 samples/s +Step 49120/50462 | Loss: 2.4852 | LR: 8.99e-07 | 5252.7ms/step | 11.4 samples/s +Step 49130/50462 | Loss: 2.5150 | LR: 8.85e-07 | 5252.3ms/step | 11.4 samples/s +Step 49140/50462 | Loss: 2.4865 | LR: 8.72e-07 | 5249.8ms/step | 11.4 samples/s +Step 49150/50462 | Loss: 2.5175 | LR: 8.59e-07 | 5250.6ms/step | 11.4 samples/s +Step 49160/50462 | Loss: 2.5079 | LR: 8.46e-07 | 5285.7ms/step | 11.4 samples/s +Step 49170/50462 | Loss: 2.5057 | LR: 8.33e-07 | 5247.1ms/step | 11.4 samples/s +Step 49180/50462 | Loss: 2.4958 | LR: 8.20e-07 | 5253.8ms/step | 11.4 samples/s +Step 49190/50462 | Loss: 2.4851 | LR: 8.07e-07 | 5244.5ms/step | 11.4 samples/s +Step 49200/50462 | Loss: 2.5382 | LR: 7.95e-07 | 5250.8ms/step | 11.4 samples/s +Step 49210/50462 | Loss: 2.5089 | LR: 7.82e-07 | 5239.1ms/step | 11.5 samples/s +Step 49220/50462 | Loss: 2.5194 | LR: 7.70e-07 | 5245.1ms/step | 11.4 samples/s +Step 49230/50462 | Loss: 2.5207 | LR: 7.57e-07 | 5252.4ms/step | 11.4 samples/s +Step 49240/50462 | Loss: 2.5109 | LR: 7.45e-07 | 5250.8ms/step | 11.4 samples/s +Step 49250/50462 | Loss: 2.5049 | LR: 7.33e-07 | 5242.5ms/step | 11.4 samples/s +Validation at step 49250: Loss = 2.5182, Perplexity = 12.41 +Step 49260/50462 | Loss: 2.4777 | LR: 7.21e-07 | 7897.9ms/step | 7.6 samples/s +Step 49270/50462 | Loss: 2.4964 | LR: 7.09e-07 | 5235.6ms/step | 11.5 samples/s +Step 49280/50462 | Loss: 2.4807 | LR: 6.97e-07 | 5247.2ms/step | 11.4 samples/s +Step 49290/50462 | Loss: 2.5015 | LR: 6.85e-07 | 5245.5ms/step | 11.4 samples/s +Step 49300/50462 | Loss: 2.4988 | LR: 6.74e-07 | 5250.1ms/step | 11.4 samples/s +Step 49310/50462 | Loss: 2.5059 | LR: 6.62e-07 | 5246.0ms/step | 11.4 samples/s +Step 49320/50462 | Loss: 2.4939 | LR: 6.51e-07 | 5250.4ms/step | 11.4 samples/s +Step 49330/50462 | Loss: 2.5025 | LR: 6.39e-07 | 5245.8ms/step | 11.4 samples/s +Step 49340/50462 | Loss: 2.5122 | LR: 6.28e-07 | 5255.0ms/step | 11.4 samples/s +Step 49350/50462 | Loss: 2.5147 | LR: 6.17e-07 | 5245.9ms/step | 11.4 samples/s +Step 49360/50462 | Loss: 2.5255 | LR: 6.06e-07 | 5253.5ms/step | 11.4 samples/s +Step 49370/50462 | Loss: 2.5077 | LR: 5.95e-07 | 5253.4ms/step | 11.4 samples/s +Step 49380/50462 | Loss: 2.5202 | LR: 5.84e-07 | 5256.5ms/step | 11.4 samples/s +Step 49390/50462 | Loss: 2.5193 | LR: 5.73e-07 | 5253.0ms/step | 11.4 samples/s +Step 49400/50462 | Loss: 2.5172 | LR: 5.63e-07 | 5247.2ms/step | 11.4 samples/s +Step 49410/50462 | Loss: 2.5100 | LR: 5.52e-07 | 5256.5ms/step | 11.4 samples/s +Step 49420/50462 | Loss: 2.5070 | LR: 5.42e-07 | 5251.2ms/step | 11.4 samples/s +Step 49430/50462 | Loss: 2.4866 | LR: 5.31e-07 | 5248.1ms/step | 11.4 samples/s +Step 49440/50462 | Loss: 2.4953 | LR: 5.21e-07 | 5288.2ms/step | 11.3 samples/s +Step 49450/50462 | Loss: 2.4830 | LR: 5.11e-07 | 5231.1ms/step | 11.5 samples/s +Step 49460/50462 | Loss: 2.5169 | LR: 5.01e-07 | 5240.5ms/step | 11.4 samples/s +Step 49470/50462 | Loss: 2.5186 | LR: 4.91e-07 | 5234.9ms/step | 11.5 samples/s +Step 49480/50462 | Loss: 2.5116 | LR: 4.81e-07 | 5236.3ms/step | 11.5 samples/s +Step 49490/50462 | Loss: 2.4924 | LR: 4.72e-07 | 5238.8ms/step | 11.5 samples/s +Step 49500/50462 | Loss: 2.5127 | LR: 4.62e-07 | 5251.6ms/step | 11.4 samples/s +Validation at step 49500: Loss = 2.5182, Perplexity = 12.41 +Step 49510/50462 | Loss: 2.5222 | LR: 4.52e-07 | 7794.9ms/step | 7.7 samples/s +Step 49520/50462 | Loss: 2.5022 | LR: 4.43e-07 | 5225.1ms/step | 11.5 samples/s +Step 49530/50462 | Loss: 2.5061 | LR: 4.34e-07 | 5235.0ms/step | 11.5 samples/s +Step 49540/50462 | Loss: 2.4980 | LR: 4.24e-07 | 5239.7ms/step | 11.5 samples/s +Step 49550/50462 | Loss: 2.5049 | LR: 4.15e-07 | 5322.5ms/step | 11.3 samples/s +Step 49560/50462 | Loss: 2.5199 | LR: 4.06e-07 | 5251.4ms/step | 11.4 samples/s +Step 49570/50462 | Loss: 2.5226 | LR: 3.97e-07 | 5250.9ms/step | 11.4 samples/s +Step 49580/50462 | Loss: 2.5221 | LR: 3.88e-07 | 5249.4ms/step | 11.4 samples/s +Step 49590/50462 | Loss: 2.4956 | LR: 3.79e-07 | 5250.9ms/step | 11.4 samples/s +Step 49600/50462 | Loss: 2.5266 | LR: 3.71e-07 | 5242.6ms/step | 11.4 samples/s +Step 49610/50462 | Loss: 2.5066 | LR: 3.62e-07 | 5245.4ms/step | 11.4 samples/s +Step 49620/50462 | Loss: 2.5053 | LR: 3.54e-07 | 5246.3ms/step | 11.4 samples/s +Step 49630/50462 | Loss: 2.5050 | LR: 3.45e-07 | 5243.3ms/step | 11.4 samples/s +Step 49640/50462 | Loss: 2.5081 | LR: 3.37e-07 | 5245.2ms/step | 11.4 samples/s +Step 49650/50462 | Loss: 2.5114 | LR: 3.29e-07 | 5249.8ms/step | 11.4 samples/s +Step 49660/50462 | Loss: 2.4942 | LR: 3.21e-07 | 5240.4ms/step | 11.4 samples/s +Step 49670/50462 | Loss: 2.4816 | LR: 3.13e-07 | 5249.6ms/step | 11.4 samples/s +Step 49680/50462 | Loss: 2.5262 | LR: 3.05e-07 | 5250.3ms/step | 11.4 samples/s +Step 49690/50462 | Loss: 2.5438 | LR: 2.97e-07 | 5245.5ms/step | 11.4 samples/s +Step 49700/50462 | Loss: 2.5157 | LR: 2.90e-07 | 5231.2ms/step | 11.5 samples/s +Step 49710/50462 | Loss: 2.5097 | LR: 2.82e-07 | 5248.3ms/step | 11.4 samples/s +Step 49720/50462 | Loss: 2.5002 | LR: 2.75e-07 | 5286.7ms/step | 11.3 samples/s +Step 49730/50462 | Loss: 2.5158 | LR: 2.67e-07 | 5249.8ms/step | 11.4 samples/s +Step 49740/50462 | Loss: 2.5361 | LR: 2.60e-07 | 5236.6ms/step | 11.5 samples/s +Step 49750/50462 | Loss: 2.5083 | LR: 2.53e-07 | 5247.2ms/step | 11.4 samples/s +Validation at step 49750: Loss = 2.5182, Perplexity = 12.41 +Step 49760/50462 | Loss: 2.5257 | LR: 2.46e-07 | 7802.5ms/step | 7.7 samples/s +Step 49770/50462 | Loss: 2.5144 | LR: 2.39e-07 | 5239.6ms/step | 11.5 samples/s +Step 49780/50462 | Loss: 2.4989 | LR: 2.32e-07 | 5250.0ms/step | 11.4 samples/s +Step 49790/50462 | Loss: 2.4928 | LR: 2.25e-07 | 5246.3ms/step | 11.4 samples/s +Step 49800/50462 | Loss: 2.5350 | LR: 2.19e-07 | 5246.1ms/step | 11.4 samples/s +Step 49810/50462 | Loss: 2.5217 | LR: 2.12e-07 | 5250.6ms/step | 11.4 samples/s +Step 49820/50462 | Loss: 2.5196 | LR: 2.06e-07 | 5249.7ms/step | 11.4 samples/s +Step 49830/50462 | Loss: 2.5084 | LR: 1.99e-07 | 5242.4ms/step | 11.4 samples/s +Step 49840/50462 | Loss: 2.5228 | LR: 1.93e-07 | 5312.9ms/step | 11.3 samples/s +Step 49850/50462 | Loss: 2.4998 | LR: 1.87e-07 | 5235.0ms/step | 11.5 samples/s +Step 49860/50462 | Loss: 2.4994 | LR: 1.81e-07 | 5246.6ms/step | 11.4 samples/s +Step 49870/50462 | Loss: 2.5146 | LR: 1.75e-07 | 5251.0ms/step | 11.4 samples/s +Step 49880/50462 | Loss: 2.4993 | LR: 1.69e-07 | 5249.9ms/step | 11.4 samples/s +Step 49890/50462 | Loss: 2.5089 | LR: 1.63e-07 | 5249.0ms/step | 11.4 samples/s +Step 49900/50462 | Loss: 2.5215 | LR: 1.58e-07 | 5250.3ms/step | 11.4 samples/s +Step 49910/50462 | Loss: 2.5079 | LR: 1.52e-07 | 5250.6ms/step | 11.4 samples/s +Step 49920/50462 | Loss: 2.5219 | LR: 1.47e-07 | 5249.8ms/step | 11.4 samples/s +Step 49930/50462 | Loss: 2.4929 | LR: 1.41e-07 | 5247.3ms/step | 11.4 samples/s +Step 49940/50462 | Loss: 2.5051 | LR: 1.36e-07 | 5248.2ms/step | 11.4 samples/s +Step 49950/50462 | Loss: 2.4658 | LR: 1.31e-07 | 5246.9ms/step | 11.4 samples/s +Step 49960/50462 | Loss: 2.4954 | LR: 1.26e-07 | 5250.7ms/step | 11.4 samples/s +Step 49970/50462 | Loss: 2.4869 | LR: 1.21e-07 | 5250.1ms/step | 11.4 samples/s +Step 49980/50462 | Loss: 2.5022 | LR: 1.16e-07 | 5246.9ms/step | 11.4 samples/s +Step 49990/50462 | Loss: 2.4800 | LR: 1.11e-07 | 5244.7ms/step | 11.4 samples/s +Step 50000/50462 | Loss: 2.4907 | LR: 1.07e-07 | 5287.6ms/step | 11.3 samples/s +Validation at step 50000: Loss = 2.5182, Perplexity = 12.41 +Saved checkpoint to checkpoints/run_20260529_182827/checkpoint-50000 +Step 50010/50462 | Loss: 2.4861 | LR: 1.02e-07 | 8701.0ms/step | 6.9 samples/s +Step 50020/50462 | Loss: 2.4939 | LR: 9.75e-08 | 5243.0ms/step | 11.4 samples/s +Step 50030/50462 | Loss: 2.5242 | LR: 9.32e-08 | 5253.8ms/step | 11.4 samples/s +Step 50040/50462 | Loss: 2.5542 | LR: 8.89e-08 | 5247.4ms/step | 11.4 samples/s +Step 50050/50462 | Loss: 2.5080 | LR: 8.47e-08 | 5248.4ms/step | 11.4 samples/s +Step 50060/50462 | Loss: 2.5007 | LR: 8.07e-08 | 5246.5ms/step | 11.4 samples/s +Step 50070/50462 | Loss: 2.5212 | LR: 7.67e-08 | 5247.2ms/step | 11.4 samples/s +Step 50080/50462 | Loss: 2.4925 | LR: 7.28e-08 | 5244.9ms/step | 11.4 samples/s +Step 50090/50462 | Loss: 2.4856 | LR: 6.91e-08 | 5253.1ms/step | 11.4 samples/s +Step 50100/50462 | Loss: 2.4915 | LR: 6.54e-08 | 5249.7ms/step | 11.4 samples/s +Step 50110/50462 | Loss: 2.5314 | LR: 6.19e-08 | 5255.8ms/step | 11.4 samples/s +Step 50120/50462 | Loss: 2.5123 | LR: 5.84e-08 | 5247.0ms/step | 11.4 samples/s +Step 50130/50462 | Loss: 2.5125 | LR: 5.50e-08 | 5334.4ms/step | 11.2 samples/s +Step 50140/50462 | Loss: 2.5308 | LR: 5.18e-08 | 5248.4ms/step | 11.4 samples/s +Step 50150/50462 | Loss: 2.4860 | LR: 4.86e-08 | 5250.9ms/step | 11.4 samples/s +Step 50160/50462 | Loss: 2.5018 | LR: 4.55e-08 | 5251.6ms/step | 11.4 samples/s +Step 50170/50462 | Loss: 2.4697 | LR: 4.26e-08 | 5247.5ms/step | 11.4 samples/s +Step 50180/50462 | Loss: 2.4745 | LR: 3.97e-08 | 5254.5ms/step | 11.4 samples/s +Step 50190/50462 | Loss: 2.5220 | LR: 3.69e-08 | 5250.2ms/step | 11.4 samples/s +Step 50200/50462 | Loss: 2.4912 | LR: 3.43e-08 | 5250.2ms/step | 11.4 samples/s +Step 50210/50462 | Loss: 2.5166 | LR: 3.17e-08 | 5252.6ms/step | 11.4 samples/s +Step 50220/50462 | Loss: 2.5125 | LR: 2.92e-08 | 5253.5ms/step | 11.4 samples/s +Step 50230/50462 | Loss: 2.5299 | LR: 2.69e-08 | 5250.6ms/step | 11.4 samples/s +Step 50240/50462 | Loss: 2.4942 | LR: 2.46e-08 | 5250.6ms/step | 11.4 samples/s +Step 50250/50462 | Loss: 2.4882 | LR: 2.24e-08 | 5249.7ms/step | 11.4 samples/s +Validation at step 50250: Loss = 2.5182, Perplexity = 12.41 +Step 50260/50462 | Loss: 2.5348 | LR: 2.04e-08 | 7818.4ms/step | 7.7 samples/s +Step 50270/50462 | Loss: 2.5470 | LR: 1.84e-08 | 5249.7ms/step | 11.4 samples/s +Step 50280/50462 | Loss: 2.5178 | LR: 1.65e-08 | 5295.5ms/step | 11.3 samples/s +Step 50290/50462 | Loss: 2.5316 | LR: 1.48e-08 | 5254.1ms/step | 11.4 samples/s +Step 50300/50462 | Loss: 2.4866 | LR: 1.31e-08 | 5252.5ms/step | 11.4 samples/s +Step 50310/50462 | Loss: 2.4822 | LR: 1.15e-08 | 5252.2ms/step | 11.4 samples/s +Step 50320/50462 | Loss: 2.5020 | LR: 1.01e-08 | 5247.2ms/step | 11.4 samples/s +Step 50330/50462 | Loss: 2.5301 | LR: 8.70e-09 | 5251.7ms/step | 11.4 samples/s +Step 50340/50462 | Loss: 2.5103 | LR: 7.43e-09 | 5249.8ms/step | 11.4 samples/s +Step 50350/50462 | Loss: 2.4918 | LR: 6.26e-09 | 5252.5ms/step | 11.4 samples/s +Step 50360/50462 | Loss: 2.5145 | LR: 5.19e-09 | 5254.1ms/step | 11.4 samples/s +Step 50370/50462 | Loss: 2.4905 | LR: 4.23e-09 | 5247.6ms/step | 11.4 samples/s +Step 50380/50462 | Loss: 2.4830 | LR: 3.36e-09 | 5247.5ms/step | 11.4 samples/s +Step 50390/50462 | Loss: 2.5222 | LR: 2.59e-09 | 5247.5ms/step | 11.4 samples/s +Step 50400/50462 | Loss: 2.5060 | LR: 1.92e-09 | 5248.9ms/step | 11.4 samples/s +Step 50410/50462 | Loss: 2.4955 | LR: 1.35e-09 | 5252.4ms/step | 11.4 samples/s +Step 50420/50462 | Loss: 2.5232 | LR: 8.81e-10 | 5285.6ms/step | 11.4 samples/s +Step 50430/50462 | Loss: 2.5133 | LR: 5.11e-10 | 5287.6ms/step | 11.3 samples/s +Step 50440/50462 | Loss: 2.4837 | LR: 2.42e-10 | 5250.6ms/step | 11.4 samples/s +Step 50450/50462 | Loss: 2.5245 | LR: 7.19e-11 | 5250.8ms/step | 11.4 samples/s +Step 50460/50462 | Loss: 2.4978 | LR: 2.00e-12 | 5247.1ms/step | 11.4 samples/s +Saved checkpoint to checkpoints/run_20260529_182827/checkpoint-50462 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log; uploading config.yaml +wandb: uploading history steps 246-246, summary, console lines 259-259 +wandb: +wandb: Run history: +wandb: eval/loss █▃▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▃▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ████████████▁███████████████████████████ +wandb: speed/steps_per_sec ████████████████████████████████▁███████ +wandb: train/epoch ▁▁▁▁▂▂▂▂▃▃▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇████ +wandb: train/global_step ▁▁▁▁▁▂▂▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▆▆▆▆▇▇▇▇▇▇█████ +wandb: train/learning_rate ██▇▇▇▇▇▆▅▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁ +wandb: train/loss ▇▅▃▁▂▅▄▄▅▄▅▃▄▅▄▁▁▆▅▄▄▁▃▄▅▅▂▃▃▆▁▄▇▄▂▅▂█▆▆ +wandb: +wandb: Run summary: +wandb: eval/loss 2.51822 +wandb: eval/perplexity 12.40654 +wandb: speed/ms_per_step 5247.137 +wandb: speed/samples_per_sec 11.43481 +wandb: speed/steps_per_sec 0.19058 +wandb: train/epoch 0.99996 +wandb: train/global_step 50460 +wandb: train/learning_rate 0.0 +wandb: train/loss 2.49778 +wandb: +wandb: 🚀 View run dense continued 48k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/wbfo0o07 +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260529_184637-wbfo0o07/logs +Training complete! diff --git a/epfl_final/log_files/1mask_continued.log b/epfl_final/log_files/1mask_continued.log new file mode 100644 index 0000000000000000000000000000000000000000..c74d455129a836bdf8fee4f404a54c5c135d5b03 --- /dev/null +++ b/epfl_final/log_files/1mask_continued.log @@ -0,0 +1,425 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260615_223806 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260615_223806", + "resume_from_checkpoint": "checkpoints/run_20260614_032718/checkpoint-40000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 40k", + "wandb_tags": [ + "qwen3", + "1.2b", + "dense-resume" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: checkpoints/run_20260614_032718/checkpoint-40000 +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) + +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from checkpoints/run_20260614_032718/checkpoint-40000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 40000 (epoch 0, micro_step 400000) +Resuming training from step 40000 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.2 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260615_224051-7jul6zqc +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense continued 40k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/7jul6zqc +Starting training... +Validation at step 40000: Loss = 2.5322, Perplexity = 12.58 +Switching attention mode for step 40001: dense (dense step interval) +Step 40010/50462 | Loss: 2.5463 | LR: 5.26e-05 | 67266.9ms/step | 0.9 samples/s +Step 40020/50462 | Loss: 2.5609 | LR: 5.25e-05 | 4208.1ms/step | 14.3 samples/s +Step 40030/50462 | Loss: 2.5036 | LR: 5.24e-05 | 4203.7ms/step | 14.3 samples/s +Step 40040/50462 | Loss: 2.5354 | LR: 5.23e-05 | 4209.2ms/step | 14.3 samples/s +Step 40050/50462 | Loss: 2.5220 | LR: 5.22e-05 | 4200.9ms/step | 14.3 samples/s +Step 40060/50462 | Loss: 2.5152 | LR: 5.21e-05 | 4206.9ms/step | 14.3 samples/s +Step 40070/50462 | Loss: 2.5253 | LR: 5.20e-05 | 4213.0ms/step | 14.2 samples/s +Step 40080/50462 | Loss: 2.4990 | LR: 5.19e-05 | 4297.2ms/step | 14.0 samples/s +Step 40090/50462 | Loss: 2.5332 | LR: 5.18e-05 | 4211.9ms/step | 14.2 samples/s +Step 40100/50462 | Loss: 2.5294 | LR: 5.17e-05 | 4213.1ms/step | 14.2 samples/s +Step 40110/50462 | Loss: 2.5173 | LR: 5.16e-05 | 4207.5ms/step | 14.3 samples/s +Step 40120/50462 | Loss: 2.5305 | LR: 5.15e-05 | 4205.5ms/step | 14.3 samples/s +Step 40130/50462 | Loss: 2.5134 | LR: 5.14e-05 | 4204.9ms/step | 14.3 samples/s +Step 40140/50462 | Loss: 2.5095 | LR: 5.13e-05 | 4210.1ms/step | 14.3 samples/s +Step 40150/50462 | Loss: 2.4877 | LR: 5.12e-05 | 4340.5ms/step | 13.8 samples/s +Step 40160/50462 | Loss: 2.4844 | LR: 5.11e-05 | 4203.2ms/step | 14.3 samples/s +Step 40170/50462 | Loss: 2.5237 | LR: 5.10e-05 | 4206.2ms/step | 14.3 samples/s +Step 40180/50462 | Loss: 2.5255 | LR: 5.09e-05 | 4206.7ms/step | 14.3 samples/s +Step 40190/50462 | Loss: 2.5225 | LR: 5.09e-05 | 4199.0ms/step | 14.3 samples/s +Step 40200/50462 | Loss: 2.5299 | LR: 5.08e-05 | 4198.2ms/step | 14.3 samples/s +Step 40210/50462 | Loss: 2.5353 | LR: 5.07e-05 | 4202.0ms/step | 14.3 samples/s +Step 40220/50462 | Loss: 2.5356 | LR: 5.06e-05 | 4211.0ms/step | 14.2 samples/s +Step 40230/50462 | Loss: 2.5268 | LR: 5.05e-05 | 4204.3ms/step | 14.3 samples/s +Step 40240/50462 | Loss: 2.5581 | LR: 5.04e-05 | 4206.9ms/step | 14.3 samples/s +Step 40250/50462 | Loss: 2.5169 | LR: 5.03e-05 | 4203.4ms/step | 14.3 samples/s +Validation at step 40250: Loss = 2.5282, Perplexity = 12.53 +Step 40260/50462 | Loss: 2.5224 | LR: 5.02e-05 | 6201.1ms/step | 9.7 samples/s +Step 40270/50462 | Loss: 2.4979 | LR: 5.01e-05 | 4201.0ms/step | 14.3 samples/s +Step 40280/50462 | Loss: 2.5227 | LR: 5.00e-05 | 4192.4ms/step | 14.3 samples/s +Step 40290/50462 | Loss: 2.5202 | LR: 4.99e-05 | 4194.9ms/step | 14.3 samples/s +Step 40300/50462 | Loss: 2.5158 | LR: 4.98e-05 | 4205.7ms/step | 14.3 samples/s +Step 40310/50462 | Loss: 2.5016 | LR: 4.97e-05 | 4205.0ms/step | 14.3 samples/s +Step 40320/50462 | Loss: 2.5193 | LR: 4.96e-05 | 4197.5ms/step | 14.3 samples/s +Step 40330/50462 | Loss: 2.5423 | LR: 4.95e-05 | 4201.5ms/step | 14.3 samples/s +Step 40340/50462 | Loss: 2.5020 | LR: 4.94e-05 | 4204.9ms/step | 14.3 samples/s +Step 40350/50462 | Loss: 2.4937 | LR: 4.93e-05 | 4196.8ms/step | 14.3 samples/s +Step 40360/50462 | Loss: 2.5224 | LR: 4.92e-05 | 4205.8ms/step | 14.3 samples/s +Step 40370/50462 | Loss: 2.5357 | LR: 4.91e-05 | 4205.5ms/step | 14.3 samples/s +Step 40380/50462 | Loss: 2.5213 | LR: 4.90e-05 | 4254.5ms/step | 14.1 samples/s +Step 40390/50462 | Loss: 2.5166 | LR: 4.90e-05 | 4206.9ms/step | 14.3 samples/s +Step 40400/50462 | Loss: 2.5290 | LR: 4.89e-05 | 4204.8ms/step | 14.3 samples/s +Step 40410/50462 | Loss: 2.5430 | LR: 4.88e-05 | 4203.8ms/step | 14.3 samples/s +Step 40420/50462 | Loss: 2.5418 | LR: 4.87e-05 | 4206.1ms/step | 14.3 samples/s +Step 40430/50462 | Loss: 2.5087 | LR: 4.86e-05 | 4206.6ms/step | 14.3 samples/s +Step 40440/50462 | Loss: 2.5303 | LR: 4.85e-05 | 4202.1ms/step | 14.3 samples/s +Step 40450/50462 | Loss: 2.5168 | LR: 4.84e-05 | 4321.7ms/step | 13.9 samples/s +Step 40460/50462 | Loss: 2.5494 | LR: 4.83e-05 | 4205.4ms/step | 14.3 samples/s +Step 40470/50462 | Loss: 2.5037 | LR: 4.82e-05 | 4196.3ms/step | 14.3 samples/s +Step 40480/50462 | Loss: 2.5148 | LR: 4.81e-05 | 4202.1ms/step | 14.3 samples/s +Step 40490/50462 | Loss: 2.5302 | LR: 4.80e-05 | 4203.1ms/step | 14.3 samples/s +Step 40500/50462 | Loss: 2.5209 | LR: 4.79e-05 | 4202.5ms/step | 14.3 samples/s +Validation at step 40500: Loss = 2.5276, Perplexity = 12.52 +Step 40510/50462 | Loss: 2.5184 | LR: 4.78e-05 | 6210.0ms/step | 9.7 samples/s +Step 40520/50462 | Loss: 2.5059 | LR: 4.77e-05 | 4204.5ms/step | 14.3 samples/s +Step 40530/50462 | Loss: 2.5088 | LR: 4.76e-05 | 4202.2ms/step | 14.3 samples/s +Step 40540/50462 | Loss: 2.4815 | LR: 4.76e-05 | 4202.0ms/step | 14.3 samples/s +Step 40550/50462 | Loss: 2.5367 | LR: 4.75e-05 | 4202.5ms/step | 14.3 samples/s +Step 40560/50462 | Loss: 2.5279 | LR: 4.74e-05 | 4211.4ms/step | 14.2 samples/s +Step 40570/50462 | Loss: 2.5038 | LR: 4.73e-05 | 4199.2ms/step | 14.3 samples/s +Step 40580/50462 | Loss: 2.5326 | LR: 4.72e-05 | 4200.6ms/step | 14.3 samples/s +Step 40590/50462 | Loss: 2.5149 | LR: 4.71e-05 | 4200.2ms/step | 14.3 samples/s +Step 40600/50462 | Loss: 2.5159 | LR: 4.70e-05 | 4201.7ms/step | 14.3 samples/s +Step 40610/50462 | Loss: 2.5341 | LR: 4.69e-05 | 4198.0ms/step | 14.3 samples/s +Step 40620/50462 | Loss: 2.4919 | LR: 4.68e-05 | 4199.5ms/step | 14.3 samples/s +Step 40630/50462 | Loss: 2.5126 | LR: 4.67e-05 | 4202.7ms/step | 14.3 samples/s +Step 40640/50462 | Loss: 2.5242 | LR: 4.66e-05 | 4208.7ms/step | 14.3 samples/s +Step 40650/50462 | Loss: 2.5230 | LR: 4.65e-05 | 4198.2ms/step | 14.3 samples/s +Step 40660/50462 | Loss: 2.5352 | LR: 4.64e-05 | 4199.3ms/step | 14.3 samples/s +Step 40670/50462 | Loss: 2.5120 | LR: 4.64e-05 | 4200.4ms/step | 14.3 samples/s +Step 40680/50462 | Loss: 2.5164 | LR: 4.63e-05 | 4257.1ms/step | 14.1 samples/s +Step 40690/50462 | Loss: 2.5124 | LR: 4.62e-05 | 4198.2ms/step | 14.3 samples/s +Step 40700/50462 | Loss: 2.4842 | LR: 4.61e-05 | 4204.3ms/step | 14.3 samples/s +Step 40710/50462 | Loss: 2.5247 | LR: 4.60e-05 | 4203.9ms/step | 14.3 samples/s +Step 40720/50462 | Loss: 2.4916 | LR: 4.59e-05 | 4203.5ms/step | 14.3 samples/s +Step 40730/50462 | Loss: 2.5023 | LR: 4.58e-05 | 4194.3ms/step | 14.3 samples/s +Step 40740/50462 | Loss: 2.4982 | LR: 4.57e-05 | 4195.7ms/step | 14.3 samples/s +Step 40750/50462 | Loss: 2.5095 | LR: 4.56e-05 | 4318.3ms/step | 13.9 samples/s +Validation at step 40750: Loss = 2.5271, Perplexity = 12.52 +Step 40760/50462 | Loss: 2.5332 | LR: 4.55e-05 | 6209.6ms/step | 9.7 samples/s +Step 40770/50462 | Loss: 2.5152 | LR: 4.54e-05 | 4200.0ms/step | 14.3 samples/s +Step 40780/50462 | Loss: 2.5297 | LR: 4.54e-05 | 4200.8ms/step | 14.3 samples/s +Step 40790/50462 | Loss: 2.5147 | LR: 4.53e-05 | 4199.2ms/step | 14.3 samples/s +Step 40800/50462 | Loss: 2.5241 | LR: 4.52e-05 | 4200.0ms/step | 14.3 samples/s +Step 40810/50462 | Loss: 2.4814 | LR: 4.51e-05 | 4202.3ms/step | 14.3 samples/s +Step 40820/50462 | Loss: 2.5553 | LR: 4.50e-05 | 4201.9ms/step | 14.3 samples/s +Step 40830/50462 | Loss: 2.4696 | LR: 4.49e-05 | 4201.8ms/step | 14.3 samples/s +Step 40840/50462 | Loss: 2.4939 | LR: 4.48e-05 | 4208.0ms/step | 14.3 samples/s +Step 40850/50462 | Loss: 2.5362 | LR: 4.47e-05 | 4194.0ms/step | 14.3 samples/s +Step 40860/50462 | Loss: 2.4831 | LR: 4.46e-05 | 4196.2ms/step | 14.3 samples/s +Step 40870/50462 | Loss: 2.5381 | LR: 4.45e-05 | 4200.0ms/step | 14.3 samples/s +Step 40880/50462 | Loss: 2.5449 | LR: 4.45e-05 | 4199.8ms/step | 14.3 samples/s +Step 40890/50462 | Loss: 2.4778 | LR: 4.44e-05 | 4207.1ms/step | 14.3 samples/s +Step 40900/50462 | Loss: 2.5016 | LR: 4.43e-05 | 4207.0ms/step | 14.3 samples/s +Step 40910/50462 | Loss: 2.5231 | LR: 4.42e-05 | 4202.0ms/step | 14.3 samples/s +Step 40920/50462 | Loss: 2.5317 | LR: 4.41e-05 | 4200.0ms/step | 14.3 samples/s +Step 40930/50462 | Loss: 2.5232 | LR: 4.40e-05 | 4202.2ms/step | 14.3 samples/s +Step 40940/50462 | Loss: 2.5401 | LR: 4.39e-05 | 4205.9ms/step | 14.3 samples/s +Step 40950/50462 | Loss: 2.4941 | LR: 4.38e-05 | 4201.9ms/step | 14.3 samples/s +Step 40960/50462 | Loss: 2.5237 | LR: 4.37e-05 | 4200.0ms/step | 14.3 samples/s +Step 40970/50462 | Loss: 2.5137 | LR: 4.36e-05 | 4258.2ms/step | 14.1 samples/s +Step 40980/50462 | Loss: 2.5148 | LR: 4.36e-05 | 4200.1ms/step | 14.3 samples/s +Step 40990/50462 | Loss: 2.4829 | LR: 4.35e-05 | 4199.9ms/step | 14.3 samples/s +Step 41000/50462 | Loss: 2.5280 | LR: 4.34e-05 | 4201.9ms/step | 14.3 samples/s +Validation at step 41000: Loss = 2.5267, Perplexity = 12.51 +Saved checkpoint to checkpoints/run_20260615_223806/checkpoint-41000 +Step 41010/50462 | Loss: 2.5342 | LR: 4.33e-05 | 7184.0ms/step | 8.4 samples/s +Step 41020/50462 | Loss: 2.5293 | LR: 4.32e-05 | 4199.8ms/step | 14.3 samples/s +Step 41030/50462 | Loss: 2.5324 | LR: 4.31e-05 | 4199.9ms/step | 14.3 samples/s +Step 41040/50462 | Loss: 2.4951 | LR: 4.30e-05 | 4200.2ms/step | 14.3 samples/s +Step 41050/50462 | Loss: 2.5257 | LR: 4.29e-05 | 4328.5ms/step | 13.9 samples/s +Step 41060/50462 | Loss: 2.5307 | LR: 4.28e-05 | 4195.5ms/step | 14.3 samples/s +Step 41070/50462 | Loss: 2.5094 | LR: 4.28e-05 | 4197.7ms/step | 14.3 samples/s +Step 41080/50462 | Loss: 2.5031 | LR: 4.27e-05 | 4192.2ms/step | 14.3 samples/s +Step 41090/50462 | Loss: 2.4810 | LR: 4.26e-05 | 4200.0ms/step | 14.3 samples/s +Step 41100/50462 | Loss: 2.5153 | LR: 4.25e-05 | 4200.1ms/step | 14.3 samples/s +Step 41110/50462 | Loss: 2.5200 | LR: 4.24e-05 | 4199.9ms/step | 14.3 samples/s +Step 41120/50462 | Loss: 2.5093 | LR: 4.23e-05 | 4201.0ms/step | 14.3 samples/s +Step 41130/50462 | Loss: 2.5327 | LR: 4.22e-05 | 4205.1ms/step | 14.3 samples/s +Step 41140/50462 | Loss: 2.5067 | LR: 4.21e-05 | 4199.9ms/step | 14.3 samples/s +Step 41150/50462 | Loss: 2.5203 | LR: 4.21e-05 | 4200.2ms/step | 14.3 samples/s +Step 41160/50462 | Loss: 2.5176 | LR: 4.20e-05 | 4201.9ms/step | 14.3 samples/s +Step 41170/50462 | Loss: 2.5230 | LR: 4.19e-05 | 4202.0ms/step | 14.3 samples/s +Step 41180/50462 | Loss: 2.5073 | LR: 4.18e-05 | 4201.5ms/step | 14.3 samples/s +Step 41190/50462 | Loss: 2.4674 | LR: 4.17e-05 | 4196.3ms/step | 14.3 samples/s +Step 41200/50462 | Loss: 2.5034 | LR: 4.16e-05 | 4200.6ms/step | 14.3 samples/s +Step 41210/50462 | Loss: 2.4810 | LR: 4.15e-05 | 4204.4ms/step | 14.3 samples/s +Step 41220/50462 | Loss: 2.4977 | LR: 4.14e-05 | 4201.1ms/step | 14.3 samples/s +Step 41230/50462 | Loss: 2.5192 | LR: 4.14e-05 | 4194.0ms/step | 14.3 samples/s +Step 41240/50462 | Loss: 2.5090 | LR: 4.13e-05 | 4202.0ms/step | 14.3 samples/s +Step 41250/50462 | Loss: 2.5236 | LR: 4.12e-05 | 4207.1ms/step | 14.3 samples/s +Validation at step 41250: Loss = 2.5265, Perplexity = 12.51 +Step 41260/50462 | Loss: 2.5193 | LR: 4.11e-05 | 6205.2ms/step | 9.7 samples/s +Step 41270/50462 | Loss: 2.4936 | LR: 4.10e-05 | 4249.1ms/step | 14.1 samples/s +Step 41280/50462 | Loss: 2.4717 | LR: 4.09e-05 | 4200.3ms/step | 14.3 samples/s +Step 41290/50462 | Loss: 2.5333 | LR: 4.08e-05 | 4195.8ms/step | 14.3 samples/s +Step 41300/50462 | Loss: 2.5240 | LR: 4.07e-05 | 4197.5ms/step | 14.3 samples/s +Step 41310/50462 | Loss: 2.4848 | LR: 4.07e-05 | 4200.6ms/step | 14.3 samples/s +Step 41320/50462 | Loss: 2.5681 | LR: 4.06e-05 | 4196.2ms/step | 14.3 samples/s +Step 41330/50462 | Loss: 2.5030 | LR: 4.05e-05 | 4199.6ms/step | 14.3 samples/s +Step 41340/50462 | Loss: 2.4963 | LR: 4.04e-05 | 4196.2ms/step | 14.3 samples/s +Step 41350/50462 | Loss: 2.5176 | LR: 4.03e-05 | 4250.3ms/step | 14.1 samples/s +Step 41360/50462 | Loss: 2.5449 | LR: 4.02e-05 | 4254.5ms/step | 14.1 samples/s +Step 41370/50462 | Loss: 2.5158 | LR: 4.01e-05 | 4202.7ms/step | 14.3 samples/s +Step 41380/50462 | Loss: 2.5086 | LR: 4.01e-05 | 4206.7ms/step | 14.3 samples/s +Step 41390/50462 | Loss: 2.4851 | LR: 4.00e-05 | 4201.1ms/step | 14.3 samples/s +Step 41400/50462 | Loss: 2.5247 | LR: 3.99e-05 | 4203.0ms/step | 14.3 samples/s +Step 41410/50462 | Loss: 2.5306 | LR: 3.98e-05 | 4203.1ms/step | 14.3 samples/s +Step 41420/50462 | Loss: 2.4950 | LR: 3.97e-05 | 4202.8ms/step | 14.3 samples/s +Step 41430/50462 | Loss: 2.5258 | LR: 3.96e-05 | 4206.1ms/step | 14.3 samples/s +Step 41440/50462 | Loss: 2.4946 | LR: 3.95e-05 | 4207.8ms/step | 14.3 samples/s +Step 41450/50462 | Loss: 2.5137 | LR: 3.95e-05 | 4214.3ms/step | 14.2 samples/s +Step 41460/50462 | Loss: 2.4960 | LR: 3.94e-05 | 4211.5ms/step | 14.2 samples/s +Step 41470/50462 | Loss: 2.4761 | LR: 3.93e-05 | 4208.4ms/step | 14.3 samples/s +Step 41480/50462 | Loss: 2.5337 | LR: 3.92e-05 | 4205.5ms/step | 14.3 samples/s +Step 41490/50462 | Loss: 2.4873 | LR: 3.91e-05 | 4206.6ms/step | 14.3 samples/s +Step 41500/50462 | Loss: 2.5293 | LR: 3.90e-05 | 4204.7ms/step | 14.3 samples/s +Validation at step 41500: Loss = 2.5263, Perplexity = 12.51 +Step 41510/50462 | Loss: 2.4999 | LR: 3.90e-05 | 6211.1ms/step | 9.7 samples/s +Step 41520/50462 | Loss: 2.5141 | LR: 3.89e-05 | 4200.8ms/step | 14.3 samples/s +Step 41530/50462 | Loss: 2.5256 | LR: 3.88e-05 | 4195.0ms/step | 14.3 samples/s +Step 41540/50462 | Loss: 2.4945 | LR: 3.87e-05 | 4201.9ms/step | 14.3 samples/s +Step 41550/50462 | Loss: 2.5210 | LR: 3.86e-05 | 4218.2ms/step | 14.2 samples/s +Step 41560/50462 | Loss: 2.5318 | LR: 3.85e-05 | 4207.7ms/step | 14.3 samples/s +Step 41570/50462 | Loss: 2.5190 | LR: 3.84e-05 | 4258.1ms/step | 14.1 samples/s +Step 41580/50462 | Loss: 2.5253 | LR: 3.84e-05 | 4202.5ms/step | 14.3 samples/s +Step 41590/50462 | Loss: 2.5071 | LR: 3.83e-05 | 4205.4ms/step | 14.3 samples/s +Step 41600/50462 | Loss: 2.5213 | LR: 3.82e-05 | 4200.1ms/step | 14.3 samples/s +Step 41610/50462 | Loss: 2.5035 | LR: 3.81e-05 | 4201.8ms/step | 14.3 samples/s +Step 41620/50462 | Loss: 2.5098 | LR: 3.80e-05 | 4200.4ms/step | 14.3 samples/s +Step 41630/50462 | Loss: 2.4912 | LR: 3.79e-05 | 4201.5ms/step | 14.3 samples/s +Step 41640/50462 | Loss: 2.5069 | LR: 3.79e-05 | 4200.2ms/step | 14.3 samples/s +Step 41650/50462 | Loss: 2.4921 | LR: 3.78e-05 | 4253.8ms/step | 14.1 samples/s +Step 41660/50462 | Loss: 2.5223 | LR: 3.77e-05 | 4264.6ms/step | 14.1 samples/s +Step 41670/50462 | Loss: 2.5142 | LR: 3.76e-05 | 4210.0ms/step | 14.3 samples/s +Step 41680/50462 | Loss: 2.4996 | LR: 3.75e-05 | 4201.3ms/step | 14.3 samples/s +Step 41690/50462 | Loss: 2.4965 | LR: 3.74e-05 | 4202.4ms/step | 14.3 samples/s +Step 41700/50462 | Loss: 2.5083 | LR: 3.74e-05 | 4205.0ms/step | 14.3 samples/s +Step 41710/50462 | Loss: 2.5003 | LR: 3.73e-05 | 4203.5ms/step | 14.3 samples/s +Step 41720/50462 | Loss: 2.4970 | LR: 3.72e-05 | 4203.0ms/step | 14.3 samples/s +Step 41730/50462 | Loss: 2.5027 | LR: 3.71e-05 | 4200.3ms/step | 14.3 samples/s +Step 41740/50462 | Loss: 2.4951 | LR: 3.70e-05 | 4204.2ms/step | 14.3 samples/s +Step 41750/50462 | Loss: 2.5031 | LR: 3.69e-05 | 4203.7ms/step | 14.3 samples/s +Validation at step 41750: Loss = 2.5260, Perplexity = 12.50 +Step 41760/50462 | Loss: 2.5147 | LR: 3.69e-05 | 6207.6ms/step | 9.7 samples/s +Step 41770/50462 | Loss: 2.5212 | LR: 3.68e-05 | 4205.2ms/step | 14.3 samples/s +Step 41780/50462 | Loss: 2.5079 | LR: 3.67e-05 | 4199.0ms/step | 14.3 samples/s +Step 41790/50462 | Loss: 2.4937 | LR: 3.66e-05 | 4198.0ms/step | 14.3 samples/s +Step 41800/50462 | Loss: 2.4985 | LR: 3.65e-05 | 4196.3ms/step | 14.3 samples/s +Step 41810/50462 | Loss: 2.5331 | LR: 3.64e-05 | 4197.8ms/step | 14.3 samples/s +Step 41820/50462 | Loss: 2.4581 | LR: 3.64e-05 | 4200.0ms/step | 14.3 samples/s +Step 41830/50462 | Loss: 2.5050 | LR: 3.63e-05 | 4199.7ms/step | 14.3 samples/s +Step 41840/50462 | Loss: 2.4806 | LR: 3.62e-05 | 4198.2ms/step | 14.3 samples/s +Step 41850/50462 | Loss: 2.5223 | LR: 3.61e-05 | 4199.8ms/step | 14.3 samples/s +Step 41860/50462 | Loss: 2.5078 | LR: 3.60e-05 | 4198.4ms/step | 14.3 samples/s +Step 41870/50462 | Loss: 2.5056 | LR: 3.60e-05 | 4256.1ms/step | 14.1 samples/s +Step 41880/50462 | Loss: 2.5179 | LR: 3.59e-05 | 4198.2ms/step | 14.3 samples/s +Step 41890/50462 | Loss: 2.5096 | LR: 3.58e-05 | 4199.6ms/step | 14.3 samples/s +Step 41900/50462 | Loss: 2.5345 | LR: 3.57e-05 | 4199.5ms/step | 14.3 samples/s +Step 41910/50462 | Loss: 2.5259 | LR: 3.56e-05 | 4198.5ms/step | 14.3 samples/s +Step 41920/50462 | Loss: 2.5167 | LR: 3.55e-05 | 4198.0ms/step | 14.3 samples/s +Step 41930/50462 | Loss: 2.4971 | LR: 3.55e-05 | 4201.9ms/step | 14.3 samples/s +Step 41940/50462 | Loss: 2.5241 | LR: 3.54e-05 | 4206.6ms/step | 14.3 samples/s +Step 41950/50462 | Loss: 2.4953 | LR: 3.53e-05 | 4201.5ms/step | 14.3 samples/s +Step 41960/50462 | Loss: 2.5219 | LR: 3.52e-05 | 4303.8ms/step | 13.9 samples/s +Step 41970/50462 | Loss: 2.5099 | LR: 3.51e-05 | 4202.1ms/step | 14.3 samples/s +Step 41980/50462 | Loss: 2.5082 | LR: 3.51e-05 | 4200.2ms/step | 14.3 samples/s +Step 41990/50462 | Loss: 2.5414 | LR: 3.50e-05 | 4201.5ms/step | 14.3 samples/s +Step 42000/50462 | Loss: 2.5186 | LR: 3.49e-05 | 4202.3ms/step | 14.3 samples/s +Validation at step 42000: Loss = 2.5259, Perplexity = 12.50 +Saved checkpoint to checkpoints/run_20260615_223806/checkpoint-42000 +Step 42010/50462 | Loss: 2.4877 | LR: 3.48e-05 | 7057.8ms/step | 8.5 samples/s +Step 42020/50462 | Loss: 2.5294 | LR: 3.47e-05 | 4200.0ms/step | 14.3 samples/s +Step 42030/50462 | Loss: 2.5481 | LR: 3.47e-05 | 4200.1ms/step | 14.3 samples/s +W0616 01:16:46.153000 3487 torch/distributed/elastic/agent/server/api.py:753] Received Signals.SIGTERM death signal, shutting down workers +W0616 01:16:46.155000 3487 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3647 closing signal SIGTERM +W0616 01:16:46.157000 3487 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3648 closing signal SIGTERM +W0616 01:16:46.158000 3487 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3649 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/usr/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1020, in + main() + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 367, in wrapper + return f(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1016, in main + run(args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1007, in run + elastic_launch( + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 191, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 362, in launch_agent + result = agent.run() + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 745, in run + result = self._invoke_run(role) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 923, in _invoke_run + time.sleep(monitor_interval) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 3487 got signal: 15 diff --git a/epfl_final/log_files/1mask_perlayerfix.log b/epfl_final/log_files/1mask_perlayerfix.log new file mode 100644 index 0000000000000000000000000000000000000000..ab80f9e5aab54db306eefb4b8c5a917c589efe7d --- /dev/null +++ b/epfl_final/log_files/1mask_perlayerfix.log @@ -0,0 +1,5122 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "sparse 1mask perlayer-fix", + "wandb_tags": [ + "qwen3", + "1.2b", + "sparse_stochastic", + "1mask", + "perlayer-fix" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +[MASKHOOK] saved 28 layers, 28 DISTINCT masks -> /mloscratch/homes/navasard/sparse/reconstructed_masks/1mask_perlayerfix_TRAINHOOK.pt + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.2 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260618_175952-33yw8345 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run sparse 1mask perlayer-fix +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/33yw8345 +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/50462 | Loss: 12.2694 | LR: 6.67e-06 | 7845.7ms/step | 7.6 samples/s +Step 20/50462 | Loss: 11.9981 | LR: 1.33e-05 | 3231.6ms/step | 18.6 samples/s +Step 30/50462 | Loss: 11.1414 | LR: 2.00e-05 | 3165.1ms/step | 19.0 samples/s +Step 40/50462 | Loss: 10.2128 | LR: 2.67e-05 | 3164.9ms/step | 19.0 samples/s +Step 50/50462 | Loss: 9.7757 | LR: 3.33e-05 | 3241.9ms/step | 18.5 samples/s +Step 60/50462 | Loss: 9.2948 | LR: 4.00e-05 | 3167.1ms/step | 18.9 samples/s +Step 70/50462 | Loss: 8.7899 | LR: 4.67e-05 | 3228.4ms/step | 18.6 samples/s +Step 80/50462 | Loss: 8.3033 | LR: 5.33e-05 | 3172.8ms/step | 18.9 samples/s +Step 90/50462 | Loss: 7.8817 | LR: 6.00e-05 | 3176.2ms/step | 18.9 samples/s +Step 100/50462 | Loss: 7.5265 | LR: 6.67e-05 | 3169.8ms/step | 18.9 samples/s +Step 110/50462 | Loss: 7.2883 | LR: 7.33e-05 | 3168.1ms/step | 18.9 samples/s +Step 120/50462 | Loss: 7.1315 | LR: 8.00e-05 | 3176.9ms/step | 18.9 samples/s +Step 130/50462 | Loss: 7.0024 | LR: 8.67e-05 | 3174.0ms/step | 18.9 samples/s +Step 140/50462 | Loss: 6.8910 | LR: 9.33e-05 | 3176.9ms/step | 18.9 samples/s +Step 150/50462 | Loss: 6.7724 | LR: 1.00e-04 | 3188.9ms/step | 18.8 samples/s +Step 160/50462 | Loss: 6.6615 | LR: 1.07e-04 | 3190.0ms/step | 18.8 samples/s +Step 170/50462 | Loss: 6.5651 | LR: 1.13e-04 | 3174.5ms/step | 18.9 samples/s +Step 180/50462 | Loss: 6.4665 | LR: 1.20e-04 | 3190.2ms/step | 18.8 samples/s +Step 190/50462 | Loss: 6.3812 | LR: 1.27e-04 | 3185.2ms/step | 18.8 samples/s +Step 200/50462 | Loss: 6.2948 | LR: 1.33e-04 | 3194.0ms/step | 18.8 samples/s +Step 210/50462 | Loss: 6.2159 | LR: 1.40e-04 | 3195.5ms/step | 18.8 samples/s +Step 220/50462 | Loss: 6.1407 | LR: 1.47e-04 | 3183.9ms/step | 18.8 samples/s +Step 230/50462 | Loss: 6.1154 | LR: 1.53e-04 | 3182.6ms/step | 18.9 samples/s +Step 240/50462 | Loss: 6.0785 | LR: 1.60e-04 | 3197.5ms/step | 18.8 samples/s +Step 250/50462 | Loss: 6.0072 | LR: 1.67e-04 | 3195.7ms/step | 18.8 samples/s +Validation at step 250: Loss = 5.9501, Perplexity = 383.80 +Step 260/50462 | Loss: 5.9277 | LR: 1.73e-04 | 7457.1ms/step | 8.0 samples/s +Step 270/50462 | Loss: 5.9349 | LR: 1.80e-04 | 3183.7ms/step | 18.8 samples/s +Step 280/50462 | Loss: 5.8526 | LR: 1.87e-04 | 3191.2ms/step | 18.8 samples/s +Step 290/50462 | Loss: 5.7902 | LR: 1.93e-04 | 3200.0ms/step | 18.7 samples/s +Step 300/50462 | Loss: 5.7223 | LR: 2.00e-04 | 3179.3ms/step | 18.9 samples/s +Step 310/50462 | Loss: 5.6947 | LR: 2.07e-04 | 3192.2ms/step | 18.8 samples/s +Step 320/50462 | Loss: 5.6365 | LR: 2.13e-04 | 3198.4ms/step | 18.8 samples/s +Step 330/50462 | Loss: 5.6102 | LR: 2.20e-04 | 3199.9ms/step | 18.8 samples/s +Step 340/50462 | Loss: 5.5438 | LR: 2.27e-04 | 3200.3ms/step | 18.7 samples/s +Step 350/50462 | Loss: 5.5048 | LR: 2.33e-04 | 3198.0ms/step | 18.8 samples/s +Step 360/50462 | Loss: 5.4351 | LR: 2.40e-04 | 3199.9ms/step | 18.8 samples/s +Step 370/50462 | Loss: 5.3824 | LR: 2.47e-04 | 3200.1ms/step | 18.7 samples/s +Step 380/50462 | Loss: 5.3619 | LR: 2.53e-04 | 3200.1ms/step | 18.7 samples/s +Step 390/50462 | Loss: 5.3153 | LR: 2.60e-04 | 3199.9ms/step | 18.8 samples/s +Step 400/50462 | Loss: 5.2465 | LR: 2.67e-04 | 3200.1ms/step | 18.7 samples/s +Step 410/50462 | Loss: 5.2119 | LR: 2.73e-04 | 3199.5ms/step | 18.8 samples/s +Step 420/50462 | Loss: 5.1869 | LR: 2.80e-04 | 3198.4ms/step | 18.8 samples/s +Step 430/50462 | Loss: 5.1323 | LR: 2.87e-04 | 3200.1ms/step | 18.7 samples/s +Step 440/50462 | Loss: 5.0908 | LR: 2.93e-04 | 3262.3ms/step | 18.4 samples/s +Step 450/50462 | Loss: 5.0278 | LR: 3.00e-04 | 3179.4ms/step | 18.9 samples/s +Step 460/50462 | Loss: 5.0272 | LR: 3.07e-04 | 3200.2ms/step | 18.7 samples/s +Step 470/50462 | Loss: 5.0380 | LR: 3.13e-04 | 3197.4ms/step | 18.8 samples/s +Step 480/50462 | Loss: 4.9642 | LR: 3.20e-04 | 3197.9ms/step | 18.8 samples/s +Step 490/50462 | Loss: 4.8953 | LR: 3.27e-04 | 3189.8ms/step | 18.8 samples/s +Step 500/50462 | Loss: 4.9359 | LR: 3.33e-04 | 3195.7ms/step | 18.8 samples/s +Validation at step 500: Loss = 4.9335, Perplexity = 138.86 +Step 510/50462 | Loss: 4.9148 | LR: 3.40e-04 | 4870.6ms/step | 12.3 samples/s +Step 520/50462 | Loss: 4.8392 | LR: 3.47e-04 | 3256.6ms/step | 18.4 samples/s +Step 530/50462 | Loss: 4.7950 | LR: 3.53e-04 | 3192.8ms/step | 18.8 samples/s +Step 540/50462 | Loss: 4.7590 | LR: 3.60e-04 | 3245.0ms/step | 18.5 samples/s +Step 550/50462 | Loss: 4.8198 | LR: 3.67e-04 | 3197.5ms/step | 18.8 samples/s +Step 560/50462 | Loss: 4.7284 | LR: 3.73e-04 | 3200.2ms/step | 18.7 samples/s +Step 570/50462 | Loss: 4.6312 | LR: 3.80e-04 | 3200.7ms/step | 18.7 samples/s +Step 580/50462 | Loss: 4.6005 | LR: 3.87e-04 | 3199.9ms/step | 18.8 samples/s +Step 590/50462 | Loss: 4.6920 | LR: 3.93e-04 | 3199.5ms/step | 18.8 samples/s +Step 600/50462 | Loss: 4.6411 | LR: 4.00e-04 | 3200.6ms/step | 18.7 samples/s +Step 610/50462 | Loss: 4.5350 | LR: 4.07e-04 | 3200.0ms/step | 18.8 samples/s +Step 620/50462 | Loss: 4.5367 | LR: 4.13e-04 | 3199.7ms/step | 18.8 samples/s +Step 630/50462 | Loss: 4.5416 | LR: 4.20e-04 | 3196.3ms/step | 18.8 samples/s +Step 640/50462 | Loss: 4.4841 | LR: 4.27e-04 | 3199.8ms/step | 18.8 samples/s +Step 650/50462 | Loss: 4.4099 | LR: 4.33e-04 | 3200.3ms/step | 18.7 samples/s +Step 660/50462 | Loss: 4.4013 | LR: 4.40e-04 | 3199.3ms/step | 18.8 samples/s +Step 670/50462 | Loss: 4.5024 | LR: 4.47e-04 | 3187.2ms/step | 18.8 samples/s +Step 680/50462 | Loss: 4.4267 | LR: 4.53e-04 | 3195.5ms/step | 18.8 samples/s +Step 690/50462 | Loss: 4.3005 | LR: 4.60e-04 | 3199.5ms/step | 18.8 samples/s +Step 700/50462 | Loss: 4.3124 | LR: 4.67e-04 | 3200.4ms/step | 18.7 samples/s +Step 710/50462 | Loss: 4.3069 | LR: 4.73e-04 | 3200.0ms/step | 18.8 samples/s +Step 720/50462 | Loss: 4.3719 | LR: 4.80e-04 | 3200.1ms/step | 18.7 samples/s +Step 730/50462 | Loss: 4.2887 | LR: 4.87e-04 | 3197.4ms/step | 18.8 samples/s +Step 740/50462 | Loss: 4.1988 | LR: 4.93e-04 | 3200.5ms/step | 18.7 samples/s +Step 750/50462 | Loss: 4.1957 | LR: 5.00e-04 | 3199.4ms/step | 18.8 samples/s +Validation at step 750: Loss = 4.1649, Perplexity = 64.39 +Step 760/50462 | Loss: 4.1483 | LR: 5.00e-04 | 4862.5ms/step | 12.3 samples/s +Step 770/50462 | Loss: 4.2137 | LR: 5.00e-04 | 3246.4ms/step | 18.5 samples/s +Step 780/50462 | Loss: 4.1507 | LR: 5.00e-04 | 3199.7ms/step | 18.8 samples/s +Step 790/50462 | Loss: 4.1131 | LR: 5.00e-04 | 3198.5ms/step | 18.8 samples/s +Step 800/50462 | Loss: 4.0647 | LR: 5.00e-04 | 3186.5ms/step | 18.8 samples/s +Step 810/50462 | Loss: 4.0922 | LR: 5.00e-04 | 3191.0ms/step | 18.8 samples/s +Step 820/50462 | Loss: 4.0600 | LR: 5.00e-04 | 3190.2ms/step | 18.8 samples/s +Step 830/50462 | Loss: 4.0305 | LR: 5.00e-04 | 3193.8ms/step | 18.8 samples/s +Step 840/50462 | Loss: 4.0111 | LR: 5.00e-04 | 3184.8ms/step | 18.8 samples/s +Step 850/50462 | Loss: 3.9637 | LR: 5.00e-04 | 3252.9ms/step | 18.4 samples/s +Step 860/50462 | Loss: 3.9672 | LR: 5.00e-04 | 3193.8ms/step | 18.8 samples/s +Step 870/50462 | Loss: 3.9399 | LR: 5.00e-04 | 3192.7ms/step | 18.8 samples/s +Step 880/50462 | Loss: 3.9079 | LR: 5.00e-04 | 3255.6ms/step | 18.4 samples/s +Step 890/50462 | Loss: 3.9148 | LR: 5.00e-04 | 3199.5ms/step | 18.8 samples/s +Step 900/50462 | Loss: 3.9024 | LR: 5.00e-04 | 3195.0ms/step | 18.8 samples/s +Step 910/50462 | Loss: 3.8762 | LR: 5.00e-04 | 3189.2ms/step | 18.8 samples/s +Step 920/50462 | Loss: 3.8348 | LR: 5.00e-04 | 3176.9ms/step | 18.9 samples/s +Step 930/50462 | Loss: 3.8820 | LR: 5.00e-04 | 3177.7ms/step | 18.9 samples/s +Step 940/50462 | Loss: 3.8217 | LR: 5.00e-04 | 3188.0ms/step | 18.8 samples/s +Step 950/50462 | Loss: 3.8111 | LR: 5.00e-04 | 3198.0ms/step | 18.8 samples/s +Step 960/50462 | Loss: 3.8125 | LR: 5.00e-04 | 3192.0ms/step | 18.8 samples/s +Step 970/50462 | Loss: 3.7968 | LR: 5.00e-04 | 3190.8ms/step | 18.8 samples/s +Step 980/50462 | Loss: 3.7559 | LR: 5.00e-04 | 3174.4ms/step | 18.9 samples/s +Step 990/50462 | Loss: 3.7726 | LR: 5.00e-04 | 3182.3ms/step | 18.9 samples/s +Step 1000/50462 | Loss: 3.7694 | LR: 5.00e-04 | 3196.1ms/step | 18.8 samples/s +Validation at step 1000: Loss = 3.7646, Perplexity = 43.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-1000 +Step 1010/50462 | Loss: 3.7446 | LR: 5.00e-04 | 5801.3ms/step | 10.3 samples/s +Step 1020/50462 | Loss: 3.7556 | LR: 5.00e-04 | 3179.5ms/step | 18.9 samples/s +Step 1030/50462 | Loss: 3.7409 | LR: 5.00e-04 | 3192.4ms/step | 18.8 samples/s +Step 1040/50462 | Loss: 3.7430 | LR: 5.00e-04 | 3195.3ms/step | 18.8 samples/s +Step 1050/50462 | Loss: 3.7168 | LR: 5.00e-04 | 3198.5ms/step | 18.8 samples/s +Step 1060/50462 | Loss: 3.7005 | LR: 5.00e-04 | 3193.2ms/step | 18.8 samples/s +Step 1070/50462 | Loss: 3.6924 | LR: 5.00e-04 | 3198.0ms/step | 18.8 samples/s +Step 1080/50462 | Loss: 3.6958 | LR: 5.00e-04 | 3192.1ms/step | 18.8 samples/s +Step 1090/50462 | Loss: 3.6702 | LR: 5.00e-04 | 3181.5ms/step | 18.9 samples/s +Step 1100/50462 | Loss: 3.6835 | LR: 5.00e-04 | 3252.7ms/step | 18.4 samples/s +Step 1110/50462 | Loss: 3.6925 | LR: 5.00e-04 | 3198.7ms/step | 18.8 samples/s +Step 1120/50462 | Loss: 3.6464 | LR: 5.00e-04 | 3199.2ms/step | 18.8 samples/s +Step 1130/50462 | Loss: 3.6589 | LR: 5.00e-04 | 3200.0ms/step | 18.8 samples/s +Step 1140/50462 | Loss: 3.6221 | LR: 5.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 1150/50462 | Loss: 3.5891 | LR: 5.00e-04 | 3200.0ms/step | 18.8 samples/s +Step 1160/50462 | Loss: 3.6149 | LR: 5.00e-04 | 3200.1ms/step | 18.7 samples/s +Step 1170/50462 | Loss: 3.6048 | LR: 5.00e-04 | 3200.2ms/step | 18.7 samples/s +Step 1180/50462 | Loss: 3.5702 | LR: 5.00e-04 | 3199.8ms/step | 18.8 samples/s +Step 1190/50462 | Loss: 3.5928 | LR: 5.00e-04 | 3254.2ms/step | 18.4 samples/s +Step 1200/50462 | Loss: 3.6069 | LR: 5.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 1210/50462 | Loss: 3.6031 | LR: 5.00e-04 | 3254.1ms/step | 18.4 samples/s +Step 1220/50462 | Loss: 3.5639 | LR: 5.00e-04 | 3199.8ms/step | 18.8 samples/s +Step 1230/50462 | Loss: 3.5343 | LR: 5.00e-04 | 3199.1ms/step | 18.8 samples/s +Step 1240/50462 | Loss: 3.5931 | LR: 5.00e-04 | 3196.2ms/step | 18.8 samples/s +Step 1250/50462 | Loss: 3.5546 | LR: 5.00e-04 | 3198.2ms/step | 18.8 samples/s +Validation at step 1250: Loss = 3.5644, Perplexity = 35.32 +Step 1260/50462 | Loss: 3.5750 | LR: 5.00e-04 | 4872.6ms/step | 12.3 samples/s +Step 1270/50462 | Loss: 3.5374 | LR: 5.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 1280/50462 | Loss: 3.5137 | LR: 5.00e-04 | 3200.2ms/step | 18.7 samples/s +Step 1290/50462 | Loss: 3.5203 | LR: 5.00e-04 | 3200.0ms/step | 18.7 samples/s +Step 1300/50462 | Loss: 3.5121 | LR: 5.00e-04 | 3199.7ms/step | 18.8 samples/s +Step 1310/50462 | Loss: 3.5246 | LR: 5.00e-04 | 3200.2ms/step | 18.7 samples/s +Step 1320/50462 | Loss: 3.5120 | LR: 5.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 1330/50462 | Loss: 3.4985 | LR: 5.00e-04 | 3199.7ms/step | 18.8 samples/s +Step 1340/50462 | Loss: 3.4901 | LR: 5.00e-04 | 3200.4ms/step | 18.7 samples/s +Step 1350/50462 | Loss: 3.4729 | LR: 5.00e-04 | 3200.0ms/step | 18.7 samples/s +Step 1360/50462 | Loss: 3.4960 | LR: 5.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 1370/50462 | Loss: 3.4810 | LR: 5.00e-04 | 3198.1ms/step | 18.8 samples/s +Step 1380/50462 | Loss: 3.5075 | LR: 5.00e-04 | 3195.2ms/step | 18.8 samples/s +Step 1390/50462 | Loss: 3.4824 | LR: 5.00e-04 | 3200.8ms/step | 18.7 samples/s +Step 1400/50462 | Loss: 3.4504 | LR: 5.00e-04 | 3199.0ms/step | 18.8 samples/s +Step 1410/50462 | Loss: 3.4773 | LR: 5.00e-04 | 3178.5ms/step | 18.9 samples/s +Step 1420/50462 | Loss: 3.4565 | LR: 5.00e-04 | 3187.6ms/step | 18.8 samples/s +Step 1430/50462 | Loss: 3.4332 | LR: 5.00e-04 | 3249.3ms/step | 18.5 samples/s +Step 1440/50462 | Loss: 3.4661 | LR: 5.00e-04 | 3191.6ms/step | 18.8 samples/s +Step 1450/50462 | Loss: 3.4296 | LR: 5.00e-04 | 3181.9ms/step | 18.9 samples/s +Step 1460/50462 | Loss: 3.4367 | LR: 5.00e-04 | 3181.8ms/step | 18.9 samples/s +Step 1470/50462 | Loss: 3.4104 | LR: 5.00e-04 | 3188.1ms/step | 18.8 samples/s +Step 1480/50462 | Loss: 3.4506 | LR: 5.00e-04 | 3196.1ms/step | 18.8 samples/s +Step 1490/50462 | Loss: 3.4319 | LR: 5.00e-04 | 3202.0ms/step | 18.7 samples/s +Step 1500/50462 | Loss: 3.4011 | LR: 5.00e-04 | 3194.1ms/step | 18.8 samples/s +Validation at step 1500: Loss = 3.4349, Perplexity = 31.03 +Step 1510/50462 | Loss: 3.4297 | LR: 5.00e-04 | 4874.0ms/step | 12.3 samples/s +Step 1520/50462 | Loss: 3.4006 | LR: 5.00e-04 | 3255.1ms/step | 18.4 samples/s +Step 1530/50462 | Loss: 3.4270 | LR: 5.00e-04 | 3200.7ms/step | 18.7 samples/s +Step 1540/50462 | Loss: 3.3946 | LR: 5.00e-04 | 3200.1ms/step | 18.7 samples/s +Step 1550/50462 | Loss: 3.3995 | LR: 5.00e-04 | 3255.8ms/step | 18.4 samples/s +Step 1560/50462 | Loss: 3.3941 | LR: 5.00e-04 | 3180.6ms/step | 18.9 samples/s +Step 1570/50462 | Loss: 3.4282 | LR: 5.00e-04 | 3188.7ms/step | 18.8 samples/s +Step 1580/50462 | Loss: 3.3904 | LR: 5.00e-04 | 3200.9ms/step | 18.7 samples/s +Step 1590/50462 | Loss: 3.4144 | LR: 5.00e-04 | 3193.7ms/step | 18.8 samples/s +Step 1600/50462 | Loss: 3.4009 | LR: 5.00e-04 | 3190.1ms/step | 18.8 samples/s +Step 1610/50462 | Loss: 3.3666 | LR: 5.00e-04 | 3200.3ms/step | 18.7 samples/s +Step 1620/50462 | Loss: 3.3844 | LR: 5.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 1630/50462 | Loss: 3.3987 | LR: 5.00e-04 | 3200.0ms/step | 18.7 samples/s +Step 1640/50462 | Loss: 3.3742 | LR: 5.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 1650/50462 | Loss: 3.3360 | LR: 5.00e-04 | 3200.1ms/step | 18.7 samples/s +Step 1660/50462 | Loss: 3.3538 | LR: 5.00e-04 | 3199.7ms/step | 18.8 samples/s +Step 1670/50462 | Loss: 3.3774 | LR: 5.00e-04 | 3200.2ms/step | 18.7 samples/s +Step 1680/50462 | Loss: 3.3521 | LR: 5.00e-04 | 3191.6ms/step | 18.8 samples/s +Step 1690/50462 | Loss: 3.3531 | LR: 5.00e-04 | 3188.1ms/step | 18.8 samples/s +Step 1700/50462 | Loss: 3.3571 | LR: 5.00e-04 | 3200.6ms/step | 18.7 samples/s +Step 1710/50462 | Loss: 3.3766 | LR: 5.00e-04 | 3199.6ms/step | 18.8 samples/s +Step 1720/50462 | Loss: 3.3392 | LR: 5.00e-04 | 3200.2ms/step | 18.7 samples/s +Step 1730/50462 | Loss: 3.3374 | LR: 5.00e-04 | 3200.1ms/step | 18.7 samples/s +Step 1740/50462 | Loss: 3.3699 | LR: 5.00e-04 | 3200.0ms/step | 18.8 samples/s +Step 1750/50462 | Loss: 3.3820 | LR: 5.00e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 1750: Loss = 3.3458, Perplexity = 28.38 +Step 1760/50462 | Loss: 3.3545 | LR: 4.99e-04 | 4877.7ms/step | 12.3 samples/s +Step 1770/50462 | Loss: 3.3381 | LR: 4.99e-04 | 3255.5ms/step | 18.4 samples/s +Step 1780/50462 | Loss: 3.3091 | LR: 4.99e-04 | 3192.4ms/step | 18.8 samples/s +Step 1790/50462 | Loss: 3.3094 | LR: 4.99e-04 | 3200.1ms/step | 18.7 samples/s +Step 1800/50462 | Loss: 3.3042 | LR: 4.99e-04 | 3196.1ms/step | 18.8 samples/s +Step 1810/50462 | Loss: 3.3169 | LR: 4.99e-04 | 3180.3ms/step | 18.9 samples/s +Step 1820/50462 | Loss: 3.2984 | LR: 4.99e-04 | 3185.6ms/step | 18.8 samples/s +Step 1830/50462 | Loss: 3.3029 | LR: 4.99e-04 | 3195.5ms/step | 18.8 samples/s +Step 1840/50462 | Loss: 3.2971 | LR: 4.99e-04 | 3200.5ms/step | 18.7 samples/s +Step 1850/50462 | Loss: 3.3081 | LR: 4.99e-04 | 3202.2ms/step | 18.7 samples/s +Step 1860/50462 | Loss: 3.2875 | LR: 4.99e-04 | 3254.0ms/step | 18.4 samples/s +Step 1870/50462 | Loss: 3.3307 | LR: 4.99e-04 | 3181.5ms/step | 18.9 samples/s +Step 1880/50462 | Loss: 3.3051 | LR: 4.99e-04 | 3180.1ms/step | 18.9 samples/s +Step 1890/50462 | Loss: 3.2739 | LR: 4.99e-04 | 3246.2ms/step | 18.5 samples/s +Step 1900/50462 | Loss: 3.2768 | LR: 4.99e-04 | 3196.2ms/step | 18.8 samples/s +Step 1910/50462 | Loss: 3.3112 | LR: 4.99e-04 | 3200.1ms/step | 18.7 samples/s +Step 1920/50462 | Loss: 3.3091 | LR: 4.99e-04 | 3199.7ms/step | 18.8 samples/s +Step 1930/50462 | Loss: 3.3115 | LR: 4.99e-04 | 3198.1ms/step | 18.8 samples/s +Step 1940/50462 | Loss: 3.3018 | LR: 4.99e-04 | 3188.3ms/step | 18.8 samples/s +Step 1950/50462 | Loss: 3.2967 | LR: 4.99e-04 | 3185.1ms/step | 18.8 samples/s +Step 1960/50462 | Loss: 3.2894 | LR: 4.99e-04 | 3191.3ms/step | 18.8 samples/s +Step 1970/50462 | Loss: 3.2883 | LR: 4.99e-04 | 3197.5ms/step | 18.8 samples/s +Step 1980/50462 | Loss: 3.2821 | LR: 4.99e-04 | 3200.3ms/step | 18.7 samples/s +Step 1990/50462 | Loss: 3.3057 | LR: 4.99e-04 | 3199.4ms/step | 18.8 samples/s +Step 2000/50462 | Loss: 3.3068 | LR: 4.99e-04 | 3200.3ms/step | 18.7 samples/s +Validation at step 2000: Loss = 3.2777, Perplexity = 26.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-2000 +Step 2010/50462 | Loss: 3.2867 | LR: 4.99e-04 | 5793.0ms/step | 10.4 samples/s +Step 2020/50462 | Loss: 3.2894 | LR: 4.99e-04 | 3182.9ms/step | 18.9 samples/s +Step 2030/50462 | Loss: 3.2580 | LR: 4.99e-04 | 3198.0ms/step | 18.8 samples/s +Step 2040/50462 | Loss: 3.2699 | LR: 4.99e-04 | 3199.9ms/step | 18.8 samples/s +Step 2050/50462 | Loss: 3.2401 | LR: 4.99e-04 | 3186.0ms/step | 18.8 samples/s +Step 2060/50462 | Loss: 3.2398 | LR: 4.99e-04 | 3182.7ms/step | 18.9 samples/s +Step 2070/50462 | Loss: 3.2591 | LR: 4.99e-04 | 3199.2ms/step | 18.8 samples/s +Step 2080/50462 | Loss: 3.2479 | LR: 4.99e-04 | 3200.1ms/step | 18.7 samples/s +Step 2090/50462 | Loss: 3.2521 | LR: 4.99e-04 | 3199.9ms/step | 18.8 samples/s +Step 2100/50462 | Loss: 3.2398 | LR: 4.99e-04 | 3253.9ms/step | 18.4 samples/s +Step 2110/50462 | Loss: 3.2860 | LR: 4.99e-04 | 3181.6ms/step | 18.9 samples/s +Step 2120/50462 | Loss: 3.2725 | LR: 4.99e-04 | 3194.2ms/step | 18.8 samples/s +Step 2130/50462 | Loss: 3.2347 | LR: 4.99e-04 | 3197.8ms/step | 18.8 samples/s +Step 2140/50462 | Loss: 3.2199 | LR: 4.99e-04 | 3200.7ms/step | 18.7 samples/s +Step 2150/50462 | Loss: 3.2256 | LR: 4.99e-04 | 3200.0ms/step | 18.7 samples/s +Step 2160/50462 | Loss: 3.2366 | LR: 4.99e-04 | 3200.0ms/step | 18.8 samples/s +Step 2170/50462 | Loss: 3.2222 | LR: 4.99e-04 | 3200.0ms/step | 18.7 samples/s +Step 2180/50462 | Loss: 3.2482 | LR: 4.99e-04 | 3201.9ms/step | 18.7 samples/s +Step 2190/50462 | Loss: 3.2214 | LR: 4.99e-04 | 3202.2ms/step | 18.7 samples/s +Step 2200/50462 | Loss: 3.2186 | LR: 4.99e-04 | 3251.9ms/step | 18.5 samples/s +Step 2210/50462 | Loss: 3.2302 | LR: 4.99e-04 | 3194.8ms/step | 18.8 samples/s +Step 2220/50462 | Loss: 3.2126 | LR: 4.99e-04 | 3237.2ms/step | 18.5 samples/s +Step 2230/50462 | Loss: 3.2278 | LR: 4.99e-04 | 3184.0ms/step | 18.8 samples/s +Step 2240/50462 | Loss: 3.2188 | LR: 4.99e-04 | 3193.6ms/step | 18.8 samples/s +Step 2250/50462 | Loss: 3.2588 | LR: 4.99e-04 | 3200.3ms/step | 18.7 samples/s +Validation at step 2250: Loss = 3.2249, Perplexity = 25.15 +Step 2260/50462 | Loss: 3.2209 | LR: 4.99e-04 | 4878.1ms/step | 12.3 samples/s +Step 2270/50462 | Loss: 3.2346 | LR: 4.99e-04 | 3200.0ms/step | 18.8 samples/s +Step 2280/50462 | Loss: 3.1861 | LR: 4.99e-04 | 3202.0ms/step | 18.7 samples/s +Step 2290/50462 | Loss: 3.1869 | LR: 4.99e-04 | 3200.1ms/step | 18.7 samples/s +Step 2300/50462 | Loss: 3.1902 | LR: 4.99e-04 | 3198.5ms/step | 18.8 samples/s +Step 2310/50462 | Loss: 3.2095 | LR: 4.99e-04 | 3201.1ms/step | 18.7 samples/s +Step 2320/50462 | Loss: 3.2069 | LR: 4.99e-04 | 3200.1ms/step | 18.7 samples/s +Step 2330/50462 | Loss: 3.1667 | LR: 4.99e-04 | 3192.1ms/step | 18.8 samples/s +Step 2340/50462 | Loss: 3.2029 | LR: 4.99e-04 | 3187.8ms/step | 18.8 samples/s +Step 2350/50462 | Loss: 3.1756 | LR: 4.99e-04 | 3189.0ms/step | 18.8 samples/s +Step 2360/50462 | Loss: 3.1751 | LR: 4.99e-04 | 3178.7ms/step | 18.9 samples/s +Step 2370/50462 | Loss: 3.1962 | LR: 4.99e-04 | 3182.6ms/step | 18.9 samples/s +Step 2380/50462 | Loss: 3.1451 | LR: 4.99e-04 | 3196.0ms/step | 18.8 samples/s +Step 2390/50462 | Loss: 3.1703 | LR: 4.99e-04 | 3200.0ms/step | 18.7 samples/s +Step 2400/50462 | Loss: 3.1939 | LR: 4.99e-04 | 3199.9ms/step | 18.8 samples/s +Step 2410/50462 | Loss: 3.1580 | LR: 4.99e-04 | 3200.2ms/step | 18.7 samples/s +Step 2420/50462 | Loss: 3.1999 | LR: 4.99e-04 | 3199.4ms/step | 18.8 samples/s +Step 2430/50462 | Loss: 3.1879 | LR: 4.99e-04 | 3194.3ms/step | 18.8 samples/s +Step 2440/50462 | Loss: 3.2017 | LR: 4.99e-04 | 3251.1ms/step | 18.5 samples/s +Step 2450/50462 | Loss: 3.2284 | LR: 4.99e-04 | 3196.1ms/step | 18.8 samples/s +Step 2460/50462 | Loss: 3.1736 | LR: 4.99e-04 | 3184.4ms/step | 18.8 samples/s +Step 2470/50462 | Loss: 3.1798 | LR: 4.99e-04 | 3192.1ms/step | 18.8 samples/s +Step 2480/50462 | Loss: 3.1945 | LR: 4.99e-04 | 3196.4ms/step | 18.8 samples/s +Step 2490/50462 | Loss: 3.1771 | LR: 4.98e-04 | 3199.8ms/step | 18.8 samples/s +Step 2500/50462 | Loss: 3.1687 | LR: 4.98e-04 | 3195.6ms/step | 18.8 samples/s +Validation at step 2500: Loss = 3.1776, Perplexity = 23.99 +Step 2510/50462 | Loss: 3.1633 | LR: 4.98e-04 | 4868.1ms/step | 12.3 samples/s +Step 2520/50462 | Loss: 3.1512 | LR: 4.98e-04 | 3195.4ms/step | 18.8 samples/s +Step 2530/50462 | Loss: 3.1529 | LR: 4.98e-04 | 3238.0ms/step | 18.5 samples/s +Step 2540/50462 | Loss: 3.1783 | LR: 4.98e-04 | 3199.1ms/step | 18.8 samples/s +Step 2550/50462 | Loss: 3.1835 | LR: 4.98e-04 | 3199.5ms/step | 18.8 samples/s +Step 2560/50462 | Loss: 3.1942 | LR: 4.98e-04 | 3256.4ms/step | 18.4 samples/s +Step 2570/50462 | Loss: 3.1362 | LR: 4.98e-04 | 3200.3ms/step | 18.7 samples/s +Step 2580/50462 | Loss: 3.1471 | LR: 4.98e-04 | 3200.0ms/step | 18.7 samples/s +Step 2590/50462 | Loss: 3.1764 | LR: 4.98e-04 | 3200.6ms/step | 18.7 samples/s +Step 2600/50462 | Loss: 3.1319 | LR: 4.98e-04 | 3199.3ms/step | 18.8 samples/s +Step 2610/50462 | Loss: 3.1457 | LR: 4.98e-04 | 3199.8ms/step | 18.8 samples/s +Step 2620/50462 | Loss: 3.1676 | LR: 4.98e-04 | 3200.2ms/step | 18.7 samples/s +Step 2630/50462 | Loss: 3.1316 | LR: 4.98e-04 | 3202.1ms/step | 18.7 samples/s +Step 2640/50462 | Loss: 3.1628 | LR: 4.98e-04 | 3199.3ms/step | 18.8 samples/s +Step 2650/50462 | Loss: 3.1594 | LR: 4.98e-04 | 3200.6ms/step | 18.7 samples/s +Step 2660/50462 | Loss: 3.1332 | LR: 4.98e-04 | 3201.9ms/step | 18.7 samples/s +Step 2670/50462 | Loss: 3.1903 | LR: 4.98e-04 | 3200.2ms/step | 18.7 samples/s +Step 2680/50462 | Loss: 3.1388 | LR: 4.98e-04 | 3199.9ms/step | 18.8 samples/s +Step 2690/50462 | Loss: 3.1592 | LR: 4.98e-04 | 3193.5ms/step | 18.8 samples/s +Step 2700/50462 | Loss: 3.1384 | LR: 4.98e-04 | 3194.2ms/step | 18.8 samples/s +Step 2710/50462 | Loss: 3.1170 | LR: 4.98e-04 | 3182.2ms/step | 18.9 samples/s +Step 2720/50462 | Loss: 3.1264 | LR: 4.98e-04 | 3193.5ms/step | 18.8 samples/s +Step 2730/50462 | Loss: 3.1428 | LR: 4.98e-04 | 3198.7ms/step | 18.8 samples/s +Step 2740/50462 | Loss: 3.1144 | LR: 4.98e-04 | 3189.7ms/step | 18.8 samples/s +Step 2750/50462 | Loss: 3.1410 | LR: 4.98e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 2750: Loss = 3.1401, Perplexity = 23.11 +Step 2760/50462 | Loss: 3.1407 | LR: 4.98e-04 | 4873.1ms/step | 12.3 samples/s +Step 2770/50462 | Loss: 3.1273 | LR: 4.98e-04 | 3256.4ms/step | 18.4 samples/s +Step 2780/50462 | Loss: 3.1299 | LR: 4.98e-04 | 3194.2ms/step | 18.8 samples/s +Step 2790/50462 | Loss: 3.1066 | LR: 4.98e-04 | 3183.4ms/step | 18.8 samples/s +Step 2800/50462 | Loss: 3.1272 | LR: 4.98e-04 | 3195.6ms/step | 18.8 samples/s +Step 2810/50462 | Loss: 3.1350 | LR: 4.98e-04 | 3199.0ms/step | 18.8 samples/s +Step 2820/50462 | Loss: 3.1203 | LR: 4.98e-04 | 3193.9ms/step | 18.8 samples/s +Step 2830/50462 | Loss: 3.0777 | LR: 4.98e-04 | 3195.5ms/step | 18.8 samples/s +Step 2840/50462 | Loss: 3.1226 | LR: 4.98e-04 | 3198.9ms/step | 18.8 samples/s +Step 2850/50462 | Loss: 3.1227 | LR: 4.98e-04 | 3200.2ms/step | 18.7 samples/s +Step 2860/50462 | Loss: 3.1300 | LR: 4.98e-04 | 3199.8ms/step | 18.8 samples/s +Step 2870/50462 | Loss: 3.1135 | LR: 4.98e-04 | 3255.5ms/step | 18.4 samples/s +Step 2880/50462 | Loss: 3.1087 | LR: 4.98e-04 | 3200.7ms/step | 18.7 samples/s +Step 2890/50462 | Loss: 3.0911 | LR: 4.98e-04 | 3200.0ms/step | 18.8 samples/s +Step 2900/50462 | Loss: 3.1136 | LR: 4.98e-04 | 3252.1ms/step | 18.4 samples/s +Step 2910/50462 | Loss: 3.1234 | LR: 4.98e-04 | 3197.9ms/step | 18.8 samples/s +Step 2920/50462 | Loss: 3.1143 | LR: 4.98e-04 | 3183.9ms/step | 18.8 samples/s +Step 2930/50462 | Loss: 3.0961 | LR: 4.98e-04 | 3176.0ms/step | 18.9 samples/s +Step 2940/50462 | Loss: 3.0846 | LR: 4.98e-04 | 3179.9ms/step | 18.9 samples/s +Step 2950/50462 | Loss: 3.1226 | LR: 4.98e-04 | 3191.4ms/step | 18.8 samples/s +Step 2960/50462 | Loss: 3.0820 | LR: 4.98e-04 | 3194.8ms/step | 18.8 samples/s +Step 2970/50462 | Loss: 3.1137 | LR: 4.98e-04 | 3200.0ms/step | 18.7 samples/s +Step 2980/50462 | Loss: 3.1259 | LR: 4.98e-04 | 3201.9ms/step | 18.7 samples/s +Step 2990/50462 | Loss: 3.1034 | LR: 4.97e-04 | 3199.3ms/step | 18.8 samples/s +Step 3000/50462 | Loss: 3.1062 | LR: 4.97e-04 | 3192.4ms/step | 18.8 samples/s +Validation at step 3000: Loss = 3.1100, Perplexity = 22.42 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-3000 +Step 3010/50462 | Loss: 3.1020 | LR: 4.97e-04 | 5819.0ms/step | 10.3 samples/s +Step 3020/50462 | Loss: 3.0888 | LR: 4.97e-04 | 3181.1ms/step | 18.9 samples/s +Step 3030/50462 | Loss: 3.0832 | LR: 4.97e-04 | 3191.5ms/step | 18.8 samples/s +Step 3040/50462 | Loss: 3.1030 | LR: 4.97e-04 | 3182.7ms/step | 18.9 samples/s +Step 3050/50462 | Loss: 3.1030 | LR: 4.97e-04 | 3187.7ms/step | 18.8 samples/s +Step 3060/50462 | Loss: 3.0848 | LR: 4.97e-04 | 3194.3ms/step | 18.8 samples/s +Step 3070/50462 | Loss: 3.1241 | LR: 4.97e-04 | 3199.2ms/step | 18.8 samples/s +Step 3080/50462 | Loss: 3.1208 | LR: 4.97e-04 | 3200.7ms/step | 18.7 samples/s +Step 3090/50462 | Loss: 3.0969 | LR: 4.97e-04 | 3200.0ms/step | 18.7 samples/s +Step 3100/50462 | Loss: 3.0926 | LR: 4.97e-04 | 3200.0ms/step | 18.8 samples/s +Step 3110/50462 | Loss: 3.1011 | LR: 4.97e-04 | 3256.0ms/step | 18.4 samples/s +Step 3120/50462 | Loss: 3.1024 | LR: 4.97e-04 | 3200.0ms/step | 18.7 samples/s +Step 3130/50462 | Loss: 3.1103 | LR: 4.97e-04 | 3197.3ms/step | 18.8 samples/s +Step 3140/50462 | Loss: 3.0907 | LR: 4.97e-04 | 3196.9ms/step | 18.8 samples/s +Step 3150/50462 | Loss: 3.0918 | LR: 4.97e-04 | 3199.9ms/step | 18.8 samples/s +Step 3160/50462 | Loss: 3.0797 | LR: 4.97e-04 | 3202.2ms/step | 18.7 samples/s +Step 3170/50462 | Loss: 3.0616 | LR: 4.97e-04 | 3200.2ms/step | 18.7 samples/s +Step 3180/50462 | Loss: 3.0601 | LR: 4.97e-04 | 3201.5ms/step | 18.7 samples/s +Step 3190/50462 | Loss: 3.0636 | LR: 4.97e-04 | 3194.2ms/step | 18.8 samples/s +Step 3200/50462 | Loss: 3.0786 | LR: 4.97e-04 | 3191.7ms/step | 18.8 samples/s +Step 3210/50462 | Loss: 3.0880 | LR: 4.97e-04 | 3256.0ms/step | 18.4 samples/s +Step 3220/50462 | Loss: 3.0721 | LR: 4.97e-04 | 3202.6ms/step | 18.7 samples/s +Step 3230/50462 | Loss: 3.0844 | LR: 4.97e-04 | 3263.3ms/step | 18.4 samples/s +Step 3240/50462 | Loss: 3.0934 | LR: 4.97e-04 | 3200.3ms/step | 18.7 samples/s +Step 3250/50462 | Loss: 3.0626 | LR: 4.97e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 3250: Loss = 3.0823, Perplexity = 21.81 +Step 3260/50462 | Loss: 3.0355 | LR: 4.97e-04 | 4876.2ms/step | 12.3 samples/s +Step 3270/50462 | Loss: 3.1016 | LR: 4.97e-04 | 3199.9ms/step | 18.8 samples/s +Step 3280/50462 | Loss: 3.0488 | LR: 4.97e-04 | 3200.1ms/step | 18.7 samples/s +Step 3290/50462 | Loss: 3.0790 | LR: 4.97e-04 | 3200.0ms/step | 18.8 samples/s +Step 3300/50462 | Loss: 3.0706 | LR: 4.97e-04 | 3199.9ms/step | 18.8 samples/s +Step 3310/50462 | Loss: 3.0529 | LR: 4.97e-04 | 3200.1ms/step | 18.7 samples/s +Step 3320/50462 | Loss: 3.0611 | LR: 4.97e-04 | 3199.9ms/step | 18.8 samples/s +Step 3330/50462 | Loss: 3.0777 | LR: 4.97e-04 | 3200.0ms/step | 18.8 samples/s +Step 3340/50462 | Loss: 3.0637 | LR: 4.97e-04 | 3200.2ms/step | 18.7 samples/s +Step 3350/50462 | Loss: 3.0358 | LR: 4.97e-04 | 3199.8ms/step | 18.8 samples/s +Step 3360/50462 | Loss: 3.0566 | LR: 4.97e-04 | 3200.9ms/step | 18.7 samples/s +Step 3370/50462 | Loss: 3.0580 | LR: 4.97e-04 | 3201.4ms/step | 18.7 samples/s +Step 3380/50462 | Loss: 3.0767 | LR: 4.97e-04 | 3200.0ms/step | 18.7 samples/s +Step 3390/50462 | Loss: 3.0665 | LR: 4.97e-04 | 3199.7ms/step | 18.8 samples/s +Step 3400/50462 | Loss: 3.0669 | LR: 4.97e-04 | 3198.1ms/step | 18.8 samples/s +Step 3410/50462 | Loss: 3.0689 | LR: 4.96e-04 | 3199.3ms/step | 18.8 samples/s +Step 3420/50462 | Loss: 3.0675 | LR: 4.96e-04 | 3198.5ms/step | 18.8 samples/s +Step 3430/50462 | Loss: 3.0547 | LR: 4.96e-04 | 3200.1ms/step | 18.7 samples/s +Step 3440/50462 | Loss: 3.0645 | LR: 4.96e-04 | 3256.1ms/step | 18.4 samples/s +Step 3450/50462 | Loss: 3.0331 | LR: 4.96e-04 | 3195.6ms/step | 18.8 samples/s +Step 3460/50462 | Loss: 3.0334 | LR: 4.96e-04 | 3198.4ms/step | 18.8 samples/s +Step 3470/50462 | Loss: 3.0572 | LR: 4.96e-04 | 3200.1ms/step | 18.7 samples/s +Step 3480/50462 | Loss: 3.0380 | LR: 4.96e-04 | 3200.0ms/step | 18.7 samples/s +Step 3490/50462 | Loss: 3.0546 | LR: 4.96e-04 | 3196.0ms/step | 18.8 samples/s +Step 3500/50462 | Loss: 3.0417 | LR: 4.96e-04 | 3195.6ms/step | 18.8 samples/s +Validation at step 3500: Loss = 3.0574, Perplexity = 21.27 +Step 3510/50462 | Loss: 3.0351 | LR: 4.96e-04 | 4871.8ms/step | 12.3 samples/s +Step 3520/50462 | Loss: 3.0351 | LR: 4.96e-04 | 3190.0ms/step | 18.8 samples/s +Step 3530/50462 | Loss: 3.0728 | LR: 4.96e-04 | 3184.7ms/step | 18.8 samples/s +Step 3540/50462 | Loss: 3.0184 | LR: 4.96e-04 | 3192.4ms/step | 18.8 samples/s +Step 3550/50462 | Loss: 3.0510 | LR: 4.96e-04 | 3249.5ms/step | 18.5 samples/s +Step 3560/50462 | Loss: 3.0385 | LR: 4.96e-04 | 3197.5ms/step | 18.8 samples/s +Step 3570/50462 | Loss: 3.0556 | LR: 4.96e-04 | 3254.6ms/step | 18.4 samples/s +Step 3580/50462 | Loss: 3.0302 | LR: 4.96e-04 | 3199.8ms/step | 18.8 samples/s +Step 3590/50462 | Loss: 3.0133 | LR: 4.96e-04 | 3200.1ms/step | 18.7 samples/s +Step 3600/50462 | Loss: 3.0208 | LR: 4.96e-04 | 3200.1ms/step | 18.7 samples/s +Step 3610/50462 | Loss: 3.0482 | LR: 4.96e-04 | 3200.0ms/step | 18.8 samples/s +Step 3620/50462 | Loss: 3.0610 | LR: 4.96e-04 | 3200.0ms/step | 18.8 samples/s +Step 3630/50462 | Loss: 3.0359 | LR: 4.96e-04 | 3198.0ms/step | 18.8 samples/s +Step 3640/50462 | Loss: 3.0183 | LR: 4.96e-04 | 3200.0ms/step | 18.7 samples/s +Step 3650/50462 | Loss: 3.0075 | LR: 4.96e-04 | 3201.9ms/step | 18.7 samples/s +Step 3660/50462 | Loss: 3.0523 | LR: 4.96e-04 | 3200.1ms/step | 18.7 samples/s +Step 3670/50462 | Loss: 3.0532 | LR: 4.96e-04 | 3199.8ms/step | 18.8 samples/s +Step 3680/50462 | Loss: 2.9929 | LR: 4.96e-04 | 3200.1ms/step | 18.7 samples/s +Step 3690/50462 | Loss: 3.0554 | LR: 4.96e-04 | 3199.9ms/step | 18.8 samples/s +Step 3700/50462 | Loss: 3.0040 | LR: 4.96e-04 | 3197.7ms/step | 18.8 samples/s +Step 3710/50462 | Loss: 3.0334 | LR: 4.96e-04 | 3200.3ms/step | 18.7 samples/s +Step 3720/50462 | Loss: 3.0573 | LR: 4.96e-04 | 3200.3ms/step | 18.7 samples/s +Step 3730/50462 | Loss: 3.0300 | LR: 4.96e-04 | 3199.8ms/step | 18.8 samples/s +Step 3740/50462 | Loss: 3.0416 | LR: 4.96e-04 | 3200.0ms/step | 18.8 samples/s +Step 3750/50462 | Loss: 3.0294 | LR: 4.96e-04 | 3200.9ms/step | 18.7 samples/s +Validation at step 3750: Loss = 3.0352, Perplexity = 20.81 +Step 3760/50462 | Loss: 3.0202 | LR: 4.95e-04 | 4881.2ms/step | 12.3 samples/s +Step 3770/50462 | Loss: 3.0199 | LR: 4.95e-04 | 3199.9ms/step | 18.8 samples/s +Step 3780/50462 | Loss: 3.0020 | LR: 4.95e-04 | 3254.0ms/step | 18.4 samples/s +Step 3790/50462 | Loss: 3.0221 | LR: 4.95e-04 | 3196.0ms/step | 18.8 samples/s +Step 3800/50462 | Loss: 3.0212 | LR: 4.95e-04 | 3198.7ms/step | 18.8 samples/s +Step 3810/50462 | Loss: 3.0350 | LR: 4.95e-04 | 3199.5ms/step | 18.8 samples/s +Step 3820/50462 | Loss: 3.0162 | LR: 4.95e-04 | 3197.5ms/step | 18.8 samples/s +Step 3830/50462 | Loss: 3.0260 | LR: 4.95e-04 | 3200.5ms/step | 18.7 samples/s +Step 3840/50462 | Loss: 3.0053 | LR: 4.95e-04 | 3197.2ms/step | 18.8 samples/s +Step 3850/50462 | Loss: 2.9607 | LR: 4.95e-04 | 3191.0ms/step | 18.8 samples/s +Step 3860/50462 | Loss: 3.0222 | LR: 4.95e-04 | 3196.1ms/step | 18.8 samples/s +Step 3870/50462 | Loss: 3.0145 | LR: 4.95e-04 | 3199.7ms/step | 18.8 samples/s +Step 3880/50462 | Loss: 3.0274 | LR: 4.95e-04 | 3253.9ms/step | 18.4 samples/s +Step 3890/50462 | Loss: 3.0085 | LR: 4.95e-04 | 3199.9ms/step | 18.8 samples/s +Step 3900/50462 | Loss: 2.9917 | LR: 4.95e-04 | 3200.2ms/step | 18.7 samples/s +Step 3910/50462 | Loss: 2.9827 | LR: 4.95e-04 | 3249.9ms/step | 18.5 samples/s +Step 3920/50462 | Loss: 3.0463 | LR: 4.95e-04 | 3193.3ms/step | 18.8 samples/s +Step 3930/50462 | Loss: 3.0280 | LR: 4.95e-04 | 3196.2ms/step | 18.8 samples/s +Step 3940/50462 | Loss: 3.0218 | LR: 4.95e-04 | 3193.4ms/step | 18.8 samples/s +Step 3950/50462 | Loss: 3.0246 | LR: 4.95e-04 | 3181.3ms/step | 18.9 samples/s +Step 3960/50462 | Loss: 2.9914 | LR: 4.95e-04 | 3191.3ms/step | 18.8 samples/s +Step 3970/50462 | Loss: 3.0284 | LR: 4.95e-04 | 3195.9ms/step | 18.8 samples/s +Step 3980/50462 | Loss: 3.0017 | LR: 4.95e-04 | 3201.0ms/step | 18.7 samples/s +Step 3990/50462 | Loss: 3.0041 | LR: 4.95e-04 | 3202.0ms/step | 18.7 samples/s +Step 4000/50462 | Loss: 3.0424 | LR: 4.95e-04 | 3199.1ms/step | 18.8 samples/s +Validation at step 4000: Loss = 3.0150, Perplexity = 20.39 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-4000 +Step 4010/50462 | Loss: 3.0123 | LR: 4.95e-04 | 5861.8ms/step | 10.2 samples/s +Step 4020/50462 | Loss: 2.9941 | LR: 4.95e-04 | 3188.8ms/step | 18.8 samples/s +Step 4030/50462 | Loss: 3.0017 | LR: 4.95e-04 | 3199.8ms/step | 18.8 samples/s +Step 4040/50462 | Loss: 2.9977 | LR: 4.95e-04 | 3200.2ms/step | 18.7 samples/s +Step 4050/50462 | Loss: 2.9835 | LR: 4.95e-04 | 3198.5ms/step | 18.8 samples/s +Step 4060/50462 | Loss: 3.0031 | LR: 4.95e-04 | 3193.0ms/step | 18.8 samples/s +Step 4070/50462 | Loss: 2.9762 | LR: 4.95e-04 | 3198.4ms/step | 18.8 samples/s +Step 4080/50462 | Loss: 2.9987 | LR: 4.94e-04 | 3197.4ms/step | 18.8 samples/s +Step 4090/50462 | Loss: 3.0342 | LR: 4.94e-04 | 3198.5ms/step | 18.8 samples/s +Step 4100/50462 | Loss: 2.9929 | LR: 4.94e-04 | 3199.3ms/step | 18.8 samples/s +Step 4110/50462 | Loss: 2.9951 | LR: 4.94e-04 | 3255.4ms/step | 18.4 samples/s +Step 4120/50462 | Loss: 2.9838 | LR: 4.94e-04 | 3190.8ms/step | 18.8 samples/s +Step 4130/50462 | Loss: 3.0000 | LR: 4.94e-04 | 3192.0ms/step | 18.8 samples/s +Step 4140/50462 | Loss: 2.9786 | LR: 4.94e-04 | 3190.0ms/step | 18.8 samples/s +Step 4150/50462 | Loss: 3.0068 | LR: 4.94e-04 | 3193.9ms/step | 18.8 samples/s +Step 4160/50462 | Loss: 3.0040 | LR: 4.94e-04 | 3192.7ms/step | 18.8 samples/s +Step 4170/50462 | Loss: 2.9746 | LR: 4.94e-04 | 3192.7ms/step | 18.8 samples/s +Step 4180/50462 | Loss: 2.9845 | LR: 4.94e-04 | 3191.0ms/step | 18.8 samples/s +Step 4190/50462 | Loss: 2.9725 | LR: 4.94e-04 | 3196.4ms/step | 18.8 samples/s +Step 4200/50462 | Loss: 2.9711 | LR: 4.94e-04 | 3191.6ms/step | 18.8 samples/s +Step 4210/50462 | Loss: 2.9750 | LR: 4.94e-04 | 3183.5ms/step | 18.8 samples/s +Step 4220/50462 | Loss: 2.9859 | LR: 4.94e-04 | 3238.9ms/step | 18.5 samples/s +Step 4230/50462 | Loss: 2.9758 | LR: 4.94e-04 | 3200.0ms/step | 18.8 samples/s +Step 4240/50462 | Loss: 2.9720 | LR: 4.94e-04 | 3199.9ms/step | 18.8 samples/s +Step 4250/50462 | Loss: 2.9830 | LR: 4.94e-04 | 3249.4ms/step | 18.5 samples/s +Validation at step 4250: Loss = 2.9946, Perplexity = 19.98 +Step 4260/50462 | Loss: 2.9748 | LR: 4.94e-04 | 4863.1ms/step | 12.3 samples/s +Step 4270/50462 | Loss: 2.9680 | LR: 4.94e-04 | 3181.5ms/step | 18.9 samples/s +Step 4280/50462 | Loss: 2.9813 | LR: 4.94e-04 | 3188.5ms/step | 18.8 samples/s +Step 4290/50462 | Loss: 2.9911 | LR: 4.94e-04 | 3194.0ms/step | 18.8 samples/s +Step 4300/50462 | Loss: 2.9794 | LR: 4.94e-04 | 3181.8ms/step | 18.9 samples/s +Step 4310/50462 | Loss: 3.0002 | LR: 4.94e-04 | 3188.1ms/step | 18.8 samples/s +Step 4320/50462 | Loss: 3.0059 | LR: 4.94e-04 | 3184.1ms/step | 18.8 samples/s +Step 4330/50462 | Loss: 2.9867 | LR: 4.94e-04 | 3191.1ms/step | 18.8 samples/s +Step 4340/50462 | Loss: 2.9641 | LR: 4.94e-04 | 3189.2ms/step | 18.8 samples/s +Step 4350/50462 | Loss: 2.9728 | LR: 4.94e-04 | 3186.4ms/step | 18.8 samples/s +Step 4360/50462 | Loss: 2.9609 | LR: 4.94e-04 | 3196.8ms/step | 18.8 samples/s +Step 4370/50462 | Loss: 2.9822 | LR: 4.93e-04 | 3192.6ms/step | 18.8 samples/s +Step 4380/50462 | Loss: 2.9848 | LR: 4.93e-04 | 3189.3ms/step | 18.8 samples/s +Step 4390/50462 | Loss: 2.9828 | LR: 4.93e-04 | 3195.8ms/step | 18.8 samples/s +Step 4400/50462 | Loss: 3.0015 | LR: 4.93e-04 | 3193.9ms/step | 18.8 samples/s +Step 4410/50462 | Loss: 2.9923 | LR: 4.93e-04 | 3196.2ms/step | 18.8 samples/s +Step 4420/50462 | Loss: 2.9701 | LR: 4.93e-04 | 3197.4ms/step | 18.8 samples/s +Step 4430/50462 | Loss: 2.9851 | LR: 4.93e-04 | 3195.5ms/step | 18.8 samples/s +Step 4440/50462 | Loss: 2.9608 | LR: 4.93e-04 | 3199.1ms/step | 18.8 samples/s +Step 4450/50462 | Loss: 2.9624 | LR: 4.93e-04 | 3254.0ms/step | 18.4 samples/s +Step 4460/50462 | Loss: 2.9854 | LR: 4.93e-04 | 3196.6ms/step | 18.8 samples/s +Step 4470/50462 | Loss: 2.9629 | LR: 4.93e-04 | 3197.5ms/step | 18.8 samples/s +Step 4480/50462 | Loss: 2.9906 | LR: 4.93e-04 | 3200.0ms/step | 18.8 samples/s +Step 4490/50462 | Loss: 2.9913 | LR: 4.93e-04 | 3200.1ms/step | 18.7 samples/s +Step 4500/50462 | Loss: 2.9363 | LR: 4.93e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 4500: Loss = 2.9785, Perplexity = 19.66 +Step 4510/50462 | Loss: 2.9683 | LR: 4.93e-04 | 4873.1ms/step | 12.3 samples/s +Step 4520/50462 | Loss: 2.9679 | LR: 4.93e-04 | 3200.9ms/step | 18.7 samples/s +Step 4530/50462 | Loss: 2.9787 | LR: 4.93e-04 | 3199.9ms/step | 18.8 samples/s +Step 4540/50462 | Loss: 3.0000 | LR: 4.93e-04 | 3195.0ms/step | 18.8 samples/s +Step 4550/50462 | Loss: 2.9605 | LR: 4.93e-04 | 3199.0ms/step | 18.8 samples/s +Step 4560/50462 | Loss: 2.9680 | LR: 4.93e-04 | 3246.0ms/step | 18.5 samples/s +Step 4570/50462 | Loss: 2.9938 | LR: 4.93e-04 | 3193.8ms/step | 18.8 samples/s +Step 4580/50462 | Loss: 2.9783 | LR: 4.93e-04 | 3249.7ms/step | 18.5 samples/s +Step 4590/50462 | Loss: 2.9602 | LR: 4.93e-04 | 3194.5ms/step | 18.8 samples/s +Step 4600/50462 | Loss: 2.9708 | LR: 4.93e-04 | 3186.9ms/step | 18.8 samples/s +Step 4610/50462 | Loss: 2.9475 | LR: 4.93e-04 | 3194.9ms/step | 18.8 samples/s +Step 4620/50462 | Loss: 2.9672 | LR: 4.93e-04 | 3194.2ms/step | 18.8 samples/s +Step 4630/50462 | Loss: 2.9447 | LR: 4.93e-04 | 3199.9ms/step | 18.8 samples/s +Step 4640/50462 | Loss: 2.9386 | LR: 4.92e-04 | 3200.1ms/step | 18.7 samples/s +Step 4650/50462 | Loss: 2.9369 | LR: 4.92e-04 | 3188.9ms/step | 18.8 samples/s +Step 4660/50462 | Loss: 2.9397 | LR: 4.92e-04 | 3185.5ms/step | 18.8 samples/s +Step 4670/50462 | Loss: 2.9460 | LR: 4.92e-04 | 3197.4ms/step | 18.8 samples/s +Step 4680/50462 | Loss: 2.9619 | LR: 4.92e-04 | 3199.3ms/step | 18.8 samples/s +Step 4690/50462 | Loss: 2.9388 | LR: 4.92e-04 | 3200.7ms/step | 18.7 samples/s +Step 4700/50462 | Loss: 2.9488 | LR: 4.92e-04 | 3200.2ms/step | 18.7 samples/s +Step 4710/50462 | Loss: 2.9887 | LR: 4.92e-04 | 3199.7ms/step | 18.8 samples/s +Step 4720/50462 | Loss: 2.9529 | LR: 4.92e-04 | 3200.2ms/step | 18.7 samples/s +Step 4730/50462 | Loss: 2.9782 | LR: 4.92e-04 | 3185.8ms/step | 18.8 samples/s +Step 4740/50462 | Loss: 2.9443 | LR: 4.92e-04 | 3187.4ms/step | 18.8 samples/s +Step 4750/50462 | Loss: 2.9590 | LR: 4.92e-04 | 3198.7ms/step | 18.8 samples/s +Validation at step 4750: Loss = 2.9628, Perplexity = 19.35 +Step 4760/50462 | Loss: 2.9566 | LR: 4.92e-04 | 4874.0ms/step | 12.3 samples/s +Step 4770/50462 | Loss: 2.9377 | LR: 4.92e-04 | 3191.9ms/step | 18.8 samples/s +Step 4780/50462 | Loss: 2.9354 | LR: 4.92e-04 | 3192.7ms/step | 18.8 samples/s +Step 4790/50462 | Loss: 2.9628 | LR: 4.92e-04 | 3253.3ms/step | 18.4 samples/s +Step 4800/50462 | Loss: 2.9482 | LR: 4.92e-04 | 3199.9ms/step | 18.8 samples/s +Step 4810/50462 | Loss: 3.0026 | LR: 4.92e-04 | 3196.3ms/step | 18.8 samples/s +Step 4820/50462 | Loss: 2.9840 | LR: 4.92e-04 | 3198.0ms/step | 18.8 samples/s +Step 4830/50462 | Loss: 2.9267 | LR: 4.92e-04 | 3200.0ms/step | 18.8 samples/s +Step 4840/50462 | Loss: 2.9323 | LR: 4.92e-04 | 3200.0ms/step | 18.8 samples/s +Step 4850/50462 | Loss: 2.9293 | LR: 4.92e-04 | 3199.7ms/step | 18.8 samples/s +Step 4860/50462 | Loss: 2.9368 | LR: 4.92e-04 | 3200.3ms/step | 18.7 samples/s +Step 4870/50462 | Loss: 2.9145 | LR: 4.92e-04 | 3200.1ms/step | 18.7 samples/s +Step 4880/50462 | Loss: 2.9473 | LR: 4.92e-04 | 3199.9ms/step | 18.8 samples/s +Step 4890/50462 | Loss: 2.8968 | LR: 4.91e-04 | 3251.2ms/step | 18.5 samples/s +Step 4900/50462 | Loss: 2.9388 | LR: 4.91e-04 | 3188.7ms/step | 18.8 samples/s +Step 4910/50462 | Loss: 2.9701 | LR: 4.91e-04 | 3180.5ms/step | 18.9 samples/s +Step 4920/50462 | Loss: 2.9382 | LR: 4.91e-04 | 3241.2ms/step | 18.5 samples/s +Step 4930/50462 | Loss: 2.9614 | LR: 4.91e-04 | 3186.0ms/step | 18.8 samples/s +Step 4940/50462 | Loss: 2.9403 | LR: 4.91e-04 | 3190.2ms/step | 18.8 samples/s +Step 4950/50462 | Loss: 2.9850 | LR: 4.91e-04 | 3198.1ms/step | 18.8 samples/s +Step 4960/50462 | Loss: 2.9614 | LR: 4.91e-04 | 3199.6ms/step | 18.8 samples/s +Step 4970/50462 | Loss: 2.9698 | LR: 4.91e-04 | 3200.2ms/step | 18.7 samples/s +Step 4980/50462 | Loss: 2.9255 | LR: 4.91e-04 | 3200.3ms/step | 18.7 samples/s +Step 4990/50462 | Loss: 2.9295 | LR: 4.91e-04 | 3199.1ms/step | 18.8 samples/s +Step 5000/50462 | Loss: 2.9694 | LR: 4.91e-04 | 3200.6ms/step | 18.7 samples/s +Validation at step 5000: Loss = 2.9490, Perplexity = 19.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-5000 +Step 5010/50462 | Loss: 2.9407 | LR: 4.91e-04 | 5836.4ms/step | 10.3 samples/s +Step 5020/50462 | Loss: 2.9568 | LR: 4.91e-04 | 3180.9ms/step | 18.9 samples/s +Step 5030/50462 | Loss: 2.9209 | LR: 4.91e-04 | 3190.1ms/step | 18.8 samples/s +Step 5040/50462 | Loss: 2.9220 | LR: 4.91e-04 | 3200.7ms/step | 18.7 samples/s +Step 5050/50462 | Loss: 2.9274 | LR: 4.91e-04 | 3199.6ms/step | 18.8 samples/s +Step 5060/50462 | Loss: 2.9083 | LR: 4.91e-04 | 3200.1ms/step | 18.7 samples/s +Step 5070/50462 | Loss: 2.9403 | LR: 4.91e-04 | 3192.1ms/step | 18.8 samples/s +Step 5080/50462 | Loss: 2.9620 | LR: 4.91e-04 | 3200.2ms/step | 18.7 samples/s +Step 5090/50462 | Loss: 2.9293 | LR: 4.91e-04 | 3200.0ms/step | 18.8 samples/s +Step 5100/50462 | Loss: 2.9090 | LR: 4.91e-04 | 3198.0ms/step | 18.8 samples/s +Step 5110/50462 | Loss: 2.9195 | LR: 4.91e-04 | 3199.3ms/step | 18.8 samples/s +Step 5120/50462 | Loss: 2.9208 | LR: 4.91e-04 | 3248.6ms/step | 18.5 samples/s +Step 5130/50462 | Loss: 2.9039 | LR: 4.90e-04 | 3200.0ms/step | 18.8 samples/s +Step 5140/50462 | Loss: 2.9183 | LR: 4.90e-04 | 3200.0ms/step | 18.8 samples/s +Step 5150/50462 | Loss: 2.9349 | LR: 4.90e-04 | 3200.3ms/step | 18.7 samples/s +Step 5160/50462 | Loss: 2.8993 | LR: 4.90e-04 | 3199.7ms/step | 18.8 samples/s +Step 5170/50462 | Loss: 2.9451 | LR: 4.90e-04 | 3199.7ms/step | 18.8 samples/s +Step 5180/50462 | Loss: 2.9244 | LR: 4.90e-04 | 3200.6ms/step | 18.7 samples/s +Step 5190/50462 | Loss: 2.9057 | LR: 4.90e-04 | 3199.7ms/step | 18.8 samples/s +Step 5200/50462 | Loss: 2.9354 | LR: 4.90e-04 | 3200.1ms/step | 18.7 samples/s +Step 5210/50462 | Loss: 2.9112 | LR: 4.90e-04 | 3200.1ms/step | 18.7 samples/s +Step 5220/50462 | Loss: 2.9143 | LR: 4.90e-04 | 3199.9ms/step | 18.8 samples/s +Step 5230/50462 | Loss: 2.9716 | LR: 4.90e-04 | 3258.2ms/step | 18.4 samples/s +Step 5240/50462 | Loss: 2.9431 | LR: 4.90e-04 | 3198.6ms/step | 18.8 samples/s +Step 5250/50462 | Loss: 2.9353 | LR: 4.90e-04 | 3195.2ms/step | 18.8 samples/s +Validation at step 5250: Loss = 2.9363, Perplexity = 18.85 +Step 5260/50462 | Loss: 2.9491 | LR: 4.90e-04 | 4920.5ms/step | 12.2 samples/s +Step 5270/50462 | Loss: 2.9371 | LR: 4.90e-04 | 3194.8ms/step | 18.8 samples/s +Step 5280/50462 | Loss: 2.9488 | LR: 4.90e-04 | 3192.0ms/step | 18.8 samples/s +Step 5290/50462 | Loss: 2.9743 | LR: 4.90e-04 | 3192.8ms/step | 18.8 samples/s +Step 5300/50462 | Loss: 2.9092 | LR: 4.90e-04 | 3199.9ms/step | 18.8 samples/s +Step 5310/50462 | Loss: 2.9366 | LR: 4.90e-04 | 3197.8ms/step | 18.8 samples/s +Step 5320/50462 | Loss: 2.9402 | LR: 4.90e-04 | 3200.3ms/step | 18.7 samples/s +Step 5330/50462 | Loss: 2.9156 | LR: 4.90e-04 | 3195.8ms/step | 18.8 samples/s +Step 5340/50462 | Loss: 2.9138 | LR: 4.90e-04 | 3197.6ms/step | 18.8 samples/s +Step 5350/50462 | Loss: 2.9585 | LR: 4.90e-04 | 3198.3ms/step | 18.8 samples/s +Step 5360/50462 | Loss: 2.9364 | LR: 4.89e-04 | 3200.0ms/step | 18.7 samples/s +Step 5370/50462 | Loss: 2.9391 | LR: 4.89e-04 | 3195.8ms/step | 18.8 samples/s +Step 5380/50462 | Loss: 2.9075 | LR: 4.89e-04 | 3194.2ms/step | 18.8 samples/s +Step 5390/50462 | Loss: 2.9509 | LR: 4.89e-04 | 3199.8ms/step | 18.8 samples/s +Step 5400/50462 | Loss: 2.9364 | LR: 4.89e-04 | 3200.2ms/step | 18.7 samples/s +Step 5410/50462 | Loss: 2.9323 | LR: 4.89e-04 | 3199.9ms/step | 18.8 samples/s +Step 5420/50462 | Loss: 2.9219 | LR: 4.89e-04 | 3201.9ms/step | 18.7 samples/s +Step 5430/50462 | Loss: 2.9317 | LR: 4.89e-04 | 3200.4ms/step | 18.7 samples/s +Step 5440/50462 | Loss: 2.9296 | LR: 4.89e-04 | 3197.2ms/step | 18.8 samples/s +Step 5450/50462 | Loss: 2.9179 | LR: 4.89e-04 | 3195.9ms/step | 18.8 samples/s +Step 5460/50462 | Loss: 2.9053 | LR: 4.89e-04 | 3241.9ms/step | 18.5 samples/s +Step 5470/50462 | Loss: 2.9158 | LR: 4.89e-04 | 3198.8ms/step | 18.8 samples/s +Step 5480/50462 | Loss: 2.9054 | LR: 4.89e-04 | 3199.7ms/step | 18.8 samples/s +Step 5490/50462 | Loss: 2.8958 | LR: 4.89e-04 | 3200.2ms/step | 18.7 samples/s +Step 5500/50462 | Loss: 2.8921 | LR: 4.89e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 5500: Loss = 2.9245, Perplexity = 18.62 +Step 5510/50462 | Loss: 2.9347 | LR: 4.89e-04 | 4875.4ms/step | 12.3 samples/s +Step 5520/50462 | Loss: 2.8869 | LR: 4.89e-04 | 3199.8ms/step | 18.8 samples/s +Step 5530/50462 | Loss: 2.8838 | LR: 4.89e-04 | 3190.6ms/step | 18.8 samples/s +Step 5540/50462 | Loss: 2.9111 | LR: 4.89e-04 | 3197.7ms/step | 18.8 samples/s +Step 5550/50462 | Loss: 2.9240 | LR: 4.89e-04 | 3200.4ms/step | 18.7 samples/s +Step 5560/50462 | Loss: 2.9218 | LR: 4.89e-04 | 3199.7ms/step | 18.8 samples/s +Step 5570/50462 | Loss: 2.9096 | LR: 4.88e-04 | 3254.3ms/step | 18.4 samples/s +Step 5580/50462 | Loss: 2.9061 | LR: 4.88e-04 | 3199.8ms/step | 18.8 samples/s +Step 5590/50462 | Loss: 2.8939 | LR: 4.88e-04 | 3198.0ms/step | 18.8 samples/s +Step 5600/50462 | Loss: 2.8869 | LR: 4.88e-04 | 3254.0ms/step | 18.4 samples/s +Step 5610/50462 | Loss: 2.9121 | LR: 4.88e-04 | 3200.1ms/step | 18.7 samples/s +Step 5620/50462 | Loss: 2.9011 | LR: 4.88e-04 | 3200.2ms/step | 18.7 samples/s +Step 5630/50462 | Loss: 2.8867 | LR: 4.88e-04 | 3199.8ms/step | 18.8 samples/s +Step 5640/50462 | Loss: 2.9112 | LR: 4.88e-04 | 3199.9ms/step | 18.8 samples/s +Step 5650/50462 | Loss: 2.8835 | LR: 4.88e-04 | 3200.2ms/step | 18.7 samples/s +Step 5660/50462 | Loss: 2.8742 | LR: 4.88e-04 | 3199.7ms/step | 18.8 samples/s +Step 5670/50462 | Loss: 2.8656 | LR: 4.88e-04 | 3200.2ms/step | 18.7 samples/s +Step 5680/50462 | Loss: 2.9175 | LR: 4.88e-04 | 3199.9ms/step | 18.8 samples/s +Step 5690/50462 | Loss: 2.9041 | LR: 4.88e-04 | 3199.9ms/step | 18.8 samples/s +Step 5700/50462 | Loss: 2.9059 | LR: 4.88e-04 | 3199.6ms/step | 18.8 samples/s +Step 5710/50462 | Loss: 2.9421 | LR: 4.88e-04 | 3182.6ms/step | 18.9 samples/s +Step 5720/50462 | Loss: 2.9042 | LR: 4.88e-04 | 3185.7ms/step | 18.8 samples/s +Step 5730/50462 | Loss: 2.8930 | LR: 4.88e-04 | 3192.1ms/step | 18.8 samples/s +Step 5740/50462 | Loss: 2.8969 | LR: 4.88e-04 | 3192.6ms/step | 18.8 samples/s +Step 5750/50462 | Loss: 2.8912 | LR: 4.88e-04 | 3192.3ms/step | 18.8 samples/s +Validation at step 5750: Loss = 2.9137, Perplexity = 18.43 +Step 5760/50462 | Loss: 2.9162 | LR: 4.88e-04 | 4874.7ms/step | 12.3 samples/s +Step 5770/50462 | Loss: 2.9189 | LR: 4.88e-04 | 3193.6ms/step | 18.8 samples/s +Step 5780/50462 | Loss: 2.9066 | LR: 4.87e-04 | 3195.3ms/step | 18.8 samples/s +Step 5790/50462 | Loss: 2.8744 | LR: 4.87e-04 | 3253.7ms/step | 18.4 samples/s +Step 5800/50462 | Loss: 2.9011 | LR: 4.87e-04 | 3197.8ms/step | 18.8 samples/s +Step 5810/50462 | Loss: 2.8899 | LR: 4.87e-04 | 3200.3ms/step | 18.7 samples/s +Step 5820/50462 | Loss: 2.8915 | LR: 4.87e-04 | 3199.5ms/step | 18.8 samples/s +Step 5830/50462 | Loss: 2.8871 | LR: 4.87e-04 | 3199.9ms/step | 18.8 samples/s +Step 5840/50462 | Loss: 2.8619 | LR: 4.87e-04 | 3190.3ms/step | 18.8 samples/s +Step 5850/50462 | Loss: 2.8897 | LR: 4.87e-04 | 3186.6ms/step | 18.8 samples/s +Step 5860/50462 | Loss: 2.8899 | LR: 4.87e-04 | 3197.5ms/step | 18.8 samples/s +Step 5870/50462 | Loss: 2.8992 | LR: 4.87e-04 | 3200.5ms/step | 18.7 samples/s +Step 5880/50462 | Loss: 2.8896 | LR: 4.87e-04 | 3200.9ms/step | 18.7 samples/s +Step 5890/50462 | Loss: 2.8871 | LR: 4.87e-04 | 3200.4ms/step | 18.7 samples/s +Step 5900/50462 | Loss: 2.8631 | LR: 4.87e-04 | 3199.1ms/step | 18.8 samples/s +Step 5910/50462 | Loss: 2.9117 | LR: 4.87e-04 | 3254.5ms/step | 18.4 samples/s +Step 5920/50462 | Loss: 2.8832 | LR: 4.87e-04 | 3200.6ms/step | 18.7 samples/s +Step 5930/50462 | Loss: 2.8889 | LR: 4.87e-04 | 3252.4ms/step | 18.4 samples/s +Step 5940/50462 | Loss: 2.8848 | LR: 4.87e-04 | 3201.7ms/step | 18.7 samples/s +Step 5950/50462 | Loss: 2.8872 | LR: 4.87e-04 | 3199.9ms/step | 18.8 samples/s +Step 5960/50462 | Loss: 2.9167 | LR: 4.87e-04 | 3199.8ms/step | 18.8 samples/s +Step 5970/50462 | Loss: 2.8555 | LR: 4.87e-04 | 3200.3ms/step | 18.7 samples/s +Step 5980/50462 | Loss: 2.8909 | LR: 4.86e-04 | 3199.8ms/step | 18.8 samples/s +Step 5990/50462 | Loss: 2.8553 | LR: 4.86e-04 | 3200.2ms/step | 18.7 samples/s +Step 6000/50462 | Loss: 2.8753 | LR: 4.86e-04 | 3196.0ms/step | 18.8 samples/s +Validation at step 6000: Loss = 2.9052, Perplexity = 18.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-6000 +Step 6010/50462 | Loss: 2.9182 | LR: 4.86e-04 | 5784.5ms/step | 10.4 samples/s +Step 6020/50462 | Loss: 2.8988 | LR: 4.86e-04 | 3177.4ms/step | 18.9 samples/s +Step 6030/50462 | Loss: 2.8791 | LR: 4.86e-04 | 3186.6ms/step | 18.8 samples/s +Step 6040/50462 | Loss: 2.8931 | LR: 4.86e-04 | 3191.4ms/step | 18.8 samples/s +Step 6050/50462 | Loss: 2.8884 | LR: 4.86e-04 | 3193.9ms/step | 18.8 samples/s +Step 6060/50462 | Loss: 2.8763 | LR: 4.86e-04 | 3197.7ms/step | 18.8 samples/s +Step 6070/50462 | Loss: 2.8751 | LR: 4.86e-04 | 3200.5ms/step | 18.7 samples/s +Step 6080/50462 | Loss: 2.8618 | LR: 4.86e-04 | 3196.6ms/step | 18.8 samples/s +Step 6090/50462 | Loss: 2.8875 | LR: 4.86e-04 | 3195.2ms/step | 18.8 samples/s +Step 6100/50462 | Loss: 2.8811 | LR: 4.86e-04 | 3199.9ms/step | 18.8 samples/s +Step 6110/50462 | Loss: 2.9102 | LR: 4.86e-04 | 3200.1ms/step | 18.7 samples/s +Step 6120/50462 | Loss: 2.8721 | LR: 4.86e-04 | 3196.6ms/step | 18.8 samples/s +Step 6130/50462 | Loss: 2.8984 | LR: 4.86e-04 | 3255.1ms/step | 18.4 samples/s +Step 6140/50462 | Loss: 2.9013 | LR: 4.86e-04 | 3192.4ms/step | 18.8 samples/s +Step 6150/50462 | Loss: 2.8475 | LR: 4.86e-04 | 3200.7ms/step | 18.7 samples/s +Step 6160/50462 | Loss: 2.9259 | LR: 4.86e-04 | 3201.3ms/step | 18.7 samples/s +Step 6170/50462 | Loss: 2.8642 | LR: 4.85e-04 | 3200.0ms/step | 18.7 samples/s +Step 6180/50462 | Loss: 2.8829 | LR: 4.85e-04 | 3202.0ms/step | 18.7 samples/s +Step 6190/50462 | Loss: 2.8654 | LR: 4.85e-04 | 3199.9ms/step | 18.8 samples/s +Step 6200/50462 | Loss: 2.8594 | LR: 4.85e-04 | 3200.1ms/step | 18.7 samples/s +Step 6210/50462 | Loss: 2.8705 | LR: 4.85e-04 | 3200.0ms/step | 18.7 samples/s +Step 6220/50462 | Loss: 2.8775 | LR: 4.85e-04 | 3201.2ms/step | 18.7 samples/s +Step 6230/50462 | Loss: 2.8771 | LR: 4.85e-04 | 3198.8ms/step | 18.8 samples/s +Step 6240/50462 | Loss: 2.8958 | LR: 4.85e-04 | 3254.0ms/step | 18.4 samples/s +Step 6250/50462 | Loss: 2.8477 | LR: 4.85e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 6250: Loss = 2.8912, Perplexity = 18.01 +Step 6260/50462 | Loss: 2.8558 | LR: 4.85e-04 | 4876.5ms/step | 12.3 samples/s +Step 6270/50462 | Loss: 2.8902 | LR: 4.85e-04 | 3251.1ms/step | 18.5 samples/s +Step 6280/50462 | Loss: 2.8758 | LR: 4.85e-04 | 3198.6ms/step | 18.8 samples/s +Step 6290/50462 | Loss: 2.8740 | LR: 4.85e-04 | 3194.9ms/step | 18.8 samples/s +Step 6300/50462 | Loss: 2.8916 | LR: 4.85e-04 | 3188.9ms/step | 18.8 samples/s +Step 6310/50462 | Loss: 2.8596 | LR: 4.85e-04 | 3195.0ms/step | 18.8 samples/s +Step 6320/50462 | Loss: 2.8889 | LR: 4.85e-04 | 3188.5ms/step | 18.8 samples/s +Step 6330/50462 | Loss: 2.8456 | LR: 4.85e-04 | 3192.8ms/step | 18.8 samples/s +Step 6340/50462 | Loss: 2.8696 | LR: 4.85e-04 | 3197.6ms/step | 18.8 samples/s +Step 6350/50462 | Loss: 2.8703 | LR: 4.85e-04 | 3199.8ms/step | 18.8 samples/s +Step 6360/50462 | Loss: 2.9028 | LR: 4.84e-04 | 3192.4ms/step | 18.8 samples/s +Step 6370/50462 | Loss: 2.8631 | LR: 4.84e-04 | 3190.2ms/step | 18.8 samples/s +Step 6380/50462 | Loss: 2.8726 | LR: 4.84e-04 | 3199.8ms/step | 18.8 samples/s +Step 6390/50462 | Loss: 2.8511 | LR: 4.84e-04 | 3198.2ms/step | 18.8 samples/s +Step 6400/50462 | Loss: 2.9021 | LR: 4.84e-04 | 3197.4ms/step | 18.8 samples/s +Step 6410/50462 | Loss: 2.8723 | LR: 4.84e-04 | 3198.6ms/step | 18.8 samples/s +Step 6420/50462 | Loss: 2.8741 | LR: 4.84e-04 | 3199.8ms/step | 18.8 samples/s +Step 6430/50462 | Loss: 2.8964 | LR: 4.84e-04 | 3200.0ms/step | 18.8 samples/s +Step 6440/50462 | Loss: 2.8516 | LR: 4.84e-04 | 3200.2ms/step | 18.7 samples/s +Step 6450/50462 | Loss: 2.8501 | LR: 4.84e-04 | 3196.1ms/step | 18.8 samples/s +Step 6460/50462 | Loss: 2.8830 | LR: 4.84e-04 | 3199.4ms/step | 18.8 samples/s +Step 6470/50462 | Loss: 2.8538 | LR: 4.84e-04 | 3254.4ms/step | 18.4 samples/s +Step 6480/50462 | Loss: 2.8756 | LR: 4.84e-04 | 3196.0ms/step | 18.8 samples/s +Step 6490/50462 | Loss: 2.8769 | LR: 4.84e-04 | 3200.2ms/step | 18.7 samples/s +Step 6500/50462 | Loss: 2.8517 | LR: 4.84e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 6500: Loss = 2.8831, Perplexity = 17.87 +Step 6510/50462 | Loss: 2.8427 | LR: 4.84e-04 | 4876.1ms/step | 12.3 samples/s +Step 6520/50462 | Loss: 2.8956 | LR: 4.84e-04 | 3194.8ms/step | 18.8 samples/s +Step 6530/50462 | Loss: 2.8545 | LR: 4.84e-04 | 3197.0ms/step | 18.8 samples/s +Step 6540/50462 | Loss: 2.9032 | LR: 4.83e-04 | 3199.3ms/step | 18.8 samples/s +Step 6550/50462 | Loss: 2.8846 | LR: 4.83e-04 | 3194.2ms/step | 18.8 samples/s +Step 6560/50462 | Loss: 2.8621 | LR: 4.83e-04 | 3192.6ms/step | 18.8 samples/s +Step 6570/50462 | Loss: 2.8743 | LR: 4.83e-04 | 3195.9ms/step | 18.8 samples/s +Step 6580/50462 | Loss: 2.8770 | LR: 4.83e-04 | 3252.2ms/step | 18.4 samples/s +Step 6590/50462 | Loss: 2.8950 | LR: 4.83e-04 | 3199.6ms/step | 18.8 samples/s +Step 6600/50462 | Loss: 2.8715 | LR: 4.83e-04 | 3197.9ms/step | 18.8 samples/s +Step 6610/50462 | Loss: 2.8776 | LR: 4.83e-04 | 3250.3ms/step | 18.5 samples/s +Step 6620/50462 | Loss: 2.8753 | LR: 4.83e-04 | 3200.0ms/step | 18.8 samples/s +Step 6630/50462 | Loss: 2.8692 | LR: 4.83e-04 | 3200.0ms/step | 18.7 samples/s +Step 6640/50462 | Loss: 2.8640 | LR: 4.83e-04 | 3199.5ms/step | 18.8 samples/s +Step 6650/50462 | Loss: 2.8733 | LR: 4.83e-04 | 3200.5ms/step | 18.7 samples/s +Step 6660/50462 | Loss: 2.8717 | LR: 4.83e-04 | 3200.0ms/step | 18.7 samples/s +Step 6670/50462 | Loss: 2.8746 | LR: 4.83e-04 | 3199.3ms/step | 18.8 samples/s +Step 6680/50462 | Loss: 2.8760 | LR: 4.83e-04 | 3200.8ms/step | 18.7 samples/s +Step 6690/50462 | Loss: 2.8690 | LR: 4.83e-04 | 3200.0ms/step | 18.8 samples/s +Step 6700/50462 | Loss: 2.8473 | LR: 4.83e-04 | 3200.1ms/step | 18.7 samples/s +Step 6710/50462 | Loss: 2.8573 | LR: 4.82e-04 | 3200.1ms/step | 18.7 samples/s +Step 6720/50462 | Loss: 2.8585 | LR: 4.82e-04 | 3199.8ms/step | 18.8 samples/s +Step 6730/50462 | Loss: 2.8831 | LR: 4.82e-04 | 3200.0ms/step | 18.7 samples/s +Step 6740/50462 | Loss: 2.8893 | LR: 4.82e-04 | 3199.8ms/step | 18.8 samples/s +Step 6750/50462 | Loss: 2.8550 | LR: 4.82e-04 | 3199.3ms/step | 18.8 samples/s +Validation at step 6750: Loss = 2.8722, Perplexity = 17.68 +Step 6760/50462 | Loss: 2.8818 | LR: 4.82e-04 | 4870.9ms/step | 12.3 samples/s +Step 6770/50462 | Loss: 2.8744 | LR: 4.82e-04 | 3200.0ms/step | 18.8 samples/s +Step 6780/50462 | Loss: 2.8881 | LR: 4.82e-04 | 3200.3ms/step | 18.7 samples/s +Step 6790/50462 | Loss: 2.8754 | LR: 4.82e-04 | 3199.9ms/step | 18.8 samples/s +Step 6800/50462 | Loss: 2.8385 | LR: 4.82e-04 | 3254.8ms/step | 18.4 samples/s +Step 6810/50462 | Loss: 2.8659 | LR: 4.82e-04 | 3188.6ms/step | 18.8 samples/s +Step 6820/50462 | Loss: 2.9085 | LR: 4.82e-04 | 3196.5ms/step | 18.8 samples/s +Step 6830/50462 | Loss: 2.8393 | LR: 4.82e-04 | 3199.8ms/step | 18.8 samples/s +Step 6840/50462 | Loss: 2.8583 | LR: 4.82e-04 | 3197.7ms/step | 18.8 samples/s +Step 6850/50462 | Loss: 2.8247 | LR: 4.82e-04 | 3198.0ms/step | 18.8 samples/s +Step 6860/50462 | Loss: 2.8393 | LR: 4.82e-04 | 3199.9ms/step | 18.8 samples/s +Step 6870/50462 | Loss: 2.8448 | LR: 4.82e-04 | 3198.5ms/step | 18.8 samples/s +Step 6880/50462 | Loss: 2.8706 | LR: 4.81e-04 | 3185.6ms/step | 18.8 samples/s +Step 6890/50462 | Loss: 2.8589 | LR: 4.81e-04 | 3180.3ms/step | 18.9 samples/s +Step 6900/50462 | Loss: 2.8646 | LR: 4.81e-04 | 3191.9ms/step | 18.8 samples/s +Step 6910/50462 | Loss: 2.8463 | LR: 4.81e-04 | 3197.6ms/step | 18.8 samples/s +Step 6920/50462 | Loss: 2.8520 | LR: 4.81e-04 | 3252.2ms/step | 18.4 samples/s +Step 6930/50462 | Loss: 2.8200 | LR: 4.81e-04 | 3199.7ms/step | 18.8 samples/s +Step 6940/50462 | Loss: 2.8486 | LR: 4.81e-04 | 3200.4ms/step | 18.7 samples/s +Step 6950/50462 | Loss: 2.8663 | LR: 4.81e-04 | 3248.1ms/step | 18.5 samples/s +Step 6960/50462 | Loss: 2.8400 | LR: 4.81e-04 | 3194.6ms/step | 18.8 samples/s +Step 6970/50462 | Loss: 2.8693 | LR: 4.81e-04 | 3197.4ms/step | 18.8 samples/s +Step 6980/50462 | Loss: 2.8660 | LR: 4.81e-04 | 3199.9ms/step | 18.8 samples/s +Step 6990/50462 | Loss: 2.8579 | LR: 4.81e-04 | 3195.7ms/step | 18.8 samples/s +Step 7000/50462 | Loss: 2.8297 | LR: 4.81e-04 | 3198.2ms/step | 18.8 samples/s +Validation at step 7000: Loss = 2.8642, Perplexity = 17.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-7000 +Step 7010/50462 | Loss: 2.8511 | LR: 4.81e-04 | 5782.4ms/step | 10.4 samples/s +Step 7020/50462 | Loss: 2.8618 | LR: 4.81e-04 | 3181.3ms/step | 18.9 samples/s +Step 7030/50462 | Loss: 2.8419 | LR: 4.81e-04 | 3191.9ms/step | 18.8 samples/s +Step 7040/50462 | Loss: 2.8777 | LR: 4.81e-04 | 3195.4ms/step | 18.8 samples/s +Step 7050/50462 | Loss: 2.8575 | LR: 4.80e-04 | 3197.3ms/step | 18.8 samples/s +Step 7060/50462 | Loss: 2.8473 | LR: 4.80e-04 | 3199.9ms/step | 18.8 samples/s +Step 7070/50462 | Loss: 2.8516 | LR: 4.80e-04 | 3187.5ms/step | 18.8 samples/s +Step 7080/50462 | Loss: 2.8383 | LR: 4.80e-04 | 3180.0ms/step | 18.9 samples/s +Step 7090/50462 | Loss: 2.8850 | LR: 4.80e-04 | 3193.8ms/step | 18.8 samples/s +Step 7100/50462 | Loss: 2.8125 | LR: 4.80e-04 | 3198.9ms/step | 18.8 samples/s +Step 7110/50462 | Loss: 2.8885 | LR: 4.80e-04 | 3199.4ms/step | 18.8 samples/s +Step 7120/50462 | Loss: 2.8362 | LR: 4.80e-04 | 3198.6ms/step | 18.8 samples/s +Step 7130/50462 | Loss: 2.8384 | LR: 4.80e-04 | 3191.9ms/step | 18.8 samples/s +Step 7140/50462 | Loss: 2.8833 | LR: 4.80e-04 | 3250.1ms/step | 18.5 samples/s +Step 7150/50462 | Loss: 2.8473 | LR: 4.80e-04 | 3200.1ms/step | 18.7 samples/s +Step 7160/50462 | Loss: 2.8378 | LR: 4.80e-04 | 3200.0ms/step | 18.7 samples/s +Step 7170/50462 | Loss: 2.8452 | LR: 4.80e-04 | 3198.2ms/step | 18.8 samples/s +Step 7180/50462 | Loss: 2.8554 | LR: 4.80e-04 | 3190.5ms/step | 18.8 samples/s +Step 7190/50462 | Loss: 2.8466 | LR: 4.80e-04 | 3192.2ms/step | 18.8 samples/s +Step 7200/50462 | Loss: 2.8631 | LR: 4.80e-04 | 3196.6ms/step | 18.8 samples/s +Step 7210/50462 | Loss: 2.8597 | LR: 4.79e-04 | 3198.4ms/step | 18.8 samples/s +Step 7220/50462 | Loss: 2.8465 | LR: 4.79e-04 | 3192.1ms/step | 18.8 samples/s +Step 7230/50462 | Loss: 2.8354 | LR: 4.79e-04 | 3199.9ms/step | 18.8 samples/s +Step 7240/50462 | Loss: 2.8317 | LR: 4.79e-04 | 3200.1ms/step | 18.7 samples/s +Step 7250/50462 | Loss: 2.8339 | LR: 4.79e-04 | 3197.5ms/step | 18.8 samples/s +Validation at step 7250: Loss = 2.8566, Perplexity = 17.40 +Step 7260/50462 | Loss: 2.8548 | LR: 4.79e-04 | 4919.7ms/step | 12.2 samples/s +Step 7270/50462 | Loss: 2.8094 | LR: 4.79e-04 | 3183.2ms/step | 18.8 samples/s +Step 7280/50462 | Loss: 2.8287 | LR: 4.79e-04 | 3190.0ms/step | 18.8 samples/s +Step 7290/50462 | Loss: 2.8587 | LR: 4.79e-04 | 3243.1ms/step | 18.5 samples/s +Step 7300/50462 | Loss: 2.8222 | LR: 4.79e-04 | 3188.1ms/step | 18.8 samples/s +Step 7310/50462 | Loss: 2.8541 | LR: 4.79e-04 | 3192.1ms/step | 18.8 samples/s +Step 7320/50462 | Loss: 2.8429 | LR: 4.79e-04 | 3196.1ms/step | 18.8 samples/s +Step 7330/50462 | Loss: 2.8075 | LR: 4.79e-04 | 3187.1ms/step | 18.8 samples/s +Step 7340/50462 | Loss: 2.8473 | LR: 4.79e-04 | 3188.4ms/step | 18.8 samples/s +Step 7350/50462 | Loss: 2.8600 | LR: 4.79e-04 | 3189.8ms/step | 18.8 samples/s +Step 7360/50462 | Loss: 2.8301 | LR: 4.79e-04 | 3192.0ms/step | 18.8 samples/s +Step 7370/50462 | Loss: 2.8751 | LR: 4.78e-04 | 3192.5ms/step | 18.8 samples/s +Step 7380/50462 | Loss: 2.8545 | LR: 4.78e-04 | 3190.8ms/step | 18.8 samples/s +Step 7390/50462 | Loss: 2.8465 | LR: 4.78e-04 | 3183.2ms/step | 18.8 samples/s +Step 7400/50462 | Loss: 2.8484 | LR: 4.78e-04 | 3184.1ms/step | 18.8 samples/s +Step 7410/50462 | Loss: 2.8139 | LR: 4.78e-04 | 3192.9ms/step | 18.8 samples/s +Step 7420/50462 | Loss: 2.8084 | LR: 4.78e-04 | 3189.8ms/step | 18.8 samples/s +Step 7430/50462 | Loss: 2.8470 | LR: 4.78e-04 | 3189.1ms/step | 18.8 samples/s +Step 7440/50462 | Loss: 2.8246 | LR: 4.78e-04 | 3192.2ms/step | 18.8 samples/s +Step 7450/50462 | Loss: 2.8542 | LR: 4.78e-04 | 3188.3ms/step | 18.8 samples/s +Step 7460/50462 | Loss: 2.8647 | LR: 4.78e-04 | 3189.5ms/step | 18.8 samples/s +Step 7470/50462 | Loss: 2.8644 | LR: 4.78e-04 | 3244.9ms/step | 18.5 samples/s +Step 7480/50462 | Loss: 2.8479 | LR: 4.78e-04 | 3196.0ms/step | 18.8 samples/s +Step 7490/50462 | Loss: 2.8410 | LR: 4.78e-04 | 3198.7ms/step | 18.8 samples/s +Step 7500/50462 | Loss: 2.8279 | LR: 4.78e-04 | 3194.6ms/step | 18.8 samples/s +Validation at step 7500: Loss = 2.8493, Perplexity = 17.28 +Step 7510/50462 | Loss: 2.8259 | LR: 4.78e-04 | 4869.4ms/step | 12.3 samples/s +Step 7520/50462 | Loss: 2.8399 | LR: 4.77e-04 | 3190.0ms/step | 18.8 samples/s +Step 7530/50462 | Loss: 2.8604 | LR: 4.77e-04 | 3196.8ms/step | 18.8 samples/s +Step 7540/50462 | Loss: 2.8460 | LR: 4.77e-04 | 3199.3ms/step | 18.8 samples/s +Step 7550/50462 | Loss: 2.8824 | LR: 4.77e-04 | 3195.8ms/step | 18.8 samples/s +Step 7560/50462 | Loss: 2.8393 | LR: 4.77e-04 | 3198.8ms/step | 18.8 samples/s +Step 7570/50462 | Loss: 2.8203 | LR: 4.77e-04 | 3200.0ms/step | 18.7 samples/s +Step 7580/50462 | Loss: 2.8629 | LR: 4.77e-04 | 3200.2ms/step | 18.7 samples/s +Step 7590/50462 | Loss: 2.8306 | LR: 4.77e-04 | 3199.8ms/step | 18.8 samples/s +Step 7600/50462 | Loss: 2.8391 | LR: 4.77e-04 | 3250.7ms/step | 18.5 samples/s +Step 7610/50462 | Loss: 2.8397 | LR: 4.77e-04 | 3199.1ms/step | 18.8 samples/s +Step 7620/50462 | Loss: 2.8176 | LR: 4.77e-04 | 3200.0ms/step | 18.7 samples/s +Step 7630/50462 | Loss: 2.8279 | LR: 4.77e-04 | 3251.7ms/step | 18.5 samples/s +Step 7640/50462 | Loss: 2.8483 | LR: 4.77e-04 | 3195.6ms/step | 18.8 samples/s +Step 7650/50462 | Loss: 2.8381 | LR: 4.77e-04 | 3196.6ms/step | 18.8 samples/s +Step 7660/50462 | Loss: 2.8266 | LR: 4.77e-04 | 3199.9ms/step | 18.8 samples/s +Step 7670/50462 | Loss: 2.8462 | LR: 4.76e-04 | 3198.8ms/step | 18.8 samples/s +Step 7680/50462 | Loss: 2.8767 | LR: 4.76e-04 | 3195.0ms/step | 18.8 samples/s +Step 7690/50462 | Loss: 2.8131 | LR: 4.76e-04 | 3200.2ms/step | 18.7 samples/s +Step 7700/50462 | Loss: 2.8249 | LR: 4.76e-04 | 3200.4ms/step | 18.7 samples/s +Step 7710/50462 | Loss: 2.8528 | LR: 4.76e-04 | 3200.0ms/step | 18.8 samples/s +Step 7720/50462 | Loss: 2.8265 | LR: 4.76e-04 | 3200.0ms/step | 18.7 samples/s +Step 7730/50462 | Loss: 2.8439 | LR: 4.76e-04 | 3197.4ms/step | 18.8 samples/s +Step 7740/50462 | Loss: 2.8588 | LR: 4.76e-04 | 3198.5ms/step | 18.8 samples/s +Step 7750/50462 | Loss: 2.8321 | LR: 4.76e-04 | 3199.5ms/step | 18.8 samples/s +Validation at step 7750: Loss = 2.8403, Perplexity = 17.12 +Step 7760/50462 | Loss: 2.8261 | LR: 4.76e-04 | 4876.8ms/step | 12.3 samples/s +Step 7770/50462 | Loss: 2.8382 | LR: 4.76e-04 | 3199.7ms/step | 18.8 samples/s +Step 7780/50462 | Loss: 2.8308 | LR: 4.76e-04 | 3199.3ms/step | 18.8 samples/s +Step 7790/50462 | Loss: 2.8553 | LR: 4.76e-04 | 3198.9ms/step | 18.8 samples/s +Step 7800/50462 | Loss: 2.8359 | LR: 4.76e-04 | 3199.8ms/step | 18.8 samples/s +Step 7810/50462 | Loss: 2.8095 | LR: 4.76e-04 | 3254.0ms/step | 18.4 samples/s +Step 7820/50462 | Loss: 2.8361 | LR: 4.75e-04 | 3185.8ms/step | 18.8 samples/s +Step 7830/50462 | Loss: 2.8470 | LR: 4.75e-04 | 3186.0ms/step | 18.8 samples/s +Step 7840/50462 | Loss: 2.8266 | LR: 4.75e-04 | 3182.2ms/step | 18.9 samples/s +Step 7850/50462 | Loss: 2.8049 | LR: 4.75e-04 | 3188.2ms/step | 18.8 samples/s +Step 7860/50462 | Loss: 2.8833 | LR: 4.75e-04 | 3189.5ms/step | 18.8 samples/s +Step 7870/50462 | Loss: 2.8195 | LR: 4.75e-04 | 3200.4ms/step | 18.7 samples/s +Step 7880/50462 | Loss: 2.8217 | LR: 4.75e-04 | 3197.9ms/step | 18.8 samples/s +Step 7890/50462 | Loss: 2.8231 | LR: 4.75e-04 | 3184.0ms/step | 18.8 samples/s +Step 7900/50462 | Loss: 2.8140 | LR: 4.75e-04 | 3190.0ms/step | 18.8 samples/s +Step 7910/50462 | Loss: 2.8261 | LR: 4.75e-04 | 3194.1ms/step | 18.8 samples/s +Step 7920/50462 | Loss: 2.8250 | LR: 4.75e-04 | 3198.0ms/step | 18.8 samples/s +Step 7930/50462 | Loss: 2.8351 | LR: 4.75e-04 | 3200.1ms/step | 18.7 samples/s +Step 7940/50462 | Loss: 2.8091 | LR: 4.75e-04 | 3252.1ms/step | 18.4 samples/s +Step 7950/50462 | Loss: 2.8207 | LR: 4.75e-04 | 3197.7ms/step | 18.8 samples/s +Step 7960/50462 | Loss: 2.8175 | LR: 4.74e-04 | 3200.2ms/step | 18.7 samples/s +Step 7970/50462 | Loss: 2.8000 | LR: 4.74e-04 | 3251.4ms/step | 18.5 samples/s +Step 7980/50462 | Loss: 2.8288 | LR: 4.74e-04 | 3200.5ms/step | 18.7 samples/s +Step 7990/50462 | Loss: 2.8230 | LR: 4.74e-04 | 3199.2ms/step | 18.8 samples/s +Step 8000/50462 | Loss: 2.8403 | LR: 4.74e-04 | 3200.0ms/step | 18.8 samples/s +Validation at step 8000: Loss = 2.8346, Perplexity = 17.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-8000 +Step 8010/50462 | Loss: 2.8039 | LR: 4.74e-04 | 5919.2ms/step | 10.1 samples/s +Step 8020/50462 | Loss: 2.8483 | LR: 4.74e-04 | 3175.0ms/step | 18.9 samples/s +Step 8030/50462 | Loss: 2.8217 | LR: 4.74e-04 | 3185.9ms/step | 18.8 samples/s +Step 8040/50462 | Loss: 2.8129 | LR: 4.74e-04 | 3193.3ms/step | 18.8 samples/s +Step 8050/50462 | Loss: 2.8176 | LR: 4.74e-04 | 3185.8ms/step | 18.8 samples/s +Step 8060/50462 | Loss: 2.8197 | LR: 4.74e-04 | 3187.1ms/step | 18.8 samples/s +Step 8070/50462 | Loss: 2.8223 | LR: 4.74e-04 | 3189.7ms/step | 18.8 samples/s +Step 8080/50462 | Loss: 2.8184 | LR: 4.74e-04 | 3191.9ms/step | 18.8 samples/s +Step 8090/50462 | Loss: 2.8485 | LR: 4.74e-04 | 3196.2ms/step | 18.8 samples/s +Step 8100/50462 | Loss: 2.8040 | LR: 4.74e-04 | 3196.1ms/step | 18.8 samples/s +Step 8110/50462 | Loss: 2.8479 | LR: 4.73e-04 | 3195.1ms/step | 18.8 samples/s +Step 8120/50462 | Loss: 2.8375 | LR: 4.73e-04 | 3199.6ms/step | 18.8 samples/s +Step 8130/50462 | Loss: 2.8397 | LR: 4.73e-04 | 3195.7ms/step | 18.8 samples/s +Step 8140/50462 | Loss: 2.8219 | LR: 4.73e-04 | 3196.0ms/step | 18.8 samples/s +Step 8150/50462 | Loss: 2.8044 | LR: 4.73e-04 | 3252.2ms/step | 18.4 samples/s +Step 8160/50462 | Loss: 2.8148 | LR: 4.73e-04 | 3199.8ms/step | 18.8 samples/s +Step 8170/50462 | Loss: 2.8212 | LR: 4.73e-04 | 3199.9ms/step | 18.8 samples/s +Step 8180/50462 | Loss: 2.8302 | LR: 4.73e-04 | 3199.9ms/step | 18.8 samples/s +Step 8190/50462 | Loss: 2.8179 | LR: 4.73e-04 | 3199.6ms/step | 18.8 samples/s +Step 8200/50462 | Loss: 2.7976 | LR: 4.73e-04 | 3200.7ms/step | 18.7 samples/s +Step 8210/50462 | Loss: 2.8010 | LR: 4.73e-04 | 3197.6ms/step | 18.8 samples/s +Step 8220/50462 | Loss: 2.8279 | LR: 4.73e-04 | 3198.5ms/step | 18.8 samples/s +Step 8230/50462 | Loss: 2.8288 | LR: 4.73e-04 | 3200.0ms/step | 18.8 samples/s +Step 8240/50462 | Loss: 2.8117 | LR: 4.73e-04 | 3197.6ms/step | 18.8 samples/s +Step 8250/50462 | Loss: 2.7842 | LR: 4.72e-04 | 3198.4ms/step | 18.8 samples/s +Validation at step 8250: Loss = 2.8265, Perplexity = 16.89 +Step 8260/50462 | Loss: 2.8188 | LR: 4.72e-04 | 4872.0ms/step | 12.3 samples/s +Step 8270/50462 | Loss: 2.8285 | LR: 4.72e-04 | 3199.8ms/step | 18.8 samples/s +Step 8280/50462 | Loss: 2.8342 | LR: 4.72e-04 | 3252.3ms/step | 18.4 samples/s +Step 8290/50462 | Loss: 2.8511 | LR: 4.72e-04 | 3198.9ms/step | 18.8 samples/s +Step 8300/50462 | Loss: 2.8137 | LR: 4.72e-04 | 3198.8ms/step | 18.8 samples/s +Step 8310/50462 | Loss: 2.8274 | LR: 4.72e-04 | 3256.0ms/step | 18.4 samples/s +Step 8320/50462 | Loss: 2.8186 | LR: 4.72e-04 | 3200.1ms/step | 18.7 samples/s +Step 8330/50462 | Loss: 2.8092 | LR: 4.72e-04 | 3200.2ms/step | 18.7 samples/s +Step 8340/50462 | Loss: 2.8227 | LR: 4.72e-04 | 3195.9ms/step | 18.8 samples/s +Step 8350/50462 | Loss: 2.8173 | LR: 4.72e-04 | 3199.8ms/step | 18.8 samples/s +Step 8360/50462 | Loss: 2.7960 | LR: 4.72e-04 | 3200.0ms/step | 18.8 samples/s +Step 8370/50462 | Loss: 2.7762 | LR: 4.72e-04 | 3200.1ms/step | 18.7 samples/s +Step 8380/50462 | Loss: 2.8245 | LR: 4.71e-04 | 3200.1ms/step | 18.7 samples/s +Step 8390/50462 | Loss: 2.8065 | LR: 4.71e-04 | 3199.9ms/step | 18.8 samples/s +Step 8400/50462 | Loss: 2.7892 | LR: 4.71e-04 | 3199.9ms/step | 18.8 samples/s +Step 8410/50462 | Loss: 2.8337 | LR: 4.71e-04 | 3199.9ms/step | 18.8 samples/s +Step 8420/50462 | Loss: 2.8087 | LR: 4.71e-04 | 3200.3ms/step | 18.7 samples/s +Step 8430/50462 | Loss: 2.8083 | LR: 4.71e-04 | 3200.0ms/step | 18.7 samples/s +Step 8440/50462 | Loss: 2.8152 | LR: 4.71e-04 | 3197.3ms/step | 18.8 samples/s +Step 8450/50462 | Loss: 2.7978 | LR: 4.71e-04 | 3198.7ms/step | 18.8 samples/s +Step 8460/50462 | Loss: 2.8044 | LR: 4.71e-04 | 3199.8ms/step | 18.8 samples/s +Step 8470/50462 | Loss: 2.8077 | LR: 4.71e-04 | 3197.2ms/step | 18.8 samples/s +Step 8480/50462 | Loss: 2.7974 | LR: 4.71e-04 | 3201.0ms/step | 18.7 samples/s +Step 8490/50462 | Loss: 2.8027 | LR: 4.71e-04 | 3250.0ms/step | 18.5 samples/s +Step 8500/50462 | Loss: 2.8143 | LR: 4.71e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 8500: Loss = 2.8203, Perplexity = 16.78 +Step 8510/50462 | Loss: 2.7819 | LR: 4.71e-04 | 4873.5ms/step | 12.3 samples/s +Step 8520/50462 | Loss: 2.8074 | LR: 4.70e-04 | 3200.5ms/step | 18.7 samples/s +Step 8530/50462 | Loss: 2.8096 | LR: 4.70e-04 | 3198.4ms/step | 18.8 samples/s +Step 8540/50462 | Loss: 2.8214 | LR: 4.70e-04 | 3199.6ms/step | 18.8 samples/s +Step 8550/50462 | Loss: 2.8192 | LR: 4.70e-04 | 3197.2ms/step | 18.8 samples/s +Step 8560/50462 | Loss: 2.8180 | LR: 4.70e-04 | 3201.0ms/step | 18.7 samples/s +Step 8570/50462 | Loss: 2.7992 | LR: 4.70e-04 | 3202.1ms/step | 18.7 samples/s +Step 8580/50462 | Loss: 2.8180 | LR: 4.70e-04 | 3199.5ms/step | 18.8 samples/s +Step 8590/50462 | Loss: 2.8206 | LR: 4.70e-04 | 3200.3ms/step | 18.7 samples/s +Step 8600/50462 | Loss: 2.8129 | LR: 4.70e-04 | 3199.9ms/step | 18.8 samples/s +Step 8610/50462 | Loss: 2.8215 | LR: 4.70e-04 | 3200.0ms/step | 18.7 samples/s +Step 8620/50462 | Loss: 2.7954 | LR: 4.70e-04 | 3249.7ms/step | 18.5 samples/s +Step 8630/50462 | Loss: 2.8220 | LR: 4.70e-04 | 3200.4ms/step | 18.7 samples/s +Step 8640/50462 | Loss: 2.8198 | LR: 4.70e-04 | 3199.9ms/step | 18.8 samples/s +Step 8650/50462 | Loss: 2.8289 | LR: 4.69e-04 | 3249.9ms/step | 18.5 samples/s +Step 8660/50462 | Loss: 2.8292 | LR: 4.69e-04 | 3200.1ms/step | 18.7 samples/s +Step 8670/50462 | Loss: 2.7858 | LR: 4.69e-04 | 3200.1ms/step | 18.7 samples/s +Step 8680/50462 | Loss: 2.8037 | LR: 4.69e-04 | 3200.0ms/step | 18.7 samples/s +Step 8690/50462 | Loss: 2.8200 | LR: 4.69e-04 | 3198.1ms/step | 18.8 samples/s +Step 8700/50462 | Loss: 2.7838 | LR: 4.69e-04 | 3199.9ms/step | 18.8 samples/s +Step 8710/50462 | Loss: 2.8274 | LR: 4.69e-04 | 3199.8ms/step | 18.8 samples/s +Step 8720/50462 | Loss: 2.8115 | LR: 4.69e-04 | 3200.0ms/step | 18.8 samples/s +Step 8730/50462 | Loss: 2.8088 | LR: 4.69e-04 | 3200.1ms/step | 18.7 samples/s +Step 8740/50462 | Loss: 2.8062 | LR: 4.69e-04 | 3200.0ms/step | 18.8 samples/s +Step 8750/50462 | Loss: 2.7965 | LR: 4.69e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 8750: Loss = 2.8143, Perplexity = 16.68 +Step 8760/50462 | Loss: 2.7964 | LR: 4.69e-04 | 4878.3ms/step | 12.3 samples/s +Step 8770/50462 | Loss: 2.8205 | LR: 4.69e-04 | 3199.7ms/step | 18.8 samples/s +Step 8780/50462 | Loss: 2.7998 | LR: 4.68e-04 | 3200.0ms/step | 18.7 samples/s +Step 8790/50462 | Loss: 2.8438 | LR: 4.68e-04 | 3199.6ms/step | 18.8 samples/s +Step 8800/50462 | Loss: 2.7925 | LR: 4.68e-04 | 3200.2ms/step | 18.7 samples/s +Step 8810/50462 | Loss: 2.8129 | LR: 4.68e-04 | 3198.2ms/step | 18.8 samples/s +Step 8820/50462 | Loss: 2.8049 | LR: 4.68e-04 | 3248.1ms/step | 18.5 samples/s +Step 8830/50462 | Loss: 2.7904 | LR: 4.68e-04 | 3194.2ms/step | 18.8 samples/s +Step 8840/50462 | Loss: 2.7930 | LR: 4.68e-04 | 3193.7ms/step | 18.8 samples/s +Step 8850/50462 | Loss: 2.7937 | LR: 4.68e-04 | 3183.1ms/step | 18.8 samples/s +Step 8860/50462 | Loss: 2.8149 | LR: 4.68e-04 | 3188.9ms/step | 18.8 samples/s +Step 8870/50462 | Loss: 2.7728 | LR: 4.68e-04 | 3187.6ms/step | 18.8 samples/s +Step 8880/50462 | Loss: 2.7855 | LR: 4.68e-04 | 3192.3ms/step | 18.8 samples/s +Step 8890/50462 | Loss: 2.8070 | LR: 4.68e-04 | 3189.3ms/step | 18.8 samples/s +Step 8900/50462 | Loss: 2.8023 | LR: 4.68e-04 | 3187.4ms/step | 18.8 samples/s +Step 8910/50462 | Loss: 2.8206 | LR: 4.67e-04 | 3183.5ms/step | 18.8 samples/s +Step 8920/50462 | Loss: 2.7895 | LR: 4.67e-04 | 3184.4ms/step | 18.8 samples/s +Step 8930/50462 | Loss: 2.7815 | LR: 4.67e-04 | 3187.6ms/step | 18.8 samples/s +Step 8940/50462 | Loss: 2.7988 | LR: 4.67e-04 | 3191.4ms/step | 18.8 samples/s +Step 8950/50462 | Loss: 2.8345 | LR: 4.67e-04 | 3194.5ms/step | 18.8 samples/s +Step 8960/50462 | Loss: 2.7923 | LR: 4.67e-04 | 3247.1ms/step | 18.5 samples/s +Step 8970/50462 | Loss: 2.8049 | LR: 4.67e-04 | 3197.0ms/step | 18.8 samples/s +Step 8980/50462 | Loss: 2.8096 | LR: 4.67e-04 | 3196.2ms/step | 18.8 samples/s +Step 8990/50462 | Loss: 2.7908 | LR: 4.67e-04 | 3250.0ms/step | 18.5 samples/s +Step 9000/50462 | Loss: 2.8187 | LR: 4.67e-04 | 3195.8ms/step | 18.8 samples/s +Validation at step 9000: Loss = 2.8063, Perplexity = 16.55 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-9000 +Step 9010/50462 | Loss: 2.8013 | LR: 4.67e-04 | 5761.9ms/step | 10.4 samples/s +Step 9020/50462 | Loss: 2.7730 | LR: 4.67e-04 | 3177.6ms/step | 18.9 samples/s +Step 9030/50462 | Loss: 2.7972 | LR: 4.67e-04 | 3186.7ms/step | 18.8 samples/s +Step 9040/50462 | Loss: 2.7956 | LR: 4.66e-04 | 3197.8ms/step | 18.8 samples/s +Step 9050/50462 | Loss: 2.7928 | LR: 4.66e-04 | 3192.2ms/step | 18.8 samples/s +Step 9060/50462 | Loss: 2.7839 | LR: 4.66e-04 | 3200.1ms/step | 18.7 samples/s +Step 9070/50462 | Loss: 2.8015 | LR: 4.66e-04 | 3200.0ms/step | 18.7 samples/s +Step 9080/50462 | Loss: 2.8136 | LR: 4.66e-04 | 3200.0ms/step | 18.8 samples/s +Step 9090/50462 | Loss: 2.8010 | LR: 4.66e-04 | 3199.8ms/step | 18.8 samples/s +Step 9100/50462 | Loss: 2.7770 | LR: 4.66e-04 | 3200.0ms/step | 18.7 samples/s +Step 9110/50462 | Loss: 2.8238 | LR: 4.66e-04 | 3200.0ms/step | 18.8 samples/s +Step 9120/50462 | Loss: 2.7888 | LR: 4.66e-04 | 3200.0ms/step | 18.7 samples/s +Step 9130/50462 | Loss: 2.7674 | LR: 4.66e-04 | 3200.2ms/step | 18.7 samples/s +Step 9140/50462 | Loss: 2.7805 | LR: 4.66e-04 | 3199.7ms/step | 18.8 samples/s +Step 9150/50462 | Loss: 2.7939 | LR: 4.66e-04 | 3199.7ms/step | 18.8 samples/s +Step 9160/50462 | Loss: 2.7917 | LR: 4.66e-04 | 3254.5ms/step | 18.4 samples/s +Step 9170/50462 | Loss: 2.7790 | LR: 4.65e-04 | 3198.0ms/step | 18.8 samples/s +Step 9180/50462 | Loss: 2.7592 | LR: 4.65e-04 | 3197.1ms/step | 18.8 samples/s +Step 9190/50462 | Loss: 2.8059 | LR: 4.65e-04 | 3199.0ms/step | 18.8 samples/s +Step 9200/50462 | Loss: 2.7892 | LR: 4.65e-04 | 3199.9ms/step | 18.8 samples/s +Step 9210/50462 | Loss: 2.7584 | LR: 4.65e-04 | 3197.9ms/step | 18.8 samples/s +Step 9220/50462 | Loss: 2.7776 | LR: 4.65e-04 | 3199.7ms/step | 18.8 samples/s +Step 9230/50462 | Loss: 2.7927 | LR: 4.65e-04 | 3190.8ms/step | 18.8 samples/s +Step 9240/50462 | Loss: 2.8130 | LR: 4.65e-04 | 3189.2ms/step | 18.8 samples/s +Step 9250/50462 | Loss: 2.8145 | LR: 4.65e-04 | 3195.6ms/step | 18.8 samples/s +Validation at step 9250: Loss = 2.8011, Perplexity = 16.46 +Step 9260/50462 | Loss: 2.7635 | LR: 4.65e-04 | 4874.7ms/step | 12.3 samples/s +Step 9270/50462 | Loss: 2.8017 | LR: 4.65e-04 | 3201.8ms/step | 18.7 samples/s +Step 9280/50462 | Loss: 2.7578 | LR: 4.65e-04 | 3200.1ms/step | 18.7 samples/s +Step 9290/50462 | Loss: 2.7989 | LR: 4.64e-04 | 3199.9ms/step | 18.8 samples/s +Step 9300/50462 | Loss: 2.8158 | LR: 4.64e-04 | 3255.5ms/step | 18.4 samples/s +Step 9310/50462 | Loss: 2.7885 | LR: 4.64e-04 | 3200.5ms/step | 18.7 samples/s +Step 9320/50462 | Loss: 2.7787 | LR: 4.64e-04 | 3200.0ms/step | 18.8 samples/s +Step 9330/50462 | Loss: 2.7756 | LR: 4.64e-04 | 3250.1ms/step | 18.5 samples/s +Step 9340/50462 | Loss: 2.7870 | LR: 4.64e-04 | 3199.8ms/step | 18.8 samples/s +Step 9350/50462 | Loss: 2.7787 | LR: 4.64e-04 | 3200.3ms/step | 18.7 samples/s +Step 9360/50462 | Loss: 2.7812 | LR: 4.64e-04 | 3200.4ms/step | 18.7 samples/s +Step 9370/50462 | Loss: 2.7856 | LR: 4.64e-04 | 3201.4ms/step | 18.7 samples/s +Step 9380/50462 | Loss: 2.8124 | LR: 4.64e-04 | 3185.9ms/step | 18.8 samples/s +Step 9390/50462 | Loss: 2.7961 | LR: 4.64e-04 | 3196.2ms/step | 18.8 samples/s +Step 9400/50462 | Loss: 2.7589 | LR: 4.64e-04 | 3200.0ms/step | 18.7 samples/s +Step 9410/50462 | Loss: 2.7687 | LR: 4.63e-04 | 3199.5ms/step | 18.8 samples/s +Step 9420/50462 | Loss: 2.7872 | LR: 4.63e-04 | 3200.5ms/step | 18.7 samples/s +Step 9430/50462 | Loss: 2.7677 | LR: 4.63e-04 | 3199.9ms/step | 18.8 samples/s +Step 9440/50462 | Loss: 2.7914 | LR: 4.63e-04 | 3199.9ms/step | 18.8 samples/s +Step 9450/50462 | Loss: 2.7839 | LR: 4.63e-04 | 3200.1ms/step | 18.7 samples/s +Step 9460/50462 | Loss: 2.7709 | LR: 4.63e-04 | 3199.9ms/step | 18.8 samples/s +Step 9470/50462 | Loss: 2.7882 | LR: 4.63e-04 | 3198.1ms/step | 18.8 samples/s +Step 9480/50462 | Loss: 2.7861 | LR: 4.63e-04 | 3197.8ms/step | 18.8 samples/s +Step 9490/50462 | Loss: 2.7887 | LR: 4.63e-04 | 3255.6ms/step | 18.4 samples/s +Step 9500/50462 | Loss: 2.8065 | LR: 4.63e-04 | 3200.4ms/step | 18.7 samples/s +Validation at step 9500: Loss = 2.7949, Perplexity = 16.36 +Step 9510/50462 | Loss: 2.7687 | LR: 4.63e-04 | 4875.9ms/step | 12.3 samples/s +Step 9520/50462 | Loss: 2.7865 | LR: 4.63e-04 | 3202.3ms/step | 18.7 samples/s +Step 9530/50462 | Loss: 2.8073 | LR: 4.62e-04 | 3199.9ms/step | 18.8 samples/s +Step 9540/50462 | Loss: 2.7611 | LR: 4.62e-04 | 3199.9ms/step | 18.8 samples/s +Step 9550/50462 | Loss: 2.7863 | LR: 4.62e-04 | 3200.2ms/step | 18.7 samples/s +Step 9560/50462 | Loss: 2.7815 | LR: 4.62e-04 | 3199.7ms/step | 18.8 samples/s +Step 9570/50462 | Loss: 2.7892 | LR: 4.62e-04 | 3200.2ms/step | 18.7 samples/s +Step 9580/50462 | Loss: 2.7769 | LR: 4.62e-04 | 3200.0ms/step | 18.7 samples/s +Step 9590/50462 | Loss: 2.8194 | LR: 4.62e-04 | 3199.9ms/step | 18.8 samples/s +Step 9600/50462 | Loss: 2.7724 | LR: 4.62e-04 | 3199.9ms/step | 18.8 samples/s +Step 9610/50462 | Loss: 2.7861 | LR: 4.62e-04 | 3200.1ms/step | 18.7 samples/s +Step 9620/50462 | Loss: 2.7678 | LR: 4.62e-04 | 3200.0ms/step | 18.8 samples/s +Step 9630/50462 | Loss: 2.7603 | LR: 4.62e-04 | 3199.9ms/step | 18.8 samples/s +Step 9640/50462 | Loss: 2.7477 | LR: 4.62e-04 | 3254.1ms/step | 18.4 samples/s +Step 9650/50462 | Loss: 2.7681 | LR: 4.61e-04 | 3199.8ms/step | 18.8 samples/s +Step 9660/50462 | Loss: 2.7848 | LR: 4.61e-04 | 3200.1ms/step | 18.7 samples/s +Step 9670/50462 | Loss: 2.7809 | LR: 4.61e-04 | 3254.0ms/step | 18.4 samples/s +Step 9680/50462 | Loss: 2.8403 | LR: 4.61e-04 | 3199.9ms/step | 18.8 samples/s +Step 9690/50462 | Loss: 2.7736 | LR: 4.61e-04 | 3199.8ms/step | 18.8 samples/s +Step 9700/50462 | Loss: 2.7907 | LR: 4.61e-04 | 3200.4ms/step | 18.7 samples/s +Step 9710/50462 | Loss: 2.7760 | LR: 4.61e-04 | 3200.0ms/step | 18.7 samples/s +Step 9720/50462 | Loss: 2.7595 | LR: 4.61e-04 | 3199.8ms/step | 18.8 samples/s +Step 9730/50462 | Loss: 2.7998 | LR: 4.61e-04 | 3200.0ms/step | 18.8 samples/s +Step 9740/50462 | Loss: 2.7807 | LR: 4.61e-04 | 3200.0ms/step | 18.7 samples/s +Step 9750/50462 | Loss: 2.7719 | LR: 4.61e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 9750: Loss = 2.7898, Perplexity = 16.28 +Step 9760/50462 | Loss: 2.7748 | LR: 4.61e-04 | 4880.1ms/step | 12.3 samples/s +Step 9770/50462 | Loss: 2.7996 | LR: 4.60e-04 | 3200.0ms/step | 18.7 samples/s +Step 9780/50462 | Loss: 2.7874 | LR: 4.60e-04 | 3199.9ms/step | 18.8 samples/s +Step 9790/50462 | Loss: 2.7704 | LR: 4.60e-04 | 3200.1ms/step | 18.7 samples/s +Step 9800/50462 | Loss: 2.7762 | LR: 4.60e-04 | 3199.9ms/step | 18.8 samples/s +Step 9810/50462 | Loss: 2.7767 | LR: 4.60e-04 | 3200.1ms/step | 18.7 samples/s +Step 9820/50462 | Loss: 2.7780 | LR: 4.60e-04 | 3200.2ms/step | 18.7 samples/s +Step 9830/50462 | Loss: 2.7453 | LR: 4.60e-04 | 3253.6ms/step | 18.4 samples/s +Step 9840/50462 | Loss: 2.7765 | LR: 4.60e-04 | 3194.7ms/step | 18.8 samples/s +Step 9850/50462 | Loss: 2.7466 | LR: 4.60e-04 | 3199.3ms/step | 18.8 samples/s +Step 9860/50462 | Loss: 2.7744 | LR: 4.60e-04 | 3200.2ms/step | 18.7 samples/s +Step 9870/50462 | Loss: 2.7494 | LR: 4.60e-04 | 3199.9ms/step | 18.8 samples/s +Step 9880/50462 | Loss: 2.7637 | LR: 4.60e-04 | 3200.3ms/step | 18.7 samples/s +Step 9890/50462 | Loss: 2.7918 | LR: 4.59e-04 | 3199.7ms/step | 18.8 samples/s +Step 9900/50462 | Loss: 2.7785 | LR: 4.59e-04 | 3200.1ms/step | 18.7 samples/s +Step 9910/50462 | Loss: 2.7990 | LR: 4.59e-04 | 3200.2ms/step | 18.7 samples/s +Step 9920/50462 | Loss: 2.7721 | LR: 4.59e-04 | 3200.0ms/step | 18.7 samples/s +Step 9930/50462 | Loss: 2.7639 | LR: 4.59e-04 | 3200.0ms/step | 18.8 samples/s +Step 9940/50462 | Loss: 2.7626 | LR: 4.59e-04 | 3202.0ms/step | 18.7 samples/s +Step 9950/50462 | Loss: 2.7723 | LR: 4.59e-04 | 3200.0ms/step | 18.7 samples/s +Step 9960/50462 | Loss: 2.7387 | LR: 4.59e-04 | 3200.0ms/step | 18.8 samples/s +Step 9970/50462 | Loss: 2.7772 | LR: 4.59e-04 | 3199.9ms/step | 18.8 samples/s +Step 9980/50462 | Loss: 2.7946 | LR: 4.59e-04 | 3254.0ms/step | 18.4 samples/s +Step 9990/50462 | Loss: 2.7610 | LR: 4.59e-04 | 3200.0ms/step | 18.8 samples/s +Step 10000/50462 | Loss: 2.7686 | LR: 4.58e-04 | 3253.2ms/step | 18.4 samples/s +Validation at step 10000: Loss = 2.7851, Perplexity = 16.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-10000 +Step 10010/50462 | Loss: 2.7624 | LR: 4.58e-04 | 5793.8ms/step | 10.4 samples/s +Step 10020/50462 | Loss: 2.8027 | LR: 4.58e-04 | 3186.7ms/step | 18.8 samples/s +Step 10030/50462 | Loss: 2.8021 | LR: 4.58e-04 | 3196.5ms/step | 18.8 samples/s +Step 10040/50462 | Loss: 2.7733 | LR: 4.58e-04 | 3197.9ms/step | 18.8 samples/s +Step 10050/50462 | Loss: 2.7874 | LR: 4.58e-04 | 3198.1ms/step | 18.8 samples/s +Step 10060/50462 | Loss: 2.7716 | LR: 4.58e-04 | 3199.3ms/step | 18.8 samples/s +Step 10070/50462 | Loss: 2.7920 | LR: 4.58e-04 | 3200.7ms/step | 18.7 samples/s +Step 10080/50462 | Loss: 2.7660 | LR: 4.58e-04 | 3199.9ms/step | 18.8 samples/s +Step 10090/50462 | Loss: 2.7996 | LR: 4.58e-04 | 3200.2ms/step | 18.7 samples/s +Step 10100/50462 | Loss: 2.7841 | LR: 4.58e-04 | 3199.8ms/step | 18.8 samples/s +Step 10110/50462 | Loss: 2.7812 | LR: 4.58e-04 | 3200.0ms/step | 18.7 samples/s +Step 10120/50462 | Loss: 2.7684 | LR: 4.57e-04 | 3200.2ms/step | 18.7 samples/s +Step 10130/50462 | Loss: 2.7675 | LR: 4.57e-04 | 3200.0ms/step | 18.7 samples/s +Step 10140/50462 | Loss: 2.7920 | LR: 4.57e-04 | 3200.0ms/step | 18.8 samples/s +Step 10150/50462 | Loss: 2.7900 | LR: 4.57e-04 | 3199.9ms/step | 18.8 samples/s +Step 10160/50462 | Loss: 2.7652 | LR: 4.57e-04 | 3254.6ms/step | 18.4 samples/s +Step 10170/50462 | Loss: 2.7690 | LR: 4.57e-04 | 3199.3ms/step | 18.8 samples/s +Step 10180/50462 | Loss: 2.7547 | LR: 4.57e-04 | 3198.0ms/step | 18.8 samples/s +Step 10190/50462 | Loss: 2.7934 | LR: 4.57e-04 | 3200.0ms/step | 18.7 samples/s +Step 10200/50462 | Loss: 2.7420 | LR: 4.57e-04 | 3200.0ms/step | 18.7 samples/s +Step 10210/50462 | Loss: 2.7727 | LR: 4.57e-04 | 3200.0ms/step | 18.8 samples/s +Step 10220/50462 | Loss: 2.7625 | LR: 4.57e-04 | 3200.2ms/step | 18.7 samples/s +Step 10230/50462 | Loss: 2.7603 | LR: 4.56e-04 | 3199.8ms/step | 18.8 samples/s +Step 10240/50462 | Loss: 2.7939 | LR: 4.56e-04 | 3200.3ms/step | 18.7 samples/s +Step 10250/50462 | Loss: 2.7560 | LR: 4.56e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 10250: Loss = 2.7786, Perplexity = 16.10 +Step 10260/50462 | Loss: 2.7572 | LR: 4.56e-04 | 4880.1ms/step | 12.3 samples/s +Step 10270/50462 | Loss: 2.7528 | LR: 4.56e-04 | 3199.8ms/step | 18.8 samples/s +Step 10280/50462 | Loss: 2.7928 | LR: 4.56e-04 | 3200.1ms/step | 18.7 samples/s +Step 10290/50462 | Loss: 2.7946 | LR: 4.56e-04 | 3200.0ms/step | 18.8 samples/s +Step 10300/50462 | Loss: 2.7613 | LR: 4.56e-04 | 3200.0ms/step | 18.8 samples/s +Step 10310/50462 | Loss: 2.7423 | LR: 4.56e-04 | 3255.1ms/step | 18.4 samples/s +Step 10320/50462 | Loss: 2.7480 | LR: 4.56e-04 | 3190.3ms/step | 18.8 samples/s +Step 10330/50462 | Loss: 2.7727 | LR: 4.56e-04 | 3192.4ms/step | 18.8 samples/s +Step 10340/50462 | Loss: 2.7614 | LR: 4.55e-04 | 3253.5ms/step | 18.4 samples/s +Step 10350/50462 | Loss: 2.7635 | LR: 4.55e-04 | 3200.6ms/step | 18.7 samples/s +Step 10360/50462 | Loss: 2.7938 | LR: 4.55e-04 | 3199.5ms/step | 18.8 samples/s +Step 10370/50462 | Loss: 2.7661 | LR: 4.55e-04 | 3200.5ms/step | 18.7 samples/s +Step 10380/50462 | Loss: 2.7632 | LR: 4.55e-04 | 3199.8ms/step | 18.8 samples/s +Step 10390/50462 | Loss: 2.7720 | LR: 4.55e-04 | 3200.0ms/step | 18.7 samples/s +Step 10400/50462 | Loss: 2.7593 | LR: 4.55e-04 | 3200.1ms/step | 18.7 samples/s +Step 10410/50462 | Loss: 2.7826 | LR: 4.55e-04 | 3200.0ms/step | 18.8 samples/s +Step 10420/50462 | Loss: 2.7499 | LR: 4.55e-04 | 3200.1ms/step | 18.7 samples/s +Step 10430/50462 | Loss: 2.7595 | LR: 4.55e-04 | 3199.9ms/step | 18.8 samples/s +Step 10440/50462 | Loss: 2.7589 | LR: 4.55e-04 | 3200.1ms/step | 18.7 samples/s +Step 10450/50462 | Loss: 2.7781 | LR: 4.54e-04 | 3200.1ms/step | 18.7 samples/s +Step 10460/50462 | Loss: 2.7610 | LR: 4.54e-04 | 3199.9ms/step | 18.8 samples/s +Step 10470/50462 | Loss: 2.7889 | LR: 4.54e-04 | 3199.9ms/step | 18.8 samples/s +Step 10480/50462 | Loss: 2.7518 | LR: 4.54e-04 | 3200.2ms/step | 18.7 samples/s +Step 10490/50462 | Loss: 2.7797 | LR: 4.54e-04 | 3198.1ms/step | 18.8 samples/s +Step 10500/50462 | Loss: 2.7957 | LR: 4.54e-04 | 3254.0ms/step | 18.4 samples/s +Validation at step 10500: Loss = 2.7739, Perplexity = 16.02 +Step 10510/50462 | Loss: 2.7536 | LR: 4.54e-04 | 4875.9ms/step | 12.3 samples/s +Step 10520/50462 | Loss: 2.7544 | LR: 4.54e-04 | 3199.9ms/step | 18.8 samples/s +Step 10530/50462 | Loss: 2.7649 | LR: 4.54e-04 | 3198.1ms/step | 18.8 samples/s +Step 10540/50462 | Loss: 2.7426 | LR: 4.54e-04 | 3200.1ms/step | 18.7 samples/s +Step 10550/50462 | Loss: 2.7526 | LR: 4.54e-04 | 3199.7ms/step | 18.8 samples/s +Step 10560/50462 | Loss: 2.7533 | LR: 4.53e-04 | 3200.5ms/step | 18.7 samples/s +Step 10570/50462 | Loss: 2.7703 | LR: 4.53e-04 | 3199.8ms/step | 18.8 samples/s +Step 10580/50462 | Loss: 2.7874 | LR: 4.53e-04 | 3199.9ms/step | 18.8 samples/s +Step 10590/50462 | Loss: 2.7517 | LR: 4.53e-04 | 3200.2ms/step | 18.7 samples/s +Step 10600/50462 | Loss: 2.7703 | LR: 4.53e-04 | 3199.8ms/step | 18.8 samples/s +Step 10610/50462 | Loss: 2.7885 | LR: 4.53e-04 | 3200.1ms/step | 18.7 samples/s +Step 10620/50462 | Loss: 2.7531 | LR: 4.53e-04 | 3200.2ms/step | 18.7 samples/s +Step 10630/50462 | Loss: 2.7583 | LR: 4.53e-04 | 3199.9ms/step | 18.8 samples/s +Step 10640/50462 | Loss: 2.7676 | LR: 4.53e-04 | 3199.8ms/step | 18.8 samples/s +Step 10650/50462 | Loss: 2.7673 | LR: 4.53e-04 | 3253.5ms/step | 18.4 samples/s +Step 10660/50462 | Loss: 2.7624 | LR: 4.53e-04 | 3200.6ms/step | 18.7 samples/s +Step 10670/50462 | Loss: 2.7566 | LR: 4.52e-04 | 3199.9ms/step | 18.8 samples/s +Step 10680/50462 | Loss: 2.7584 | LR: 4.52e-04 | 3251.8ms/step | 18.5 samples/s +Step 10690/50462 | Loss: 2.7513 | LR: 4.52e-04 | 3200.5ms/step | 18.7 samples/s +Step 10700/50462 | Loss: 2.7527 | LR: 4.52e-04 | 3200.0ms/step | 18.7 samples/s +Step 10710/50462 | Loss: 2.7460 | LR: 4.52e-04 | 3199.9ms/step | 18.8 samples/s +Step 10720/50462 | Loss: 2.7576 | LR: 4.52e-04 | 3200.1ms/step | 18.7 samples/s +Step 10730/50462 | Loss: 2.7913 | LR: 4.52e-04 | 3199.9ms/step | 18.8 samples/s +Step 10740/50462 | Loss: 2.7759 | LR: 4.52e-04 | 3199.6ms/step | 18.8 samples/s +Step 10750/50462 | Loss: 2.7491 | LR: 4.52e-04 | 3200.4ms/step | 18.7 samples/s +Validation at step 10750: Loss = 2.7698, Perplexity = 15.96 +Step 10760/50462 | Loss: 2.7272 | LR: 4.52e-04 | 4870.0ms/step | 12.3 samples/s +Step 10770/50462 | Loss: 2.7859 | LR: 4.52e-04 | 3200.0ms/step | 18.7 samples/s +Step 10780/50462 | Loss: 2.7700 | LR: 4.51e-04 | 3200.1ms/step | 18.7 samples/s +Step 10790/50462 | Loss: 2.7716 | LR: 4.51e-04 | 3200.0ms/step | 18.7 samples/s +Step 10800/50462 | Loss: 2.7331 | LR: 4.51e-04 | 3199.7ms/step | 18.8 samples/s +Step 10810/50462 | Loss: 2.7476 | LR: 4.51e-04 | 3200.2ms/step | 18.7 samples/s +Step 10820/50462 | Loss: 2.7741 | LR: 4.51e-04 | 3199.9ms/step | 18.8 samples/s +Step 10830/50462 | Loss: 2.7736 | LR: 4.51e-04 | 3200.1ms/step | 18.7 samples/s +Step 10840/50462 | Loss: 2.7413 | LR: 4.51e-04 | 3253.8ms/step | 18.4 samples/s +Step 10850/50462 | Loss: 2.7309 | LR: 4.51e-04 | 3200.0ms/step | 18.7 samples/s +Step 10860/50462 | Loss: 2.7756 | LR: 4.51e-04 | 3200.1ms/step | 18.7 samples/s +Step 10870/50462 | Loss: 2.7646 | LR: 4.51e-04 | 3200.1ms/step | 18.7 samples/s +Step 10880/50462 | Loss: 2.7759 | LR: 4.50e-04 | 3197.9ms/step | 18.8 samples/s +Step 10890/50462 | Loss: 2.7573 | LR: 4.50e-04 | 3200.0ms/step | 18.8 samples/s +Step 10900/50462 | Loss: 2.7537 | LR: 4.50e-04 | 3199.9ms/step | 18.8 samples/s +Step 10910/50462 | Loss: 2.7580 | LR: 4.50e-04 | 3200.2ms/step | 18.7 samples/s +Step 10920/50462 | Loss: 2.7766 | LR: 4.50e-04 | 3199.7ms/step | 18.8 samples/s +Step 10930/50462 | Loss: 2.7744 | LR: 4.50e-04 | 3200.3ms/step | 18.7 samples/s +Step 10940/50462 | Loss: 2.7183 | LR: 4.50e-04 | 3199.9ms/step | 18.8 samples/s +Step 10950/50462 | Loss: 2.7886 | LR: 4.50e-04 | 3200.2ms/step | 18.7 samples/s +Step 10960/50462 | Loss: 2.7571 | LR: 4.50e-04 | 3200.1ms/step | 18.7 samples/s +Step 10970/50462 | Loss: 2.7541 | LR: 4.50e-04 | 3199.8ms/step | 18.8 samples/s +Step 10980/50462 | Loss: 2.7584 | LR: 4.50e-04 | 3200.0ms/step | 18.8 samples/s +Step 10990/50462 | Loss: 2.7517 | LR: 4.49e-04 | 3200.1ms/step | 18.7 samples/s +Step 11000/50462 | Loss: 2.7662 | LR: 4.49e-04 | 3253.9ms/step | 18.4 samples/s +Validation at step 11000: Loss = 2.7638, Perplexity = 15.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-11000 +Step 11010/50462 | Loss: 2.7357 | LR: 4.49e-04 | 5806.9ms/step | 10.3 samples/s +Step 11020/50462 | Loss: 2.7434 | LR: 4.49e-04 | 3237.7ms/step | 18.5 samples/s +Step 11030/50462 | Loss: 2.7347 | LR: 4.49e-04 | 3180.8ms/step | 18.9 samples/s +Step 11040/50462 | Loss: 2.7642 | LR: 4.49e-04 | 3189.7ms/step | 18.8 samples/s +Step 11050/50462 | Loss: 2.7871 | LR: 4.49e-04 | 3194.2ms/step | 18.8 samples/s +Step 11060/50462 | Loss: 2.7624 | LR: 4.49e-04 | 3200.3ms/step | 18.7 samples/s +Step 11070/50462 | Loss: 2.7643 | LR: 4.49e-04 | 3200.3ms/step | 18.7 samples/s +Step 11080/50462 | Loss: 2.7319 | LR: 4.49e-04 | 3198.8ms/step | 18.8 samples/s +Step 11090/50462 | Loss: 2.7519 | LR: 4.48e-04 | 3189.2ms/step | 18.8 samples/s +Step 11100/50462 | Loss: 2.7499 | LR: 4.48e-04 | 3199.4ms/step | 18.8 samples/s +Step 11110/50462 | Loss: 2.7588 | LR: 4.48e-04 | 3200.7ms/step | 18.7 samples/s +Step 11120/50462 | Loss: 2.7330 | LR: 4.48e-04 | 3197.8ms/step | 18.8 samples/s +Step 11130/50462 | Loss: 2.7805 | LR: 4.48e-04 | 3200.1ms/step | 18.7 samples/s +Step 11140/50462 | Loss: 2.7377 | LR: 4.48e-04 | 3199.9ms/step | 18.8 samples/s +Step 11150/50462 | Loss: 2.7506 | LR: 4.48e-04 | 3200.3ms/step | 18.7 samples/s +Step 11160/50462 | Loss: 2.7381 | LR: 4.48e-04 | 3199.9ms/step | 18.8 samples/s +Step 11170/50462 | Loss: 2.7730 | LR: 4.48e-04 | 3200.0ms/step | 18.8 samples/s +Step 11180/50462 | Loss: 2.7601 | LR: 4.48e-04 | 3251.8ms/step | 18.5 samples/s +Step 11190/50462 | Loss: 2.7365 | LR: 4.48e-04 | 3200.1ms/step | 18.7 samples/s +Step 11200/50462 | Loss: 2.7484 | LR: 4.47e-04 | 3199.9ms/step | 18.8 samples/s +Step 11210/50462 | Loss: 2.7574 | LR: 4.47e-04 | 3197.9ms/step | 18.8 samples/s +Step 11220/50462 | Loss: 2.7514 | LR: 4.47e-04 | 3200.2ms/step | 18.7 samples/s +Step 11230/50462 | Loss: 2.7581 | LR: 4.47e-04 | 3199.9ms/step | 18.8 samples/s +Step 11240/50462 | Loss: 2.7818 | LR: 4.47e-04 | 3200.0ms/step | 18.8 samples/s +Step 11250/50462 | Loss: 2.7432 | LR: 4.47e-04 | 3200.0ms/step | 18.8 samples/s +Validation at step 11250: Loss = 2.7593, Perplexity = 15.79 +Step 11260/50462 | Loss: 2.7436 | LR: 4.47e-04 | 4877.8ms/step | 12.3 samples/s +Step 11270/50462 | Loss: 2.7668 | LR: 4.47e-04 | 3200.3ms/step | 18.7 samples/s +Step 11280/50462 | Loss: 2.7311 | LR: 4.47e-04 | 3199.9ms/step | 18.8 samples/s +Step 11290/50462 | Loss: 2.7208 | LR: 4.47e-04 | 3200.2ms/step | 18.7 samples/s +Step 11300/50462 | Loss: 2.7537 | LR: 4.46e-04 | 3200.0ms/step | 18.7 samples/s +Step 11310/50462 | Loss: 2.7633 | LR: 4.46e-04 | 3199.4ms/step | 18.8 samples/s +Step 11320/50462 | Loss: 2.7462 | LR: 4.46e-04 | 3200.3ms/step | 18.7 samples/s +Step 11330/50462 | Loss: 2.7492 | LR: 4.46e-04 | 3200.2ms/step | 18.7 samples/s +Step 11340/50462 | Loss: 2.7251 | LR: 4.46e-04 | 3250.0ms/step | 18.5 samples/s +Step 11350/50462 | Loss: 2.7630 | LR: 4.46e-04 | 3193.3ms/step | 18.8 samples/s +Step 11360/50462 | Loss: 2.7451 | LR: 4.46e-04 | 3252.1ms/step | 18.4 samples/s +Step 11370/50462 | Loss: 2.7503 | LR: 4.46e-04 | 3200.2ms/step | 18.7 samples/s +Step 11380/50462 | Loss: 2.7398 | LR: 4.46e-04 | 3200.4ms/step | 18.7 samples/s +Step 11390/50462 | Loss: 2.7270 | LR: 4.46e-04 | 3200.0ms/step | 18.8 samples/s +Step 11400/50462 | Loss: 2.7617 | LR: 4.45e-04 | 3200.1ms/step | 18.7 samples/s +Step 11410/50462 | Loss: 2.7165 | LR: 4.45e-04 | 3199.8ms/step | 18.8 samples/s +Step 11420/50462 | Loss: 2.7297 | LR: 4.45e-04 | 3198.2ms/step | 18.8 samples/s +Step 11430/50462 | Loss: 2.7452 | LR: 4.45e-04 | 3199.6ms/step | 18.8 samples/s +Step 11440/50462 | Loss: 2.7287 | LR: 4.45e-04 | 3198.2ms/step | 18.8 samples/s +Step 11450/50462 | Loss: 2.7546 | LR: 4.45e-04 | 3200.4ms/step | 18.7 samples/s +Step 11460/50462 | Loss: 2.7564 | LR: 4.45e-04 | 3201.6ms/step | 18.7 samples/s +Step 11470/50462 | Loss: 2.7671 | LR: 4.45e-04 | 3199.7ms/step | 18.8 samples/s +Step 11480/50462 | Loss: 2.7401 | LR: 4.45e-04 | 3200.2ms/step | 18.7 samples/s +Step 11490/50462 | Loss: 2.7579 | LR: 4.45e-04 | 3200.1ms/step | 18.7 samples/s +Step 11500/50462 | Loss: 2.7485 | LR: 4.44e-04 | 3200.0ms/step | 18.7 samples/s +Validation at step 11500: Loss = 2.7554, Perplexity = 15.73 +Step 11510/50462 | Loss: 2.7145 | LR: 4.44e-04 | 4876.1ms/step | 12.3 samples/s +Step 11520/50462 | Loss: 2.7130 | LR: 4.44e-04 | 3255.8ms/step | 18.4 samples/s +Step 11530/50462 | Loss: 2.7494 | LR: 4.44e-04 | 3200.1ms/step | 18.7 samples/s +Step 11540/50462 | Loss: 2.7615 | LR: 4.44e-04 | 3197.9ms/step | 18.8 samples/s +Step 11550/50462 | Loss: 2.7237 | LR: 4.44e-04 | 3200.0ms/step | 18.8 samples/s +Step 11560/50462 | Loss: 2.7638 | LR: 4.44e-04 | 3196.0ms/step | 18.8 samples/s +Step 11570/50462 | Loss: 2.7404 | LR: 4.44e-04 | 3200.1ms/step | 18.7 samples/s +Step 11580/50462 | Loss: 2.7396 | LR: 4.44e-04 | 3199.9ms/step | 18.8 samples/s +Step 11590/50462 | Loss: 2.7560 | LR: 4.44e-04 | 3200.2ms/step | 18.7 samples/s +Step 11600/50462 | Loss: 2.7365 | LR: 4.43e-04 | 3199.9ms/step | 18.8 samples/s +Step 11610/50462 | Loss: 2.7699 | LR: 4.43e-04 | 3199.9ms/step | 18.8 samples/s +Step 11620/50462 | Loss: 2.7193 | LR: 4.43e-04 | 3200.2ms/step | 18.7 samples/s +Step 11630/50462 | Loss: 2.7486 | LR: 4.43e-04 | 3199.6ms/step | 18.8 samples/s +Step 11640/50462 | Loss: 2.7614 | LR: 4.43e-04 | 3200.2ms/step | 18.7 samples/s +Step 11650/50462 | Loss: 2.7274 | LR: 4.43e-04 | 3198.0ms/step | 18.8 samples/s +Step 11660/50462 | Loss: 2.7305 | LR: 4.43e-04 | 3200.0ms/step | 18.8 samples/s +Step 11670/50462 | Loss: 2.7486 | LR: 4.43e-04 | 3199.8ms/step | 18.8 samples/s +Step 11680/50462 | Loss: 2.7334 | LR: 4.43e-04 | 3252.3ms/step | 18.4 samples/s +Step 11690/50462 | Loss: 2.7307 | LR: 4.43e-04 | 3200.0ms/step | 18.7 samples/s +Step 11700/50462 | Loss: 2.7392 | LR: 4.42e-04 | 3253.6ms/step | 18.4 samples/s +Step 11710/50462 | Loss: 2.7453 | LR: 4.42e-04 | 3186.2ms/step | 18.8 samples/s +Step 11720/50462 | Loss: 2.7411 | LR: 4.42e-04 | 3182.9ms/step | 18.9 samples/s +Step 11730/50462 | Loss: 2.7427 | LR: 4.42e-04 | 3194.8ms/step | 18.8 samples/s +Step 11740/50462 | Loss: 2.7415 | LR: 4.42e-04 | 3196.3ms/step | 18.8 samples/s +Step 11750/50462 | Loss: 2.7332 | LR: 4.42e-04 | 3199.5ms/step | 18.8 samples/s +Validation at step 11750: Loss = 2.7506, Perplexity = 15.65 +Step 11760/50462 | Loss: 2.7182 | LR: 4.42e-04 | 4876.6ms/step | 12.3 samples/s +Step 11770/50462 | Loss: 2.7343 | LR: 4.42e-04 | 3199.9ms/step | 18.8 samples/s +Step 11780/50462 | Loss: 2.7405 | LR: 4.42e-04 | 3200.2ms/step | 18.7 samples/s +Step 11790/50462 | Loss: 2.7280 | LR: 4.42e-04 | 3202.1ms/step | 18.7 samples/s +Step 11800/50462 | Loss: 2.7328 | LR: 4.41e-04 | 3200.0ms/step | 18.8 samples/s +Step 11810/50462 | Loss: 2.7777 | LR: 4.41e-04 | 3199.8ms/step | 18.8 samples/s +Step 11820/50462 | Loss: 2.7581 | LR: 4.41e-04 | 3199.4ms/step | 18.8 samples/s +Step 11830/50462 | Loss: 2.7253 | LR: 4.41e-04 | 3200.7ms/step | 18.7 samples/s +Step 11840/50462 | Loss: 2.7440 | LR: 4.41e-04 | 3200.0ms/step | 18.8 samples/s +Step 11850/50462 | Loss: 2.7377 | LR: 4.41e-04 | 3254.0ms/step | 18.4 samples/s +Step 11860/50462 | Loss: 2.7178 | LR: 4.41e-04 | 3198.1ms/step | 18.8 samples/s +Step 11870/50462 | Loss: 2.7369 | LR: 4.41e-04 | 3199.8ms/step | 18.8 samples/s +Step 11880/50462 | Loss: 2.7345 | LR: 4.41e-04 | 3200.1ms/step | 18.7 samples/s +Step 11890/50462 | Loss: 2.7235 | LR: 4.41e-04 | 3197.3ms/step | 18.8 samples/s +Step 11900/50462 | Loss: 2.7647 | LR: 4.40e-04 | 3198.7ms/step | 18.8 samples/s +Step 11910/50462 | Loss: 2.7578 | LR: 4.40e-04 | 3199.9ms/step | 18.8 samples/s +Step 11920/50462 | Loss: 2.7308 | LR: 4.40e-04 | 3199.9ms/step | 18.8 samples/s +Step 11930/50462 | Loss: 2.7414 | LR: 4.40e-04 | 3199.6ms/step | 18.8 samples/s +Step 11940/50462 | Loss: 2.7268 | LR: 4.40e-04 | 3200.6ms/step | 18.7 samples/s +Step 11950/50462 | Loss: 2.7442 | LR: 4.40e-04 | 3199.5ms/step | 18.8 samples/s +Step 11960/50462 | Loss: 2.7325 | LR: 4.40e-04 | 3195.0ms/step | 18.8 samples/s +Step 11970/50462 | Loss: 2.7618 | LR: 4.40e-04 | 3196.4ms/step | 18.8 samples/s +Step 11980/50462 | Loss: 2.7627 | LR: 4.40e-04 | 3198.7ms/step | 18.8 samples/s +Step 11990/50462 | Loss: 2.7341 | LR: 4.40e-04 | 3200.4ms/step | 18.7 samples/s +Step 12000/50462 | Loss: 2.7734 | LR: 4.39e-04 | 3197.8ms/step | 18.8 samples/s +Validation at step 12000: Loss = 2.7474, Perplexity = 15.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-12000 +Step 12010/50462 | Loss: 2.7342 | LR: 4.39e-04 | 5832.0ms/step | 10.3 samples/s +Step 12020/50462 | Loss: 2.7379 | LR: 4.39e-04 | 3233.4ms/step | 18.6 samples/s +Step 12030/50462 | Loss: 2.7382 | LR: 4.39e-04 | 3187.3ms/step | 18.8 samples/s +Step 12040/50462 | Loss: 2.7328 | LR: 4.39e-04 | 3243.3ms/step | 18.5 samples/s +Step 12050/50462 | Loss: 2.6997 | LR: 4.39e-04 | 3182.3ms/step | 18.9 samples/s +Step 12060/50462 | Loss: 2.7697 | LR: 4.39e-04 | 3191.2ms/step | 18.8 samples/s +Step 12070/50462 | Loss: 2.7305 | LR: 4.39e-04 | 3194.5ms/step | 18.8 samples/s +Step 12080/50462 | Loss: 2.7293 | LR: 4.39e-04 | 3195.4ms/step | 18.8 samples/s +Step 12090/50462 | Loss: 2.7364 | LR: 4.39e-04 | 3196.7ms/step | 18.8 samples/s +Step 12100/50462 | Loss: 2.7394 | LR: 4.38e-04 | 3197.5ms/step | 18.8 samples/s +Step 12110/50462 | Loss: 2.7426 | LR: 4.38e-04 | 3196.6ms/step | 18.8 samples/s +Step 12120/50462 | Loss: 2.7288 | LR: 4.38e-04 | 3200.1ms/step | 18.7 samples/s +Step 12130/50462 | Loss: 2.7194 | LR: 4.38e-04 | 3199.5ms/step | 18.8 samples/s +Step 12140/50462 | Loss: 2.7115 | LR: 4.38e-04 | 3202.2ms/step | 18.7 samples/s +Step 12150/50462 | Loss: 2.7274 | LR: 4.38e-04 | 3197.9ms/step | 18.8 samples/s +Step 12160/50462 | Loss: 2.7557 | LR: 4.38e-04 | 3200.2ms/step | 18.7 samples/s +Step 12170/50462 | Loss: 2.7507 | LR: 4.38e-04 | 3198.2ms/step | 18.8 samples/s +Step 12180/50462 | Loss: 2.7281 | LR: 4.38e-04 | 3199.4ms/step | 18.8 samples/s +Step 12190/50462 | Loss: 2.7309 | LR: 4.37e-04 | 3252.6ms/step | 18.4 samples/s +Step 12200/50462 | Loss: 2.6901 | LR: 4.37e-04 | 3197.1ms/step | 18.8 samples/s +Step 12210/50462 | Loss: 2.7145 | LR: 4.37e-04 | 3196.9ms/step | 18.8 samples/s +Step 12220/50462 | Loss: 2.7185 | LR: 4.37e-04 | 3195.5ms/step | 18.8 samples/s +Step 12230/50462 | Loss: 2.7342 | LR: 4.37e-04 | 3193.7ms/step | 18.8 samples/s +Step 12240/50462 | Loss: 2.7491 | LR: 4.37e-04 | 3196.2ms/step | 18.8 samples/s +Step 12250/50462 | Loss: 2.7193 | LR: 4.37e-04 | 3197.6ms/step | 18.8 samples/s +Validation at step 12250: Loss = 2.7437, Perplexity = 15.54 +Step 12260/50462 | Loss: 2.7134 | LR: 4.37e-04 | 4870.4ms/step | 12.3 samples/s +Step 12270/50462 | Loss: 2.7203 | LR: 4.37e-04 | 3198.5ms/step | 18.8 samples/s +Step 12280/50462 | Loss: 2.7024 | LR: 4.37e-04 | 3192.8ms/step | 18.8 samples/s +Step 12290/50462 | Loss: 2.7496 | LR: 4.36e-04 | 3190.6ms/step | 18.8 samples/s +Step 12300/50462 | Loss: 2.7261 | LR: 4.36e-04 | 3190.6ms/step | 18.8 samples/s +Step 12310/50462 | Loss: 2.7633 | LR: 4.36e-04 | 3198.0ms/step | 18.8 samples/s +Step 12320/50462 | Loss: 2.7375 | LR: 4.36e-04 | 3200.1ms/step | 18.7 samples/s +Step 12330/50462 | Loss: 2.7250 | LR: 4.36e-04 | 3200.0ms/step | 18.8 samples/s +Step 12340/50462 | Loss: 2.7535 | LR: 4.36e-04 | 3200.1ms/step | 18.7 samples/s +Step 12350/50462 | Loss: 2.7267 | LR: 4.36e-04 | 3195.8ms/step | 18.8 samples/s +Step 12360/50462 | Loss: 2.7344 | LR: 4.36e-04 | 3248.0ms/step | 18.5 samples/s +Step 12370/50462 | Loss: 2.7400 | LR: 4.36e-04 | 3200.0ms/step | 18.7 samples/s +Step 12380/50462 | Loss: 2.7418 | LR: 4.35e-04 | 3252.1ms/step | 18.4 samples/s +Step 12390/50462 | Loss: 2.7353 | LR: 4.35e-04 | 3199.9ms/step | 18.8 samples/s +Step 12400/50462 | Loss: 2.7147 | LR: 4.35e-04 | 3199.8ms/step | 18.8 samples/s +Step 12410/50462 | Loss: 2.7352 | LR: 4.35e-04 | 3199.8ms/step | 18.8 samples/s +Step 12420/50462 | Loss: 2.7554 | LR: 4.35e-04 | 3198.3ms/step | 18.8 samples/s +Step 12430/50462 | Loss: 2.7632 | LR: 4.35e-04 | 3199.6ms/step | 18.8 samples/s +Step 12440/50462 | Loss: 2.7256 | LR: 4.35e-04 | 3200.5ms/step | 18.7 samples/s +Step 12450/50462 | Loss: 2.7344 | LR: 4.35e-04 | 3199.9ms/step | 18.8 samples/s +Step 12460/50462 | Loss: 2.7545 | LR: 4.35e-04 | 3200.1ms/step | 18.7 samples/s +Step 12470/50462 | Loss: 2.7385 | LR: 4.35e-04 | 3200.1ms/step | 18.7 samples/s +Step 12480/50462 | Loss: 2.7136 | LR: 4.34e-04 | 3197.7ms/step | 18.8 samples/s +Step 12490/50462 | Loss: 2.7485 | LR: 4.34e-04 | 3200.0ms/step | 18.8 samples/s +Step 12500/50462 | Loss: 2.7584 | LR: 4.34e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 12500: Loss = 2.7384, Perplexity = 15.46 +Step 12510/50462 | Loss: 2.7081 | LR: 4.34e-04 | 4876.3ms/step | 12.3 samples/s +Step 12520/50462 | Loss: 2.7178 | LR: 4.34e-04 | 3200.4ms/step | 18.7 samples/s +Step 12530/50462 | Loss: 2.7458 | LR: 4.34e-04 | 3253.7ms/step | 18.4 samples/s +Step 12540/50462 | Loss: 2.7578 | LR: 4.34e-04 | 3200.1ms/step | 18.7 samples/s +Step 12550/50462 | Loss: 2.7028 | LR: 4.34e-04 | 3200.0ms/step | 18.8 samples/s +Step 12560/50462 | Loss: 2.7478 | LR: 4.34e-04 | 3200.0ms/step | 18.7 samples/s +Step 12570/50462 | Loss: 2.7303 | LR: 4.33e-04 | 3197.8ms/step | 18.8 samples/s +Step 12580/50462 | Loss: 2.7239 | LR: 4.33e-04 | 3199.8ms/step | 18.8 samples/s +Step 12590/50462 | Loss: 2.7419 | LR: 4.33e-04 | 3200.5ms/step | 18.7 samples/s +Step 12600/50462 | Loss: 2.7040 | LR: 4.33e-04 | 3197.0ms/step | 18.8 samples/s +Step 12610/50462 | Loss: 2.7299 | LR: 4.33e-04 | 3198.7ms/step | 18.8 samples/s +Step 12620/50462 | Loss: 2.7146 | LR: 4.33e-04 | 3200.1ms/step | 18.7 samples/s +Step 12630/50462 | Loss: 2.7498 | LR: 4.33e-04 | 3200.1ms/step | 18.7 samples/s +Step 12640/50462 | Loss: 2.7170 | LR: 4.33e-04 | 3197.9ms/step | 18.8 samples/s +Step 12650/50462 | Loss: 2.7388 | LR: 4.33e-04 | 3199.7ms/step | 18.8 samples/s +Step 12660/50462 | Loss: 2.7416 | LR: 4.32e-04 | 3200.5ms/step | 18.7 samples/s +Step 12670/50462 | Loss: 2.7431 | LR: 4.32e-04 | 3199.7ms/step | 18.8 samples/s +Step 12680/50462 | Loss: 2.7091 | LR: 4.32e-04 | 3200.2ms/step | 18.7 samples/s +Step 12690/50462 | Loss: 2.7143 | LR: 4.32e-04 | 3199.9ms/step | 18.8 samples/s +Step 12700/50462 | Loss: 2.7373 | LR: 4.32e-04 | 3252.2ms/step | 18.4 samples/s +Step 12710/50462 | Loss: 2.7262 | LR: 4.32e-04 | 3199.8ms/step | 18.8 samples/s +Step 12720/50462 | Loss: 2.7028 | LR: 4.32e-04 | 3249.2ms/step | 18.5 samples/s +Step 12730/50462 | Loss: 2.7269 | LR: 4.32e-04 | 3200.9ms/step | 18.7 samples/s +Step 12740/50462 | Loss: 2.7075 | LR: 4.32e-04 | 3199.4ms/step | 18.8 samples/s +Step 12750/50462 | Loss: 2.7219 | LR: 4.31e-04 | 3200.4ms/step | 18.7 samples/s +Validation at step 12750: Loss = 2.7344, Perplexity = 15.40 +Step 12760/50462 | Loss: 2.7079 | LR: 4.31e-04 | 4876.2ms/step | 12.3 samples/s +Step 12770/50462 | Loss: 2.7343 | LR: 4.31e-04 | 3200.0ms/step | 18.7 samples/s +Step 12780/50462 | Loss: 2.7357 | LR: 4.31e-04 | 3199.8ms/step | 18.8 samples/s +Step 12790/50462 | Loss: 2.7127 | LR: 4.31e-04 | 3200.0ms/step | 18.8 samples/s +Step 12800/50462 | Loss: 2.7272 | LR: 4.31e-04 | 3199.6ms/step | 18.8 samples/s +Step 12810/50462 | Loss: 2.7196 | LR: 4.31e-04 | 3198.5ms/step | 18.8 samples/s +Step 12820/50462 | Loss: 2.7509 | LR: 4.31e-04 | 3199.7ms/step | 18.8 samples/s +Step 12830/50462 | Loss: 2.7134 | LR: 4.31e-04 | 3200.3ms/step | 18.7 samples/s +Step 12840/50462 | Loss: 2.7324 | LR: 4.31e-04 | 3199.9ms/step | 18.8 samples/s +Step 12850/50462 | Loss: 2.7336 | LR: 4.30e-04 | 3200.1ms/step | 18.7 samples/s +Step 12860/50462 | Loss: 2.7014 | LR: 4.30e-04 | 3200.3ms/step | 18.7 samples/s +Step 12870/50462 | Loss: 2.7351 | LR: 4.30e-04 | 3253.8ms/step | 18.4 samples/s +Step 12880/50462 | Loss: 2.6995 | LR: 4.30e-04 | 3200.1ms/step | 18.7 samples/s +Step 12890/50462 | Loss: 2.7365 | LR: 4.30e-04 | 3199.7ms/step | 18.8 samples/s +Step 12900/50462 | Loss: 2.7538 | LR: 4.30e-04 | 3200.1ms/step | 18.7 samples/s +Step 12910/50462 | Loss: 2.7080 | LR: 4.30e-04 | 3200.1ms/step | 18.7 samples/s +Step 12920/50462 | Loss: 2.7137 | LR: 4.30e-04 | 3199.9ms/step | 18.8 samples/s +Step 12930/50462 | Loss: 2.7192 | LR: 4.30e-04 | 3199.9ms/step | 18.8 samples/s +Step 12940/50462 | Loss: 2.7417 | LR: 4.29e-04 | 3200.2ms/step | 18.7 samples/s +Step 12950/50462 | Loss: 2.7292 | LR: 4.29e-04 | 3200.0ms/step | 18.8 samples/s +Step 12960/50462 | Loss: 2.7079 | LR: 4.29e-04 | 3200.1ms/step | 18.7 samples/s +Step 12970/50462 | Loss: 2.7280 | LR: 4.29e-04 | 3199.8ms/step | 18.8 samples/s +Step 12980/50462 | Loss: 2.7362 | LR: 4.29e-04 | 3200.2ms/step | 18.7 samples/s +Step 12990/50462 | Loss: 2.7008 | LR: 4.29e-04 | 3200.0ms/step | 18.8 samples/s +Step 13000/50462 | Loss: 2.7300 | LR: 4.29e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 13000: Loss = 2.7310, Perplexity = 15.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-13000 +Step 13010/50462 | Loss: 2.7004 | LR: 4.29e-04 | 5941.6ms/step | 10.1 samples/s +Step 13020/50462 | Loss: 2.7135 | LR: 4.29e-04 | 3181.5ms/step | 18.9 samples/s +Step 13030/50462 | Loss: 2.7239 | LR: 4.28e-04 | 3194.5ms/step | 18.8 samples/s +Step 13040/50462 | Loss: 2.7035 | LR: 4.28e-04 | 3248.0ms/step | 18.5 samples/s +Step 13050/50462 | Loss: 2.7214 | LR: 4.28e-04 | 3198.0ms/step | 18.8 samples/s +Step 13060/50462 | Loss: 2.7556 | LR: 4.28e-04 | 3250.1ms/step | 18.5 samples/s +Step 13070/50462 | Loss: 2.7297 | LR: 4.28e-04 | 3199.9ms/step | 18.8 samples/s +Step 13080/50462 | Loss: 2.7371 | LR: 4.28e-04 | 3199.5ms/step | 18.8 samples/s +Step 13090/50462 | Loss: 2.7071 | LR: 4.28e-04 | 3200.4ms/step | 18.7 samples/s +Step 13100/50462 | Loss: 2.6998 | LR: 4.28e-04 | 3198.0ms/step | 18.8 samples/s +Step 13110/50462 | Loss: 2.7142 | LR: 4.28e-04 | 3200.2ms/step | 18.7 samples/s +Step 13120/50462 | Loss: 2.7366 | LR: 4.27e-04 | 3199.9ms/step | 18.8 samples/s +Step 13130/50462 | Loss: 2.7232 | LR: 4.27e-04 | 3199.2ms/step | 18.8 samples/s +Step 13140/50462 | Loss: 2.7464 | LR: 4.27e-04 | 3200.4ms/step | 18.7 samples/s +Step 13150/50462 | Loss: 2.7118 | LR: 4.27e-04 | 3198.3ms/step | 18.8 samples/s +Step 13160/50462 | Loss: 2.7144 | LR: 4.27e-04 | 3200.1ms/step | 18.7 samples/s +Step 13170/50462 | Loss: 2.7109 | LR: 4.27e-04 | 3198.0ms/step | 18.8 samples/s +Step 13180/50462 | Loss: 2.7395 | LR: 4.27e-04 | 3200.0ms/step | 18.8 samples/s +Step 13190/50462 | Loss: 2.7136 | LR: 4.27e-04 | 3199.8ms/step | 18.8 samples/s +Step 13200/50462 | Loss: 2.7202 | LR: 4.27e-04 | 3254.2ms/step | 18.4 samples/s +Step 13210/50462 | Loss: 2.7099 | LR: 4.26e-04 | 3199.2ms/step | 18.8 samples/s +Step 13220/50462 | Loss: 2.7359 | LR: 4.26e-04 | 3200.4ms/step | 18.7 samples/s +Step 13230/50462 | Loss: 2.6800 | LR: 4.26e-04 | 3200.5ms/step | 18.7 samples/s +Step 13240/50462 | Loss: 2.7309 | LR: 4.26e-04 | 3199.9ms/step | 18.8 samples/s +Step 13250/50462 | Loss: 2.7055 | LR: 4.26e-04 | 3197.8ms/step | 18.8 samples/s +Validation at step 13250: Loss = 2.7271, Perplexity = 15.29 +Step 13260/50462 | Loss: 2.7524 | LR: 4.26e-04 | 4876.1ms/step | 12.3 samples/s +Step 13270/50462 | Loss: 2.7294 | LR: 4.26e-04 | 3199.8ms/step | 18.8 samples/s +Step 13280/50462 | Loss: 2.7084 | LR: 4.26e-04 | 3200.2ms/step | 18.7 samples/s +Step 13290/50462 | Loss: 2.7386 | LR: 4.26e-04 | 3199.9ms/step | 18.8 samples/s +Step 13300/50462 | Loss: 2.7249 | LR: 4.25e-04 | 3200.2ms/step | 18.7 samples/s +Step 13310/50462 | Loss: 2.7262 | LR: 4.25e-04 | 3199.8ms/step | 18.8 samples/s +Step 13320/50462 | Loss: 2.7077 | LR: 4.25e-04 | 3200.0ms/step | 18.7 samples/s +Step 13330/50462 | Loss: 2.6737 | LR: 4.25e-04 | 3200.0ms/step | 18.8 samples/s +Step 13340/50462 | Loss: 2.7189 | LR: 4.25e-04 | 3200.0ms/step | 18.7 samples/s +Step 13350/50462 | Loss: 2.7225 | LR: 4.25e-04 | 3200.1ms/step | 18.7 samples/s +Step 13360/50462 | Loss: 2.7311 | LR: 4.25e-04 | 3200.0ms/step | 18.8 samples/s +Step 13370/50462 | Loss: 2.6863 | LR: 4.25e-04 | 3194.6ms/step | 18.8 samples/s +Step 13380/50462 | Loss: 2.6944 | LR: 4.25e-04 | 3246.0ms/step | 18.5 samples/s +Step 13390/50462 | Loss: 2.7192 | LR: 4.24e-04 | 3189.4ms/step | 18.8 samples/s +Step 13400/50462 | Loss: 2.7010 | LR: 4.24e-04 | 3243.4ms/step | 18.5 samples/s +Step 13410/50462 | Loss: 2.7270 | LR: 4.24e-04 | 3196.8ms/step | 18.8 samples/s +Step 13420/50462 | Loss: 2.6980 | LR: 4.24e-04 | 3195.7ms/step | 18.8 samples/s +Step 13430/50462 | Loss: 2.7524 | LR: 4.24e-04 | 3200.1ms/step | 18.7 samples/s +Step 13440/50462 | Loss: 2.7224 | LR: 4.24e-04 | 3198.1ms/step | 18.8 samples/s +Step 13450/50462 | Loss: 2.7131 | LR: 4.24e-04 | 3200.0ms/step | 18.8 samples/s +Step 13460/50462 | Loss: 2.7410 | LR: 4.24e-04 | 3200.1ms/step | 18.7 samples/s +Step 13470/50462 | Loss: 2.7203 | LR: 4.23e-04 | 3197.9ms/step | 18.8 samples/s +Step 13480/50462 | Loss: 2.6977 | LR: 4.23e-04 | 3199.9ms/step | 18.8 samples/s +Step 13490/50462 | Loss: 2.6981 | LR: 4.23e-04 | 3199.8ms/step | 18.8 samples/s +Step 13500/50462 | Loss: 2.7297 | LR: 4.23e-04 | 3200.3ms/step | 18.7 samples/s +Validation at step 13500: Loss = 2.7248, Perplexity = 15.25 +Step 13510/50462 | Loss: 2.7361 | LR: 4.23e-04 | 4874.0ms/step | 12.3 samples/s +Step 13520/50462 | Loss: 2.6914 | LR: 4.23e-04 | 3199.7ms/step | 18.8 samples/s +Step 13530/50462 | Loss: 2.7312 | LR: 4.23e-04 | 3200.1ms/step | 18.7 samples/s +Step 13540/50462 | Loss: 2.7137 | LR: 4.23e-04 | 3252.3ms/step | 18.4 samples/s +Step 13550/50462 | Loss: 2.7360 | LR: 4.23e-04 | 3199.8ms/step | 18.8 samples/s +Step 13560/50462 | Loss: 2.7028 | LR: 4.22e-04 | 3198.1ms/step | 18.8 samples/s +Step 13570/50462 | Loss: 2.7078 | LR: 4.22e-04 | 3200.1ms/step | 18.7 samples/s +Step 13580/50462 | Loss: 2.7149 | LR: 4.22e-04 | 3200.0ms/step | 18.8 samples/s +Step 13590/50462 | Loss: 2.7245 | LR: 4.22e-04 | 3198.7ms/step | 18.8 samples/s +Step 13600/50462 | Loss: 2.7188 | LR: 4.22e-04 | 3194.5ms/step | 18.8 samples/s +Step 13610/50462 | Loss: 2.7167 | LR: 4.22e-04 | 3198.3ms/step | 18.8 samples/s +Step 13620/50462 | Loss: 2.6955 | LR: 4.22e-04 | 3199.8ms/step | 18.8 samples/s +Step 13630/50462 | Loss: 2.7032 | LR: 4.22e-04 | 3198.6ms/step | 18.8 samples/s +Step 13640/50462 | Loss: 2.7300 | LR: 4.22e-04 | 3200.0ms/step | 18.8 samples/s +Step 13650/50462 | Loss: 2.7051 | LR: 4.21e-04 | 3199.7ms/step | 18.8 samples/s +Step 13660/50462 | Loss: 2.7269 | LR: 4.21e-04 | 3198.4ms/step | 18.8 samples/s +Step 13670/50462 | Loss: 2.7052 | LR: 4.21e-04 | 3197.9ms/step | 18.8 samples/s +Step 13680/50462 | Loss: 2.7204 | LR: 4.21e-04 | 3199.4ms/step | 18.8 samples/s +Step 13690/50462 | Loss: 2.7065 | LR: 4.21e-04 | 3200.7ms/step | 18.7 samples/s +Step 13700/50462 | Loss: 2.7315 | LR: 4.21e-04 | 3199.9ms/step | 18.8 samples/s +Step 13710/50462 | Loss: 2.6909 | LR: 4.21e-04 | 3199.4ms/step | 18.8 samples/s +Step 13720/50462 | Loss: 2.7184 | LR: 4.21e-04 | 3250.6ms/step | 18.5 samples/s +Step 13730/50462 | Loss: 2.7016 | LR: 4.21e-04 | 3196.1ms/step | 18.8 samples/s +Step 13740/50462 | Loss: 2.7141 | LR: 4.20e-04 | 3251.5ms/step | 18.5 samples/s +Step 13750/50462 | Loss: 2.7368 | LR: 4.20e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 13750: Loss = 2.7200, Perplexity = 15.18 +Step 13760/50462 | Loss: 2.7041 | LR: 4.20e-04 | 4875.8ms/step | 12.3 samples/s +Step 13770/50462 | Loss: 2.7082 | LR: 4.20e-04 | 3198.3ms/step | 18.8 samples/s +Step 13780/50462 | Loss: 2.7240 | LR: 4.20e-04 | 3200.1ms/step | 18.7 samples/s +Step 13790/50462 | Loss: 2.7021 | LR: 4.20e-04 | 3199.9ms/step | 18.8 samples/s +Step 13800/50462 | Loss: 2.7145 | LR: 4.20e-04 | 3199.8ms/step | 18.8 samples/s +Step 13810/50462 | Loss: 2.6868 | LR: 4.20e-04 | 3198.1ms/step | 18.8 samples/s +Step 13820/50462 | Loss: 2.7018 | LR: 4.19e-04 | 3198.1ms/step | 18.8 samples/s +Step 13830/50462 | Loss: 2.7109 | LR: 4.19e-04 | 3200.1ms/step | 18.7 samples/s +Step 13840/50462 | Loss: 2.6973 | LR: 4.19e-04 | 3199.8ms/step | 18.8 samples/s +Step 13850/50462 | Loss: 2.6805 | LR: 4.19e-04 | 3199.5ms/step | 18.8 samples/s +Step 13860/50462 | Loss: 2.6923 | LR: 4.19e-04 | 3200.5ms/step | 18.7 samples/s +Step 13870/50462 | Loss: 2.6976 | LR: 4.19e-04 | 3199.9ms/step | 18.8 samples/s +Step 13880/50462 | Loss: 2.7108 | LR: 4.19e-04 | 3252.4ms/step | 18.4 samples/s +Step 13890/50462 | Loss: 2.7098 | LR: 4.19e-04 | 3193.1ms/step | 18.8 samples/s +Step 13900/50462 | Loss: 2.7329 | LR: 4.19e-04 | 3195.5ms/step | 18.8 samples/s +Step 13910/50462 | Loss: 2.7434 | LR: 4.18e-04 | 3196.8ms/step | 18.8 samples/s +Step 13920/50462 | Loss: 2.6990 | LR: 4.18e-04 | 3198.0ms/step | 18.8 samples/s +Step 13930/50462 | Loss: 2.7014 | LR: 4.18e-04 | 3190.2ms/step | 18.8 samples/s +Step 13940/50462 | Loss: 2.7136 | LR: 4.18e-04 | 3192.6ms/step | 18.8 samples/s +Step 13950/50462 | Loss: 2.7044 | LR: 4.18e-04 | 3195.2ms/step | 18.8 samples/s +Step 13960/50462 | Loss: 2.6996 | LR: 4.18e-04 | 3200.1ms/step | 18.7 samples/s +Step 13970/50462 | Loss: 2.7265 | LR: 4.18e-04 | 3193.3ms/step | 18.8 samples/s +Step 13980/50462 | Loss: 2.6842 | LR: 4.18e-04 | 3193.2ms/step | 18.8 samples/s +Step 13990/50462 | Loss: 2.6989 | LR: 4.17e-04 | 3191.6ms/step | 18.8 samples/s +Step 14000/50462 | Loss: 2.7254 | LR: 4.17e-04 | 3185.0ms/step | 18.8 samples/s +Validation at step 14000: Loss = 2.7176, Perplexity = 15.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-14000 +Step 14010/50462 | Loss: 2.7409 | LR: 4.17e-04 | 5816.1ms/step | 10.3 samples/s +Step 14020/50462 | Loss: 2.7158 | LR: 4.17e-04 | 3189.8ms/step | 18.8 samples/s +Step 14030/50462 | Loss: 2.6908 | LR: 4.17e-04 | 3200.8ms/step | 18.7 samples/s +Step 14040/50462 | Loss: 2.6995 | LR: 4.17e-04 | 3198.4ms/step | 18.8 samples/s +Step 14050/50462 | Loss: 2.6986 | LR: 4.17e-04 | 3199.5ms/step | 18.8 samples/s +Step 14060/50462 | Loss: 2.7209 | LR: 4.17e-04 | 3252.2ms/step | 18.4 samples/s +Step 14070/50462 | Loss: 2.6820 | LR: 4.17e-04 | 3199.8ms/step | 18.8 samples/s +Step 14080/50462 | Loss: 2.7219 | LR: 4.16e-04 | 3251.6ms/step | 18.5 samples/s +Step 14090/50462 | Loss: 2.7055 | LR: 4.16e-04 | 3196.3ms/step | 18.8 samples/s +Step 14100/50462 | Loss: 2.6880 | LR: 4.16e-04 | 3194.4ms/step | 18.8 samples/s +Step 14110/50462 | Loss: 2.6811 | LR: 4.16e-04 | 3191.8ms/step | 18.8 samples/s +Step 14120/50462 | Loss: 2.7039 | LR: 4.16e-04 | 3197.5ms/step | 18.8 samples/s +Step 14130/50462 | Loss: 2.7182 | LR: 4.16e-04 | 3200.4ms/step | 18.7 samples/s +Step 14140/50462 | Loss: 2.7290 | LR: 4.16e-04 | 3199.4ms/step | 18.8 samples/s +Step 14150/50462 | Loss: 2.7189 | LR: 4.16e-04 | 3199.9ms/step | 18.8 samples/s +Step 14160/50462 | Loss: 2.7055 | LR: 4.15e-04 | 3200.5ms/step | 18.7 samples/s +Step 14170/50462 | Loss: 2.7070 | LR: 4.15e-04 | 3200.1ms/step | 18.7 samples/s +Step 14180/50462 | Loss: 2.7526 | LR: 4.15e-04 | 3199.9ms/step | 18.8 samples/s +Step 14190/50462 | Loss: 2.7137 | LR: 4.15e-04 | 3200.2ms/step | 18.7 samples/s +Step 14200/50462 | Loss: 2.7396 | LR: 4.15e-04 | 3200.1ms/step | 18.7 samples/s +Step 14210/50462 | Loss: 2.6755 | LR: 4.15e-04 | 3199.6ms/step | 18.8 samples/s +Step 14220/50462 | Loss: 2.7297 | LR: 4.15e-04 | 3254.5ms/step | 18.4 samples/s +Step 14230/50462 | Loss: 2.7282 | LR: 4.15e-04 | 3199.4ms/step | 18.8 samples/s +Step 14240/50462 | Loss: 2.6979 | LR: 4.15e-04 | 3200.4ms/step | 18.7 samples/s +Step 14250/50462 | Loss: 2.7288 | LR: 4.14e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 14250: Loss = 2.7139, Perplexity = 15.09 +Step 14260/50462 | Loss: 2.7006 | LR: 4.14e-04 | 4874.1ms/step | 12.3 samples/s +Step 14270/50462 | Loss: 2.7240 | LR: 4.14e-04 | 3200.0ms/step | 18.7 samples/s +Step 14280/50462 | Loss: 2.6844 | LR: 4.14e-04 | 3200.1ms/step | 18.7 samples/s +Step 14290/50462 | Loss: 2.6976 | LR: 4.14e-04 | 3199.8ms/step | 18.8 samples/s +Step 14300/50462 | Loss: 2.7243 | LR: 4.14e-04 | 3200.1ms/step | 18.7 samples/s +Step 14310/50462 | Loss: 2.6759 | LR: 4.14e-04 | 3200.0ms/step | 18.8 samples/s +Step 14320/50462 | Loss: 2.6966 | LR: 4.14e-04 | 3197.6ms/step | 18.8 samples/s +Step 14330/50462 | Loss: 2.6913 | LR: 4.13e-04 | 3200.2ms/step | 18.7 samples/s +Step 14340/50462 | Loss: 2.6849 | LR: 4.13e-04 | 3200.2ms/step | 18.7 samples/s +Step 14350/50462 | Loss: 2.6988 | LR: 4.13e-04 | 3197.4ms/step | 18.8 samples/s +Step 14360/50462 | Loss: 2.6818 | LR: 4.13e-04 | 3195.9ms/step | 18.8 samples/s +Step 14370/50462 | Loss: 2.7168 | LR: 4.13e-04 | 3198.5ms/step | 18.8 samples/s +Step 14380/50462 | Loss: 2.7253 | LR: 4.13e-04 | 3199.9ms/step | 18.8 samples/s +Step 14390/50462 | Loss: 2.7179 | LR: 4.13e-04 | 3200.0ms/step | 18.8 samples/s +Step 14400/50462 | Loss: 2.7130 | LR: 4.13e-04 | 3251.7ms/step | 18.5 samples/s +Step 14410/50462 | Loss: 2.7244 | LR: 4.12e-04 | 3198.6ms/step | 18.8 samples/s +Step 14420/50462 | Loss: 2.6900 | LR: 4.12e-04 | 3252.2ms/step | 18.4 samples/s +Step 14430/50462 | Loss: 2.6994 | LR: 4.12e-04 | 3199.5ms/step | 18.8 samples/s +Step 14440/50462 | Loss: 2.7205 | LR: 4.12e-04 | 3200.0ms/step | 18.7 samples/s +Step 14450/50462 | Loss: 2.7069 | LR: 4.12e-04 | 3199.6ms/step | 18.8 samples/s +Step 14460/50462 | Loss: 2.6903 | LR: 4.12e-04 | 3200.4ms/step | 18.7 samples/s +Step 14470/50462 | Loss: 2.7089 | LR: 4.12e-04 | 3199.8ms/step | 18.8 samples/s +Step 14480/50462 | Loss: 2.6715 | LR: 4.12e-04 | 3200.3ms/step | 18.7 samples/s +Step 14490/50462 | Loss: 2.7242 | LR: 4.12e-04 | 3200.2ms/step | 18.7 samples/s +Step 14500/50462 | Loss: 2.7040 | LR: 4.11e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 14500: Loss = 2.7103, Perplexity = 15.03 +Step 14510/50462 | Loss: 2.7190 | LR: 4.11e-04 | 4873.6ms/step | 12.3 samples/s +Step 14520/50462 | Loss: 2.7036 | LR: 4.11e-04 | 3200.1ms/step | 18.7 samples/s +Step 14530/50462 | Loss: 2.7152 | LR: 4.11e-04 | 3196.5ms/step | 18.8 samples/s +Step 14540/50462 | Loss: 2.6903 | LR: 4.11e-04 | 3198.0ms/step | 18.8 samples/s +Step 14550/50462 | Loss: 2.7277 | LR: 4.11e-04 | 3251.2ms/step | 18.5 samples/s +Step 14560/50462 | Loss: 2.7263 | LR: 4.11e-04 | 3190.3ms/step | 18.8 samples/s +Step 14570/50462 | Loss: 2.6921 | LR: 4.11e-04 | 3194.3ms/step | 18.8 samples/s +Step 14580/50462 | Loss: 2.7046 | LR: 4.10e-04 | 3192.8ms/step | 18.8 samples/s +Step 14590/50462 | Loss: 2.7112 | LR: 4.10e-04 | 3189.0ms/step | 18.8 samples/s +Step 14600/50462 | Loss: 2.6980 | LR: 4.10e-04 | 3189.7ms/step | 18.8 samples/s +Step 14610/50462 | Loss: 2.7117 | LR: 4.10e-04 | 3190.4ms/step | 18.8 samples/s +Step 14620/50462 | Loss: 2.6733 | LR: 4.10e-04 | 3195.1ms/step | 18.8 samples/s +Step 14630/50462 | Loss: 2.6824 | LR: 4.10e-04 | 3201.3ms/step | 18.7 samples/s +Step 14640/50462 | Loss: 2.6925 | LR: 4.10e-04 | 3198.0ms/step | 18.8 samples/s +Step 14650/50462 | Loss: 2.6829 | LR: 4.10e-04 | 3194.0ms/step | 18.8 samples/s +Step 14660/50462 | Loss: 2.7352 | LR: 4.09e-04 | 3197.4ms/step | 18.8 samples/s +Step 14670/50462 | Loss: 2.6822 | LR: 4.09e-04 | 3198.5ms/step | 18.8 samples/s +Step 14680/50462 | Loss: 2.6963 | LR: 4.09e-04 | 3199.9ms/step | 18.8 samples/s +Step 14690/50462 | Loss: 2.6849 | LR: 4.09e-04 | 3200.0ms/step | 18.7 samples/s +Step 14700/50462 | Loss: 2.6980 | LR: 4.09e-04 | 3200.1ms/step | 18.7 samples/s +Step 14710/50462 | Loss: 2.6816 | LR: 4.09e-04 | 3200.1ms/step | 18.7 samples/s +Step 14720/50462 | Loss: 2.6913 | LR: 4.09e-04 | 3199.9ms/step | 18.8 samples/s +Step 14730/50462 | Loss: 2.6723 | LR: 4.09e-04 | 3200.1ms/step | 18.7 samples/s +Step 14740/50462 | Loss: 2.6763 | LR: 4.08e-04 | 3254.1ms/step | 18.4 samples/s +Step 14750/50462 | Loss: 2.6854 | LR: 4.08e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 14750: Loss = 2.7082, Perplexity = 15.00 +Step 14760/50462 | Loss: 2.6937 | LR: 4.08e-04 | 4928.1ms/step | 12.2 samples/s +Step 14770/50462 | Loss: 2.6882 | LR: 4.08e-04 | 3199.9ms/step | 18.8 samples/s +Step 14780/50462 | Loss: 2.6869 | LR: 4.08e-04 | 3200.0ms/step | 18.8 samples/s +Step 14790/50462 | Loss: 2.7212 | LR: 4.08e-04 | 3199.9ms/step | 18.8 samples/s +Step 14800/50462 | Loss: 2.6806 | LR: 4.08e-04 | 3200.0ms/step | 18.8 samples/s +Step 14810/50462 | Loss: 2.6772 | LR: 4.08e-04 | 3200.0ms/step | 18.7 samples/s +Step 14820/50462 | Loss: 2.7019 | LR: 4.08e-04 | 3199.6ms/step | 18.8 samples/s +Step 14830/50462 | Loss: 2.6894 | LR: 4.07e-04 | 3200.3ms/step | 18.7 samples/s +Step 14840/50462 | Loss: 2.6948 | LR: 4.07e-04 | 3200.3ms/step | 18.7 samples/s +Step 14850/50462 | Loss: 2.6718 | LR: 4.07e-04 | 3199.9ms/step | 18.8 samples/s +Step 14860/50462 | Loss: 2.7021 | LR: 4.07e-04 | 3199.9ms/step | 18.8 samples/s +Step 14870/50462 | Loss: 2.6571 | LR: 4.07e-04 | 3198.6ms/step | 18.8 samples/s +Step 14880/50462 | Loss: 2.7237 | LR: 4.07e-04 | 3199.3ms/step | 18.8 samples/s +Step 14890/50462 | Loss: 2.7198 | LR: 4.07e-04 | 3250.1ms/step | 18.5 samples/s +Step 14900/50462 | Loss: 2.6892 | LR: 4.07e-04 | 3200.3ms/step | 18.7 samples/s +Step 14910/50462 | Loss: 2.6898 | LR: 4.06e-04 | 3199.9ms/step | 18.8 samples/s +Step 14920/50462 | Loss: 2.6745 | LR: 4.06e-04 | 3200.0ms/step | 18.8 samples/s +Step 14930/50462 | Loss: 2.6889 | LR: 4.06e-04 | 3199.4ms/step | 18.8 samples/s +Step 14940/50462 | Loss: 2.6902 | LR: 4.06e-04 | 3200.7ms/step | 18.7 samples/s +Step 14950/50462 | Loss: 2.6909 | LR: 4.06e-04 | 3199.9ms/step | 18.8 samples/s +Step 14960/50462 | Loss: 2.7219 | LR: 4.06e-04 | 3200.1ms/step | 18.7 samples/s +Step 14970/50462 | Loss: 2.6743 | LR: 4.06e-04 | 3200.0ms/step | 18.8 samples/s +Step 14980/50462 | Loss: 2.6939 | LR: 4.06e-04 | 3198.8ms/step | 18.8 samples/s +Step 14990/50462 | Loss: 2.6489 | LR: 4.05e-04 | 3198.9ms/step | 18.8 samples/s +Step 15000/50462 | Loss: 2.6989 | LR: 4.05e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 15000: Loss = 2.7044, Perplexity = 14.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-15000 +Step 15010/50462 | Loss: 2.6975 | LR: 4.05e-04 | 5805.3ms/step | 10.3 samples/s +Step 15020/50462 | Loss: 2.6792 | LR: 4.05e-04 | 3183.7ms/step | 18.8 samples/s +Step 15030/50462 | Loss: 2.7073 | LR: 4.05e-04 | 3195.1ms/step | 18.8 samples/s +Step 15040/50462 | Loss: 2.6962 | LR: 4.05e-04 | 3199.7ms/step | 18.8 samples/s +Step 15050/50462 | Loss: 2.7206 | LR: 4.05e-04 | 3200.4ms/step | 18.7 samples/s +Step 15060/50462 | Loss: 2.6953 | LR: 4.05e-04 | 3199.9ms/step | 18.8 samples/s +Step 15070/50462 | Loss: 2.6706 | LR: 4.04e-04 | 3200.2ms/step | 18.7 samples/s +Step 15080/50462 | Loss: 2.6900 | LR: 4.04e-04 | 3253.9ms/step | 18.4 samples/s +Step 15090/50462 | Loss: 2.7082 | LR: 4.04e-04 | 3199.9ms/step | 18.8 samples/s +Step 15100/50462 | Loss: 2.7033 | LR: 4.04e-04 | 3251.7ms/step | 18.5 samples/s +Step 15110/50462 | Loss: 2.6958 | LR: 4.04e-04 | 3193.6ms/step | 18.8 samples/s +Step 15120/50462 | Loss: 2.6925 | LR: 4.04e-04 | 3194.7ms/step | 18.8 samples/s +Step 15130/50462 | Loss: 2.6932 | LR: 4.04e-04 | 3198.1ms/step | 18.8 samples/s +Step 15140/50462 | Loss: 2.6981 | LR: 4.04e-04 | 3200.1ms/step | 18.7 samples/s +Step 15150/50462 | Loss: 2.6756 | LR: 4.03e-04 | 3199.8ms/step | 18.8 samples/s +Step 15160/50462 | Loss: 2.6697 | LR: 4.03e-04 | 3197.6ms/step | 18.8 samples/s +Step 15170/50462 | Loss: 2.6962 | LR: 4.03e-04 | 3194.1ms/step | 18.8 samples/s +Step 15180/50462 | Loss: 2.6853 | LR: 4.03e-04 | 3196.2ms/step | 18.8 samples/s +Step 15190/50462 | Loss: 2.7104 | LR: 4.03e-04 | 3199.9ms/step | 18.8 samples/s +Step 15200/50462 | Loss: 2.6729 | LR: 4.03e-04 | 3198.2ms/step | 18.8 samples/s +Step 15210/50462 | Loss: 2.6768 | LR: 4.03e-04 | 3200.1ms/step | 18.7 samples/s +Step 15220/50462 | Loss: 2.6628 | LR: 4.03e-04 | 3199.9ms/step | 18.8 samples/s +Step 15230/50462 | Loss: 2.6792 | LR: 4.02e-04 | 3254.1ms/step | 18.4 samples/s +Step 15240/50462 | Loss: 2.7016 | LR: 4.02e-04 | 3199.8ms/step | 18.8 samples/s +Step 15250/50462 | Loss: 2.7205 | LR: 4.02e-04 | 3200.0ms/step | 18.7 samples/s +Validation at step 15250: Loss = 2.7007, Perplexity = 14.89 +Step 15260/50462 | Loss: 2.6975 | LR: 4.02e-04 | 4875.9ms/step | 12.3 samples/s +Step 15270/50462 | Loss: 2.6912 | LR: 4.02e-04 | 3200.4ms/step | 18.7 samples/s +Step 15280/50462 | Loss: 2.6826 | LR: 4.02e-04 | 3199.9ms/step | 18.8 samples/s +Step 15290/50462 | Loss: 2.6756 | LR: 4.02e-04 | 3199.5ms/step | 18.8 samples/s +Step 15300/50462 | Loss: 2.7078 | LR: 4.02e-04 | 3200.6ms/step | 18.7 samples/s +Step 15310/50462 | Loss: 2.6996 | LR: 4.01e-04 | 3199.9ms/step | 18.8 samples/s +Step 15320/50462 | Loss: 2.7034 | LR: 4.01e-04 | 3199.7ms/step | 18.8 samples/s +Step 15330/50462 | Loss: 2.7003 | LR: 4.01e-04 | 3199.7ms/step | 18.8 samples/s +Step 15340/50462 | Loss: 2.6963 | LR: 4.01e-04 | 3200.7ms/step | 18.7 samples/s +Step 15350/50462 | Loss: 2.7018 | LR: 4.01e-04 | 3199.9ms/step | 18.8 samples/s +Step 15360/50462 | Loss: 2.6997 | LR: 4.01e-04 | 3200.0ms/step | 18.8 samples/s +Step 15370/50462 | Loss: 2.7008 | LR: 4.01e-04 | 3200.0ms/step | 18.7 samples/s +Step 15380/50462 | Loss: 2.6646 | LR: 4.01e-04 | 3199.4ms/step | 18.8 samples/s +Step 15390/50462 | Loss: 2.7112 | LR: 4.00e-04 | 3200.7ms/step | 18.7 samples/s +Step 15400/50462 | Loss: 2.6774 | LR: 4.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 15410/50462 | Loss: 2.6732 | LR: 4.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 15420/50462 | Loss: 2.6847 | LR: 4.00e-04 | 3251.8ms/step | 18.5 samples/s +Step 15430/50462 | Loss: 2.6699 | LR: 4.00e-04 | 3200.2ms/step | 18.7 samples/s +Step 15440/50462 | Loss: 2.6755 | LR: 4.00e-04 | 3252.7ms/step | 18.4 samples/s +Step 15450/50462 | Loss: 2.6818 | LR: 4.00e-04 | 3199.4ms/step | 18.8 samples/s +Step 15460/50462 | Loss: 2.6617 | LR: 4.00e-04 | 3199.9ms/step | 18.8 samples/s +Step 15470/50462 | Loss: 2.6927 | LR: 3.99e-04 | 3200.1ms/step | 18.7 samples/s +Step 15480/50462 | Loss: 2.6728 | LR: 3.99e-04 | 3200.0ms/step | 18.8 samples/s +Step 15490/50462 | Loss: 2.6782 | LR: 3.99e-04 | 3196.1ms/step | 18.8 samples/s +Step 15500/50462 | Loss: 2.6533 | LR: 3.99e-04 | 3199.7ms/step | 18.8 samples/s +Validation at step 15500: Loss = 2.6981, Perplexity = 14.85 +Step 15510/50462 | Loss: 2.6902 | LR: 3.99e-04 | 4874.0ms/step | 12.3 samples/s +Step 15520/50462 | Loss: 2.6679 | LR: 3.99e-04 | 3198.0ms/step | 18.8 samples/s +Step 15530/50462 | Loss: 2.6679 | LR: 3.99e-04 | 3200.4ms/step | 18.7 samples/s +Step 15540/50462 | Loss: 2.6712 | LR: 3.99e-04 | 3197.7ms/step | 18.8 samples/s +Step 15550/50462 | Loss: 2.6770 | LR: 3.98e-04 | 3197.9ms/step | 18.8 samples/s +Step 15560/50462 | Loss: 2.6951 | LR: 3.98e-04 | 3200.0ms/step | 18.8 samples/s +Step 15570/50462 | Loss: 2.7300 | LR: 3.98e-04 | 3256.2ms/step | 18.4 samples/s +Step 15580/50462 | Loss: 2.6749 | LR: 3.98e-04 | 3200.0ms/step | 18.8 samples/s +Step 15590/50462 | Loss: 2.6525 | LR: 3.98e-04 | 3200.1ms/step | 18.7 samples/s +Step 15600/50462 | Loss: 2.6663 | LR: 3.98e-04 | 3200.0ms/step | 18.7 samples/s +Step 15610/50462 | Loss: 2.6847 | LR: 3.98e-04 | 3200.1ms/step | 18.7 samples/s +Step 15620/50462 | Loss: 2.6881 | LR: 3.98e-04 | 3200.0ms/step | 18.8 samples/s +Step 15630/50462 | Loss: 2.6913 | LR: 3.97e-04 | 3199.9ms/step | 18.8 samples/s +Step 15640/50462 | Loss: 2.6476 | LR: 3.97e-04 | 3199.8ms/step | 18.8 samples/s +Step 15650/50462 | Loss: 2.6947 | LR: 3.97e-04 | 3200.0ms/step | 18.7 samples/s +Step 15660/50462 | Loss: 2.6871 | LR: 3.97e-04 | 3199.9ms/step | 18.8 samples/s +Step 15670/50462 | Loss: 2.6656 | LR: 3.97e-04 | 3200.0ms/step | 18.8 samples/s +Step 15680/50462 | Loss: 2.6823 | LR: 3.97e-04 | 3200.1ms/step | 18.7 samples/s +Step 15690/50462 | Loss: 2.6649 | LR: 3.97e-04 | 3200.0ms/step | 18.8 samples/s +Step 15700/50462 | Loss: 2.6843 | LR: 3.96e-04 | 3200.1ms/step | 18.7 samples/s +Step 15710/50462 | Loss: 2.6912 | LR: 3.96e-04 | 3200.0ms/step | 18.8 samples/s +Step 15720/50462 | Loss: 2.6486 | LR: 3.96e-04 | 3198.0ms/step | 18.8 samples/s +Step 15730/50462 | Loss: 2.6599 | LR: 3.96e-04 | 3200.1ms/step | 18.7 samples/s +Step 15740/50462 | Loss: 2.6792 | LR: 3.96e-04 | 3199.7ms/step | 18.8 samples/s +Step 15750/50462 | Loss: 2.6753 | LR: 3.96e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 15750: Loss = 2.6936, Perplexity = 14.78 +Step 15760/50462 | Loss: 2.6930 | LR: 3.96e-04 | 4925.2ms/step | 12.2 samples/s +Step 15770/50462 | Loss: 2.6915 | LR: 3.96e-04 | 3200.8ms/step | 18.7 samples/s +Step 15780/50462 | Loss: 2.6467 | LR: 3.95e-04 | 3251.7ms/step | 18.5 samples/s +Step 15790/50462 | Loss: 2.6660 | LR: 3.95e-04 | 3196.3ms/step | 18.8 samples/s +Step 15800/50462 | Loss: 2.6672 | LR: 3.95e-04 | 3195.9ms/step | 18.8 samples/s +Step 15810/50462 | Loss: 2.7018 | LR: 3.95e-04 | 3197.5ms/step | 18.8 samples/s +Step 15820/50462 | Loss: 2.6794 | LR: 3.95e-04 | 3192.5ms/step | 18.8 samples/s +Step 15830/50462 | Loss: 2.6752 | LR: 3.95e-04 | 3200.0ms/step | 18.8 samples/s +Step 15840/50462 | Loss: 2.6695 | LR: 3.95e-04 | 3199.6ms/step | 18.8 samples/s +Step 15850/50462 | Loss: 2.6490 | LR: 3.95e-04 | 3200.5ms/step | 18.7 samples/s +Step 15860/50462 | Loss: 2.6822 | LR: 3.94e-04 | 3198.1ms/step | 18.8 samples/s +Step 15870/50462 | Loss: 2.6540 | LR: 3.94e-04 | 3199.7ms/step | 18.8 samples/s +Step 15880/50462 | Loss: 2.6708 | LR: 3.94e-04 | 3200.2ms/step | 18.7 samples/s +Step 15890/50462 | Loss: 2.6475 | LR: 3.94e-04 | 3199.8ms/step | 18.8 samples/s +Step 15900/50462 | Loss: 2.6548 | LR: 3.94e-04 | 3256.1ms/step | 18.4 samples/s +Step 15910/50462 | Loss: 2.6302 | LR: 3.94e-04 | 3198.1ms/step | 18.8 samples/s +Step 15920/50462 | Loss: 2.6833 | LR: 3.94e-04 | 3200.0ms/step | 18.8 samples/s +Step 15930/50462 | Loss: 2.6660 | LR: 3.94e-04 | 3199.9ms/step | 18.8 samples/s +Step 15940/50462 | Loss: 2.6985 | LR: 3.93e-04 | 3200.2ms/step | 18.7 samples/s +Step 15950/50462 | Loss: 2.6744 | LR: 3.93e-04 | 3199.8ms/step | 18.8 samples/s +Step 15960/50462 | Loss: 2.6925 | LR: 3.93e-04 | 3200.0ms/step | 18.7 samples/s +Step 15970/50462 | Loss: 2.6786 | LR: 3.93e-04 | 3200.3ms/step | 18.7 samples/s +Step 15980/50462 | Loss: 2.6894 | LR: 3.93e-04 | 3199.8ms/step | 18.8 samples/s +Step 15990/50462 | Loss: 2.6629 | LR: 3.93e-04 | 3199.9ms/step | 18.8 samples/s +Step 16000/50462 | Loss: 2.6895 | LR: 3.93e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 16000: Loss = 2.6900, Perplexity = 14.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-16000 +Step 16010/50462 | Loss: 2.6792 | LR: 3.92e-04 | 5773.6ms/step | 10.4 samples/s +Step 16020/50462 | Loss: 2.6851 | LR: 3.92e-04 | 3177.0ms/step | 18.9 samples/s +Step 16030/50462 | Loss: 2.7002 | LR: 3.92e-04 | 3180.8ms/step | 18.9 samples/s +Step 16040/50462 | Loss: 2.6781 | LR: 3.92e-04 | 3192.6ms/step | 18.8 samples/s +Step 16050/50462 | Loss: 2.7030 | LR: 3.92e-04 | 3199.9ms/step | 18.8 samples/s +Step 16060/50462 | Loss: 2.6768 | LR: 3.92e-04 | 3200.1ms/step | 18.7 samples/s +Step 16070/50462 | Loss: 2.6672 | LR: 3.92e-04 | 3199.9ms/step | 18.8 samples/s +Step 16080/50462 | Loss: 2.6826 | LR: 3.92e-04 | 3197.4ms/step | 18.8 samples/s +Step 16090/50462 | Loss: 2.6762 | LR: 3.91e-04 | 3200.5ms/step | 18.7 samples/s +Step 16100/50462 | Loss: 2.7091 | LR: 3.91e-04 | 3252.0ms/step | 18.5 samples/s +Step 16110/50462 | Loss: 2.6851 | LR: 3.91e-04 | 3200.1ms/step | 18.7 samples/s +Step 16120/50462 | Loss: 2.6774 | LR: 3.91e-04 | 3255.8ms/step | 18.4 samples/s +Step 16130/50462 | Loss: 2.6707 | LR: 3.91e-04 | 3198.6ms/step | 18.8 samples/s +Step 16140/50462 | Loss: 2.6663 | LR: 3.91e-04 | 3199.7ms/step | 18.8 samples/s +Step 16150/50462 | Loss: 2.6854 | LR: 3.91e-04 | 3199.7ms/step | 18.8 samples/s +Step 16160/50462 | Loss: 2.6702 | LR: 3.91e-04 | 3199.8ms/step | 18.8 samples/s +Step 16170/50462 | Loss: 2.6917 | LR: 3.90e-04 | 3198.7ms/step | 18.8 samples/s +Step 16180/50462 | Loss: 2.6746 | LR: 3.90e-04 | 3199.3ms/step | 18.8 samples/s +Step 16190/50462 | Loss: 2.6752 | LR: 3.90e-04 | 3200.3ms/step | 18.7 samples/s +Step 16200/50462 | Loss: 2.6693 | LR: 3.90e-04 | 3198.2ms/step | 18.8 samples/s +Step 16210/50462 | Loss: 2.6884 | LR: 3.90e-04 | 3200.1ms/step | 18.7 samples/s +Step 16220/50462 | Loss: 2.6731 | LR: 3.90e-04 | 3199.8ms/step | 18.8 samples/s +Step 16230/50462 | Loss: 2.6742 | LR: 3.90e-04 | 3197.9ms/step | 18.8 samples/s +Step 16240/50462 | Loss: 2.6757 | LR: 3.89e-04 | 3252.0ms/step | 18.5 samples/s +Step 16250/50462 | Loss: 2.6748 | LR: 3.89e-04 | 3191.5ms/step | 18.8 samples/s +Validation at step 16250: Loss = 2.6878, Perplexity = 14.70 +Step 16260/50462 | Loss: 2.6905 | LR: 3.89e-04 | 4870.2ms/step | 12.3 samples/s +Step 16270/50462 | Loss: 2.6796 | LR: 3.89e-04 | 3198.6ms/step | 18.8 samples/s +Step 16280/50462 | Loss: 2.6894 | LR: 3.89e-04 | 3199.7ms/step | 18.8 samples/s +Step 16290/50462 | Loss: 2.6855 | LR: 3.89e-04 | 3199.8ms/step | 18.8 samples/s +Step 16300/50462 | Loss: 2.6920 | LR: 3.89e-04 | 3199.3ms/step | 18.8 samples/s +Step 16310/50462 | Loss: 2.6842 | LR: 3.89e-04 | 3199.0ms/step | 18.8 samples/s +Step 16320/50462 | Loss: 2.6641 | LR: 3.88e-04 | 3200.2ms/step | 18.7 samples/s +Step 16330/50462 | Loss: 2.7320 | LR: 3.88e-04 | 3197.7ms/step | 18.8 samples/s +Step 16340/50462 | Loss: 2.6902 | LR: 3.88e-04 | 3200.1ms/step | 18.7 samples/s +Step 16350/50462 | Loss: 2.6799 | LR: 3.88e-04 | 3200.0ms/step | 18.7 samples/s +Step 16360/50462 | Loss: 2.6765 | LR: 3.88e-04 | 3197.9ms/step | 18.8 samples/s +Step 16370/50462 | Loss: 2.6395 | LR: 3.88e-04 | 3199.7ms/step | 18.8 samples/s +Step 16380/50462 | Loss: 2.6634 | LR: 3.88e-04 | 3200.5ms/step | 18.7 samples/s +Step 16390/50462 | Loss: 2.6807 | LR: 3.88e-04 | 3199.8ms/step | 18.8 samples/s +Step 16400/50462 | Loss: 2.6608 | LR: 3.87e-04 | 3200.0ms/step | 18.7 samples/s +Step 16410/50462 | Loss: 2.6524 | LR: 3.87e-04 | 3199.8ms/step | 18.8 samples/s +Step 16420/50462 | Loss: 2.6559 | LR: 3.87e-04 | 3200.3ms/step | 18.7 samples/s +Step 16430/50462 | Loss: 2.6692 | LR: 3.87e-04 | 3199.3ms/step | 18.8 samples/s +Step 16440/50462 | Loss: 2.7056 | LR: 3.87e-04 | 3251.9ms/step | 18.5 samples/s +Step 16450/50462 | Loss: 2.6727 | LR: 3.87e-04 | 3200.8ms/step | 18.7 samples/s +Step 16460/50462 | Loss: 2.6963 | LR: 3.87e-04 | 3252.9ms/step | 18.4 samples/s +Step 16470/50462 | Loss: 2.6780 | LR: 3.86e-04 | 3199.1ms/step | 18.8 samples/s +Step 16480/50462 | Loss: 2.6934 | LR: 3.86e-04 | 3199.6ms/step | 18.8 samples/s +Step 16490/50462 | Loss: 2.6480 | LR: 3.86e-04 | 3200.4ms/step | 18.7 samples/s +Step 16500/50462 | Loss: 2.6572 | LR: 3.86e-04 | 3200.0ms/step | 18.8 samples/s +Validation at step 16500: Loss = 2.6839, Perplexity = 14.64 +Step 16510/50462 | Loss: 2.6431 | LR: 3.86e-04 | 4874.0ms/step | 12.3 samples/s +Step 16520/50462 | Loss: 2.6729 | LR: 3.86e-04 | 3199.4ms/step | 18.8 samples/s +Step 16530/50462 | Loss: 2.6314 | LR: 3.86e-04 | 3200.4ms/step | 18.7 samples/s +Step 16540/50462 | Loss: 2.6516 | LR: 3.86e-04 | 3200.3ms/step | 18.7 samples/s +Step 16550/50462 | Loss: 2.6730 | LR: 3.85e-04 | 3199.7ms/step | 18.8 samples/s +Step 16560/50462 | Loss: 2.6807 | LR: 3.85e-04 | 3200.2ms/step | 18.7 samples/s +Step 16570/50462 | Loss: 2.6638 | LR: 3.85e-04 | 3199.9ms/step | 18.8 samples/s +Step 16580/50462 | Loss: 2.6796 | LR: 3.85e-04 | 3252.0ms/step | 18.5 samples/s +Step 16590/50462 | Loss: 2.6697 | LR: 3.85e-04 | 3192.2ms/step | 18.8 samples/s +Step 16600/50462 | Loss: 2.6891 | LR: 3.85e-04 | 3198.6ms/step | 18.8 samples/s +Step 16610/50462 | Loss: 2.6704 | LR: 3.85e-04 | 3199.4ms/step | 18.8 samples/s +Step 16620/50462 | Loss: 2.6866 | LR: 3.84e-04 | 3199.9ms/step | 18.8 samples/s +Step 16630/50462 | Loss: 2.6661 | LR: 3.84e-04 | 3199.6ms/step | 18.8 samples/s +Step 16640/50462 | Loss: 2.6421 | LR: 3.84e-04 | 3200.4ms/step | 18.7 samples/s +Step 16650/50462 | Loss: 2.6712 | LR: 3.84e-04 | 3199.9ms/step | 18.8 samples/s +Step 16660/50462 | Loss: 2.6722 | LR: 3.84e-04 | 3200.2ms/step | 18.7 samples/s +Step 16670/50462 | Loss: 2.6594 | LR: 3.84e-04 | 3199.9ms/step | 18.8 samples/s +Step 16680/50462 | Loss: 2.6313 | LR: 3.84e-04 | 3200.0ms/step | 18.7 samples/s +Step 16690/50462 | Loss: 2.6572 | LR: 3.84e-04 | 3200.2ms/step | 18.7 samples/s +Step 16700/50462 | Loss: 2.6857 | LR: 3.83e-04 | 3199.3ms/step | 18.8 samples/s +Step 16710/50462 | Loss: 2.6983 | LR: 3.83e-04 | 3200.7ms/step | 18.7 samples/s +Step 16720/50462 | Loss: 2.6739 | LR: 3.83e-04 | 3200.1ms/step | 18.7 samples/s +Step 16730/50462 | Loss: 2.6605 | LR: 3.83e-04 | 3200.0ms/step | 18.7 samples/s +Step 16740/50462 | Loss: 2.6478 | LR: 3.83e-04 | 3199.9ms/step | 18.8 samples/s +Step 16750/50462 | Loss: 2.7070 | LR: 3.83e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 16750: Loss = 2.6807, Perplexity = 14.60 +Step 16760/50462 | Loss: 2.6700 | LR: 3.83e-04 | 4876.2ms/step | 12.3 samples/s +Step 16770/50462 | Loss: 2.6706 | LR: 3.82e-04 | 3200.0ms/step | 18.7 samples/s +Step 16780/50462 | Loss: 2.6522 | LR: 3.82e-04 | 3253.9ms/step | 18.4 samples/s +Step 16790/50462 | Loss: 2.7025 | LR: 3.82e-04 | 3195.9ms/step | 18.8 samples/s +Step 16800/50462 | Loss: 2.6763 | LR: 3.82e-04 | 3199.5ms/step | 18.8 samples/s +Step 16810/50462 | Loss: 2.6914 | LR: 3.82e-04 | 3248.1ms/step | 18.5 samples/s +Step 16820/50462 | Loss: 2.6760 | LR: 3.82e-04 | 3196.2ms/step | 18.8 samples/s +Step 16830/50462 | Loss: 2.6736 | LR: 3.82e-04 | 3185.1ms/step | 18.8 samples/s +Step 16840/50462 | Loss: 2.6780 | LR: 3.82e-04 | 3189.0ms/step | 18.8 samples/s +Step 16850/50462 | Loss: 2.6585 | LR: 3.81e-04 | 3191.0ms/step | 18.8 samples/s +Step 16860/50462 | Loss: 2.6609 | LR: 3.81e-04 | 3194.4ms/step | 18.8 samples/s +Step 16870/50462 | Loss: 2.6558 | LR: 3.81e-04 | 3192.4ms/step | 18.8 samples/s +Step 16880/50462 | Loss: 2.6506 | LR: 3.81e-04 | 3194.4ms/step | 18.8 samples/s +Step 16890/50462 | Loss: 2.6526 | LR: 3.81e-04 | 3192.8ms/step | 18.8 samples/s +Step 16900/50462 | Loss: 2.6754 | LR: 3.81e-04 | 3194.6ms/step | 18.8 samples/s +Step 16910/50462 | Loss: 2.6666 | LR: 3.81e-04 | 3196.6ms/step | 18.8 samples/s +Step 16920/50462 | Loss: 2.6857 | LR: 3.80e-04 | 3253.9ms/step | 18.4 samples/s +Step 16930/50462 | Loss: 2.6612 | LR: 3.80e-04 | 3200.3ms/step | 18.7 samples/s +Step 16940/50462 | Loss: 2.6727 | LR: 3.80e-04 | 3199.9ms/step | 18.8 samples/s +Step 16950/50462 | Loss: 2.6617 | LR: 3.80e-04 | 3199.7ms/step | 18.8 samples/s +Step 16960/50462 | Loss: 2.6828 | LR: 3.80e-04 | 3200.2ms/step | 18.7 samples/s +Step 16970/50462 | Loss: 2.6551 | LR: 3.80e-04 | 3199.8ms/step | 18.8 samples/s +Step 16980/50462 | Loss: 2.6550 | LR: 3.80e-04 | 3200.1ms/step | 18.7 samples/s +Step 16990/50462 | Loss: 2.6931 | LR: 3.79e-04 | 3199.7ms/step | 18.8 samples/s +Step 17000/50462 | Loss: 2.6630 | LR: 3.79e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 17000: Loss = 2.6781, Perplexity = 14.56 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-17000 +Step 17010/50462 | Loss: 2.6994 | LR: 3.79e-04 | 5749.1ms/step | 10.4 samples/s +Step 17020/50462 | Loss: 2.6346 | LR: 3.79e-04 | 3189.0ms/step | 18.8 samples/s +Step 17030/50462 | Loss: 2.6568 | LR: 3.79e-04 | 3198.1ms/step | 18.8 samples/s +Step 17040/50462 | Loss: 2.6935 | LR: 3.79e-04 | 3198.1ms/step | 18.8 samples/s +Step 17050/50462 | Loss: 2.6402 | LR: 3.79e-04 | 3197.9ms/step | 18.8 samples/s +Step 17060/50462 | Loss: 2.6468 | LR: 3.79e-04 | 3200.1ms/step | 18.7 samples/s +Step 17070/50462 | Loss: 2.6781 | LR: 3.78e-04 | 3199.8ms/step | 18.8 samples/s +Step 17080/50462 | Loss: 2.6834 | LR: 3.78e-04 | 3200.2ms/step | 18.7 samples/s +Step 17090/50462 | Loss: 2.6875 | LR: 3.78e-04 | 3198.1ms/step | 18.8 samples/s +Step 17100/50462 | Loss: 2.6948 | LR: 3.78e-04 | 3197.9ms/step | 18.8 samples/s +Step 17110/50462 | Loss: 2.6848 | LR: 3.78e-04 | 3199.9ms/step | 18.8 samples/s +Step 17120/50462 | Loss: 2.6597 | LR: 3.78e-04 | 3241.7ms/step | 18.5 samples/s +Step 17130/50462 | Loss: 2.6598 | LR: 3.78e-04 | 3186.6ms/step | 18.8 samples/s +Step 17140/50462 | Loss: 2.6781 | LR: 3.77e-04 | 3197.9ms/step | 18.8 samples/s +Step 17150/50462 | Loss: 2.6641 | LR: 3.77e-04 | 3253.8ms/step | 18.4 samples/s +Step 17160/50462 | Loss: 2.6386 | LR: 3.77e-04 | 3200.3ms/step | 18.7 samples/s +Step 17170/50462 | Loss: 2.6631 | LR: 3.77e-04 | 3199.9ms/step | 18.8 samples/s +Step 17180/50462 | Loss: 2.6339 | LR: 3.77e-04 | 3199.9ms/step | 18.8 samples/s +Step 17190/50462 | Loss: 2.6772 | LR: 3.77e-04 | 3200.0ms/step | 18.7 samples/s +Step 17200/50462 | Loss: 2.6505 | LR: 3.77e-04 | 3200.1ms/step | 18.7 samples/s +Step 17210/50462 | Loss: 2.6904 | LR: 3.77e-04 | 3200.1ms/step | 18.7 samples/s +Step 17220/50462 | Loss: 2.6914 | LR: 3.76e-04 | 3200.0ms/step | 18.7 samples/s +Step 17230/50462 | Loss: 2.6930 | LR: 3.76e-04 | 3199.9ms/step | 18.8 samples/s +Step 17240/50462 | Loss: 2.6600 | LR: 3.76e-04 | 3200.0ms/step | 18.8 samples/s +Step 17250/50462 | Loss: 2.6607 | LR: 3.76e-04 | 3254.0ms/step | 18.4 samples/s +Validation at step 17250: Loss = 2.6749, Perplexity = 14.51 +Step 17260/50462 | Loss: 2.6814 | LR: 3.76e-04 | 4875.9ms/step | 12.3 samples/s +Step 17270/50462 | Loss: 2.6280 | LR: 3.76e-04 | 3199.6ms/step | 18.8 samples/s +Step 17280/50462 | Loss: 2.6491 | LR: 3.76e-04 | 3200.4ms/step | 18.7 samples/s +Step 17290/50462 | Loss: 2.6445 | LR: 3.75e-04 | 3199.8ms/step | 18.8 samples/s +Step 17300/50462 | Loss: 2.6461 | LR: 3.75e-04 | 3200.2ms/step | 18.7 samples/s +Step 17310/50462 | Loss: 2.6432 | LR: 3.75e-04 | 3199.9ms/step | 18.8 samples/s +Step 17320/50462 | Loss: 2.6534 | LR: 3.75e-04 | 3200.1ms/step | 18.7 samples/s +Step 17330/50462 | Loss: 2.6697 | LR: 3.75e-04 | 3200.0ms/step | 18.7 samples/s +Step 17340/50462 | Loss: 2.6520 | LR: 3.75e-04 | 3199.9ms/step | 18.8 samples/s +Step 17350/50462 | Loss: 2.6782 | LR: 3.75e-04 | 3202.8ms/step | 18.7 samples/s +Step 17360/50462 | Loss: 2.6742 | LR: 3.74e-04 | 3201.4ms/step | 18.7 samples/s +Step 17370/50462 | Loss: 2.6531 | LR: 3.74e-04 | 3200.0ms/step | 18.7 samples/s +Step 17380/50462 | Loss: 2.6575 | LR: 3.74e-04 | 3199.8ms/step | 18.8 samples/s +Step 17390/50462 | Loss: 2.6647 | LR: 3.74e-04 | 3197.5ms/step | 18.8 samples/s +Step 17400/50462 | Loss: 2.6530 | LR: 3.74e-04 | 3198.6ms/step | 18.8 samples/s +Step 17410/50462 | Loss: 2.6581 | LR: 3.74e-04 | 3200.2ms/step | 18.7 samples/s +Step 17420/50462 | Loss: 2.6540 | LR: 3.74e-04 | 3199.9ms/step | 18.8 samples/s +Step 17430/50462 | Loss: 2.6473 | LR: 3.74e-04 | 3199.9ms/step | 18.8 samples/s +Step 17440/50462 | Loss: 2.6506 | LR: 3.73e-04 | 3200.1ms/step | 18.7 samples/s +Step 17450/50462 | Loss: 2.6700 | LR: 3.73e-04 | 3200.1ms/step | 18.7 samples/s +Step 17460/50462 | Loss: 2.6697 | LR: 3.73e-04 | 3251.9ms/step | 18.5 samples/s +Step 17470/50462 | Loss: 2.6465 | LR: 3.73e-04 | 3200.1ms/step | 18.7 samples/s +Step 17480/50462 | Loss: 2.6616 | LR: 3.73e-04 | 3200.0ms/step | 18.7 samples/s +Step 17490/50462 | Loss: 2.6864 | LR: 3.73e-04 | 3251.9ms/step | 18.5 samples/s +Step 17500/50462 | Loss: 2.6398 | LR: 3.73e-04 | 3199.7ms/step | 18.8 samples/s +Validation at step 17500: Loss = 2.6728, Perplexity = 14.48 +Step 17510/50462 | Loss: 2.6528 | LR: 3.72e-04 | 4872.2ms/step | 12.3 samples/s +Step 17520/50462 | Loss: 2.6497 | LR: 3.72e-04 | 3200.2ms/step | 18.7 samples/s +Step 17530/50462 | Loss: 2.6593 | LR: 3.72e-04 | 3199.9ms/step | 18.8 samples/s +Step 17540/50462 | Loss: 2.6420 | LR: 3.72e-04 | 3201.9ms/step | 18.7 samples/s +Step 17550/50462 | Loss: 2.6598 | LR: 3.72e-04 | 3200.1ms/step | 18.7 samples/s +Step 17560/50462 | Loss: 2.6672 | LR: 3.72e-04 | 3200.2ms/step | 18.7 samples/s +Step 17570/50462 | Loss: 2.6756 | LR: 3.72e-04 | 3199.7ms/step | 18.8 samples/s +Step 17580/50462 | Loss: 2.6642 | LR: 3.71e-04 | 3199.8ms/step | 18.8 samples/s +Step 17590/50462 | Loss: 2.6262 | LR: 3.71e-04 | 3254.5ms/step | 18.4 samples/s +Step 17600/50462 | Loss: 2.6616 | LR: 3.71e-04 | 3199.8ms/step | 18.8 samples/s +Step 17610/50462 | Loss: 2.6348 | LR: 3.71e-04 | 3200.1ms/step | 18.7 samples/s +Step 17620/50462 | Loss: 2.6505 | LR: 3.71e-04 | 3202.1ms/step | 18.7 samples/s +Step 17630/50462 | Loss: 2.6377 | LR: 3.71e-04 | 3197.7ms/step | 18.8 samples/s +Step 17640/50462 | Loss: 2.6465 | LR: 3.71e-04 | 3200.0ms/step | 18.7 samples/s +Step 17650/50462 | Loss: 2.6481 | LR: 3.70e-04 | 3200.1ms/step | 18.7 samples/s +Step 17660/50462 | Loss: 2.6501 | LR: 3.70e-04 | 3200.0ms/step | 18.8 samples/s +Step 17670/50462 | Loss: 2.6726 | LR: 3.70e-04 | 3199.6ms/step | 18.8 samples/s +Step 17680/50462 | Loss: 2.6531 | LR: 3.70e-04 | 3200.2ms/step | 18.7 samples/s +Step 17690/50462 | Loss: 2.6616 | LR: 3.70e-04 | 3184.1ms/step | 18.8 samples/s +Step 17700/50462 | Loss: 2.6294 | LR: 3.70e-04 | 3194.2ms/step | 18.8 samples/s +Step 17710/50462 | Loss: 2.6835 | LR: 3.70e-04 | 3199.8ms/step | 18.8 samples/s +Step 17720/50462 | Loss: 2.6330 | LR: 3.69e-04 | 3200.0ms/step | 18.7 samples/s +Step 17730/50462 | Loss: 2.6766 | LR: 3.69e-04 | 3200.1ms/step | 18.7 samples/s +Step 17740/50462 | Loss: 2.6664 | LR: 3.69e-04 | 3200.0ms/step | 18.8 samples/s +Step 17750/50462 | Loss: 2.6503 | LR: 3.69e-04 | 3200.0ms/step | 18.8 samples/s +Validation at step 17750: Loss = 2.6700, Perplexity = 14.44 +Step 17760/50462 | Loss: 2.6470 | LR: 3.69e-04 | 4876.0ms/step | 12.3 samples/s +Step 17770/50462 | Loss: 2.6551 | LR: 3.69e-04 | 3199.9ms/step | 18.8 samples/s +Step 17780/50462 | Loss: 2.6914 | LR: 3.69e-04 | 3199.9ms/step | 18.8 samples/s +Step 17790/50462 | Loss: 2.6522 | LR: 3.69e-04 | 3200.9ms/step | 18.7 samples/s +Step 17800/50462 | Loss: 2.6380 | LR: 3.68e-04 | 3251.3ms/step | 18.5 samples/s +Step 17810/50462 | Loss: 2.6606 | LR: 3.68e-04 | 3199.9ms/step | 18.8 samples/s +Step 17820/50462 | Loss: 2.6662 | LR: 3.68e-04 | 3200.0ms/step | 18.8 samples/s +Step 17830/50462 | Loss: 2.6711 | LR: 3.68e-04 | 3253.9ms/step | 18.4 samples/s +Step 17840/50462 | Loss: 2.6719 | LR: 3.68e-04 | 3200.2ms/step | 18.7 samples/s +Step 17850/50462 | Loss: 2.6411 | LR: 3.68e-04 | 3199.9ms/step | 18.8 samples/s +Step 17860/50462 | Loss: 2.6367 | LR: 3.68e-04 | 3200.1ms/step | 18.7 samples/s +Step 17870/50462 | Loss: 2.6541 | LR: 3.67e-04 | 3202.2ms/step | 18.7 samples/s +Step 17880/50462 | Loss: 2.6516 | LR: 3.67e-04 | 3199.5ms/step | 18.8 samples/s +Step 17890/50462 | Loss: 2.6616 | LR: 3.67e-04 | 3199.7ms/step | 18.8 samples/s +Step 17900/50462 | Loss: 2.6789 | LR: 3.67e-04 | 3200.5ms/step | 18.7 samples/s +Step 17910/50462 | Loss: 2.6387 | LR: 3.67e-04 | 3200.0ms/step | 18.8 samples/s +Step 17920/50462 | Loss: 2.6695 | LR: 3.67e-04 | 3200.1ms/step | 18.7 samples/s +Step 17930/50462 | Loss: 2.6779 | LR: 3.67e-04 | 3253.8ms/step | 18.4 samples/s +Step 17940/50462 | Loss: 2.6477 | LR: 3.66e-04 | 3200.0ms/step | 18.7 samples/s +Step 17950/50462 | Loss: 2.6501 | LR: 3.66e-04 | 3200.0ms/step | 18.7 samples/s +Step 17960/50462 | Loss: 2.6455 | LR: 3.66e-04 | 3200.1ms/step | 18.7 samples/s +Step 17970/50462 | Loss: 2.6525 | LR: 3.66e-04 | 3200.0ms/step | 18.8 samples/s +Step 17980/50462 | Loss: 2.6529 | LR: 3.66e-04 | 3200.1ms/step | 18.7 samples/s +Step 17990/50462 | Loss: 2.7010 | LR: 3.66e-04 | 3200.1ms/step | 18.7 samples/s +Step 18000/50462 | Loss: 2.6833 | LR: 3.66e-04 | 3199.7ms/step | 18.8 samples/s +Validation at step 18000: Loss = 2.6673, Perplexity = 14.40 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-18000 +Step 18010/50462 | Loss: 2.6408 | LR: 3.65e-04 | 5783.8ms/step | 10.4 samples/s +Step 18020/50462 | Loss: 2.6510 | LR: 3.65e-04 | 3194.1ms/step | 18.8 samples/s +Step 18030/50462 | Loss: 2.6331 | LR: 3.65e-04 | 3200.1ms/step | 18.7 samples/s +Step 18040/50462 | Loss: 2.6573 | LR: 3.65e-04 | 3200.0ms/step | 18.8 samples/s +Step 18050/50462 | Loss: 2.6447 | LR: 3.65e-04 | 3199.9ms/step | 18.8 samples/s +Step 18060/50462 | Loss: 2.6883 | LR: 3.65e-04 | 3200.1ms/step | 18.7 samples/s +Step 18070/50462 | Loss: 2.6772 | LR: 3.65e-04 | 3199.8ms/step | 18.8 samples/s +Step 18080/50462 | Loss: 2.6599 | LR: 3.64e-04 | 3200.1ms/step | 18.7 samples/s +Step 18090/50462 | Loss: 2.6209 | LR: 3.64e-04 | 3200.0ms/step | 18.7 samples/s +Step 18100/50462 | Loss: 2.6397 | LR: 3.64e-04 | 3199.9ms/step | 18.8 samples/s +Step 18110/50462 | Loss: 2.6447 | LR: 3.64e-04 | 3200.0ms/step | 18.7 samples/s +Step 18120/50462 | Loss: 2.6247 | LR: 3.64e-04 | 3198.0ms/step | 18.8 samples/s +Step 18130/50462 | Loss: 2.6225 | LR: 3.64e-04 | 3200.0ms/step | 18.8 samples/s +Step 18140/50462 | Loss: 2.6596 | LR: 3.64e-04 | 3252.1ms/step | 18.4 samples/s +Step 18150/50462 | Loss: 2.6448 | LR: 3.63e-04 | 3199.9ms/step | 18.8 samples/s +Step 18160/50462 | Loss: 2.6580 | LR: 3.63e-04 | 3200.0ms/step | 18.7 samples/s +Step 18170/50462 | Loss: 2.6711 | LR: 3.63e-04 | 3251.8ms/step | 18.5 samples/s +Step 18180/50462 | Loss: 2.6752 | LR: 3.63e-04 | 3187.6ms/step | 18.8 samples/s +Step 18190/50462 | Loss: 2.6455 | LR: 3.63e-04 | 3200.7ms/step | 18.7 samples/s +Step 18200/50462 | Loss: 2.6396 | LR: 3.63e-04 | 3199.9ms/step | 18.8 samples/s +Step 18210/50462 | Loss: 2.6417 | LR: 3.63e-04 | 3200.2ms/step | 18.7 samples/s +Step 18220/50462 | Loss: 2.6450 | LR: 3.62e-04 | 3200.0ms/step | 18.7 samples/s +Step 18230/50462 | Loss: 2.6673 | LR: 3.62e-04 | 3199.9ms/step | 18.8 samples/s +Step 18240/50462 | Loss: 2.6213 | LR: 3.62e-04 | 3199.8ms/step | 18.8 samples/s +Step 18250/50462 | Loss: 2.6508 | LR: 3.62e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 18250: Loss = 2.6647, Perplexity = 14.36 +Step 18260/50462 | Loss: 2.6507 | LR: 3.62e-04 | 4924.3ms/step | 12.2 samples/s +Step 18270/50462 | Loss: 2.6570 | LR: 3.62e-04 | 3190.4ms/step | 18.8 samples/s +Step 18280/50462 | Loss: 2.6153 | LR: 3.62e-04 | 3199.3ms/step | 18.8 samples/s +Step 18290/50462 | Loss: 2.6492 | LR: 3.62e-04 | 3200.2ms/step | 18.7 samples/s +Step 18300/50462 | Loss: 2.6672 | LR: 3.61e-04 | 3199.8ms/step | 18.8 samples/s +Step 18310/50462 | Loss: 2.6487 | LR: 3.61e-04 | 3200.0ms/step | 18.8 samples/s +Step 18320/50462 | Loss: 2.6465 | LR: 3.61e-04 | 3200.0ms/step | 18.7 samples/s +Step 18330/50462 | Loss: 2.6953 | LR: 3.61e-04 | 3200.0ms/step | 18.8 samples/s +Step 18340/50462 | Loss: 2.6427 | LR: 3.61e-04 | 3196.0ms/step | 18.8 samples/s +Step 18350/50462 | Loss: 2.6854 | LR: 3.61e-04 | 3200.0ms/step | 18.8 samples/s +Step 18360/50462 | Loss: 2.6353 | LR: 3.61e-04 | 3199.9ms/step | 18.8 samples/s +Step 18370/50462 | Loss: 2.6584 | LR: 3.60e-04 | 3200.2ms/step | 18.7 samples/s +Step 18380/50462 | Loss: 2.6175 | LR: 3.60e-04 | 3200.0ms/step | 18.8 samples/s +Step 18390/50462 | Loss: 2.6548 | LR: 3.60e-04 | 3201.9ms/step | 18.7 samples/s +Step 18400/50462 | Loss: 2.6162 | LR: 3.60e-04 | 3200.1ms/step | 18.7 samples/s +Step 18410/50462 | Loss: 2.6534 | LR: 3.60e-04 | 3200.2ms/step | 18.7 samples/s +Step 18420/50462 | Loss: 2.6589 | LR: 3.60e-04 | 3199.8ms/step | 18.8 samples/s +Step 18430/50462 | Loss: 2.6802 | LR: 3.60e-04 | 3199.8ms/step | 18.8 samples/s +Step 18440/50462 | Loss: 2.6441 | LR: 3.59e-04 | 3200.2ms/step | 18.7 samples/s +Step 18450/50462 | Loss: 2.6512 | LR: 3.59e-04 | 3200.1ms/step | 18.7 samples/s +Step 18460/50462 | Loss: 2.6508 | LR: 3.59e-04 | 3201.7ms/step | 18.7 samples/s +Step 18470/50462 | Loss: 2.6503 | LR: 3.59e-04 | 3200.1ms/step | 18.7 samples/s +Step 18480/50462 | Loss: 2.6411 | LR: 3.59e-04 | 3251.7ms/step | 18.5 samples/s +Step 18490/50462 | Loss: 2.6433 | LR: 3.59e-04 | 3200.4ms/step | 18.7 samples/s +Step 18500/50462 | Loss: 2.6571 | LR: 3.59e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 18500: Loss = 2.6615, Perplexity = 14.32 +Step 18510/50462 | Loss: 2.6360 | LR: 3.58e-04 | 4930.2ms/step | 12.2 samples/s +Step 18520/50462 | Loss: 2.6486 | LR: 3.58e-04 | 3200.0ms/step | 18.7 samples/s +Step 18530/50462 | Loss: 2.6656 | LR: 3.58e-04 | 3199.9ms/step | 18.8 samples/s +Step 18540/50462 | Loss: 2.6359 | LR: 3.58e-04 | 3199.9ms/step | 18.8 samples/s +Step 18550/50462 | Loss: 2.6697 | LR: 3.58e-04 | 3199.9ms/step | 18.8 samples/s +Step 18560/50462 | Loss: 2.6332 | LR: 3.58e-04 | 3200.1ms/step | 18.7 samples/s +Step 18570/50462 | Loss: 2.6394 | LR: 3.58e-04 | 3200.1ms/step | 18.7 samples/s +Step 18580/50462 | Loss: 2.6749 | LR: 3.57e-04 | 3200.0ms/step | 18.7 samples/s +Step 18590/50462 | Loss: 2.6518 | LR: 3.57e-04 | 3199.9ms/step | 18.8 samples/s +Step 18600/50462 | Loss: 2.6242 | LR: 3.57e-04 | 3256.1ms/step | 18.4 samples/s +Step 18610/50462 | Loss: 2.6530 | LR: 3.57e-04 | 3200.3ms/step | 18.7 samples/s +Step 18620/50462 | Loss: 2.6556 | LR: 3.57e-04 | 3199.6ms/step | 18.8 samples/s +Step 18630/50462 | Loss: 2.6668 | LR: 3.57e-04 | 3199.9ms/step | 18.8 samples/s +Step 18640/50462 | Loss: 2.6594 | LR: 3.57e-04 | 3200.2ms/step | 18.7 samples/s +Step 18650/50462 | Loss: 2.6662 | LR: 3.56e-04 | 3199.7ms/step | 18.8 samples/s +Step 18660/50462 | Loss: 2.6394 | LR: 3.56e-04 | 3200.0ms/step | 18.7 samples/s +Step 18670/50462 | Loss: 2.6337 | LR: 3.56e-04 | 3200.1ms/step | 18.7 samples/s +Step 18680/50462 | Loss: 2.6582 | LR: 3.56e-04 | 3200.1ms/step | 18.7 samples/s +Step 18690/50462 | Loss: 2.6610 | LR: 3.56e-04 | 3199.5ms/step | 18.8 samples/s +Step 18700/50462 | Loss: 2.6594 | LR: 3.56e-04 | 3200.4ms/step | 18.7 samples/s +Step 18710/50462 | Loss: 2.6507 | LR: 3.56e-04 | 3200.1ms/step | 18.7 samples/s +Step 18720/50462 | Loss: 2.6510 | LR: 3.55e-04 | 3199.9ms/step | 18.8 samples/s +Step 18730/50462 | Loss: 2.6568 | LR: 3.55e-04 | 3200.0ms/step | 18.7 samples/s +Step 18740/50462 | Loss: 2.6416 | LR: 3.55e-04 | 3200.0ms/step | 18.7 samples/s +Step 18750/50462 | Loss: 2.6688 | LR: 3.55e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 18750: Loss = 2.6588, Perplexity = 14.28 +Step 18760/50462 | Loss: 2.6487 | LR: 3.55e-04 | 4874.1ms/step | 12.3 samples/s +Step 18770/50462 | Loss: 2.6456 | LR: 3.55e-04 | 3199.8ms/step | 18.8 samples/s +Step 18780/50462 | Loss: 2.6748 | LR: 3.55e-04 | 3200.1ms/step | 18.7 samples/s +Step 18790/50462 | Loss: 2.6612 | LR: 3.54e-04 | 3200.0ms/step | 18.7 samples/s +Step 18800/50462 | Loss: 2.6749 | LR: 3.54e-04 | 3200.2ms/step | 18.7 samples/s +Step 18810/50462 | Loss: 2.6410 | LR: 3.54e-04 | 3199.8ms/step | 18.8 samples/s +Step 18820/50462 | Loss: 2.6238 | LR: 3.54e-04 | 3252.0ms/step | 18.4 samples/s +Step 18830/50462 | Loss: 2.6339 | LR: 3.54e-04 | 3200.1ms/step | 18.7 samples/s +Step 18840/50462 | Loss: 2.6370 | LR: 3.54e-04 | 3200.0ms/step | 18.7 samples/s +Step 18850/50462 | Loss: 2.6287 | LR: 3.54e-04 | 3253.8ms/step | 18.4 samples/s +Step 18860/50462 | Loss: 2.6578 | LR: 3.53e-04 | 3200.0ms/step | 18.7 samples/s +Step 18870/50462 | Loss: 2.6647 | LR: 3.53e-04 | 3200.0ms/step | 18.8 samples/s +Step 18880/50462 | Loss: 2.7068 | LR: 3.53e-04 | 3199.8ms/step | 18.8 samples/s +Step 18890/50462 | Loss: 2.6352 | LR: 3.53e-04 | 3200.2ms/step | 18.7 samples/s +Step 18900/50462 | Loss: 2.6580 | LR: 3.53e-04 | 3200.0ms/step | 18.7 samples/s +Step 18910/50462 | Loss: 2.6536 | LR: 3.53e-04 | 3200.0ms/step | 18.7 samples/s +Step 18920/50462 | Loss: 2.6418 | LR: 3.53e-04 | 3200.1ms/step | 18.7 samples/s +Step 18930/50462 | Loss: 2.6759 | LR: 3.52e-04 | 3200.0ms/step | 18.7 samples/s +Step 18940/50462 | Loss: 2.6552 | LR: 3.52e-04 | 3251.9ms/step | 18.5 samples/s +Step 18950/50462 | Loss: 2.6540 | LR: 3.52e-04 | 3199.8ms/step | 18.8 samples/s +Step 18960/50462 | Loss: 2.6626 | LR: 3.52e-04 | 3200.2ms/step | 18.7 samples/s +Step 18970/50462 | Loss: 2.6429 | LR: 3.52e-04 | 3199.9ms/step | 18.8 samples/s +Step 18980/50462 | Loss: 2.6660 | LR: 3.52e-04 | 3199.9ms/step | 18.8 samples/s +Step 18990/50462 | Loss: 2.6313 | LR: 3.52e-04 | 3197.9ms/step | 18.8 samples/s +Step 19000/50462 | Loss: 2.6530 | LR: 3.51e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 19000: Loss = 2.6553, Perplexity = 14.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-19000 +Step 19010/50462 | Loss: 2.6463 | LR: 3.51e-04 | 5790.3ms/step | 10.4 samples/s +Step 19020/50462 | Loss: 2.6636 | LR: 3.51e-04 | 3184.3ms/step | 18.8 samples/s +Step 19030/50462 | Loss: 2.6631 | LR: 3.51e-04 | 3199.7ms/step | 18.8 samples/s +Step 19040/50462 | Loss: 2.6418 | LR: 3.51e-04 | 3200.0ms/step | 18.7 samples/s +Step 19050/50462 | Loss: 2.6443 | LR: 3.51e-04 | 3199.9ms/step | 18.8 samples/s +Step 19060/50462 | Loss: 2.6346 | LR: 3.50e-04 | 3200.2ms/step | 18.7 samples/s +Step 19070/50462 | Loss: 2.6279 | LR: 3.50e-04 | 3200.0ms/step | 18.8 samples/s +Step 19080/50462 | Loss: 2.6265 | LR: 3.50e-04 | 3199.9ms/step | 18.8 samples/s +Step 19090/50462 | Loss: 2.6406 | LR: 3.50e-04 | 3200.2ms/step | 18.7 samples/s +Step 19100/50462 | Loss: 2.6422 | LR: 3.50e-04 | 3199.8ms/step | 18.8 samples/s +Step 19110/50462 | Loss: 2.6243 | LR: 3.50e-04 | 3200.1ms/step | 18.7 samples/s +Step 19120/50462 | Loss: 2.6489 | LR: 3.50e-04 | 3199.9ms/step | 18.8 samples/s +Step 19130/50462 | Loss: 2.6824 | LR: 3.49e-04 | 3200.1ms/step | 18.7 samples/s +Step 19140/50462 | Loss: 2.6402 | LR: 3.49e-04 | 3200.0ms/step | 18.7 samples/s +Step 19150/50462 | Loss: 2.6546 | LR: 3.49e-04 | 3199.8ms/step | 18.8 samples/s +Step 19160/50462 | Loss: 2.6471 | LR: 3.49e-04 | 3252.1ms/step | 18.4 samples/s +Step 19170/50462 | Loss: 2.6316 | LR: 3.49e-04 | 3199.8ms/step | 18.8 samples/s +Step 19180/50462 | Loss: 2.6443 | LR: 3.49e-04 | 3200.2ms/step | 18.7 samples/s +Step 19190/50462 | Loss: 2.6204 | LR: 3.49e-04 | 3251.9ms/step | 18.5 samples/s +Step 19200/50462 | Loss: 2.6354 | LR: 3.48e-04 | 3200.0ms/step | 18.7 samples/s +Step 19210/50462 | Loss: 2.6303 | LR: 3.48e-04 | 3199.8ms/step | 18.8 samples/s +Step 19220/50462 | Loss: 2.6447 | LR: 3.48e-04 | 3200.2ms/step | 18.7 samples/s +Step 19230/50462 | Loss: 2.6321 | LR: 3.48e-04 | 3199.7ms/step | 18.8 samples/s +Step 19240/50462 | Loss: 2.6481 | LR: 3.48e-04 | 3200.3ms/step | 18.7 samples/s +Step 19250/50462 | Loss: 2.6359 | LR: 3.48e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 19250: Loss = 2.6538, Perplexity = 14.21 +Step 19260/50462 | Loss: 2.6218 | LR: 3.48e-04 | 4874.1ms/step | 12.3 samples/s +Step 19270/50462 | Loss: 2.6318 | LR: 3.47e-04 | 3199.7ms/step | 18.8 samples/s +Step 19280/50462 | Loss: 2.6336 | LR: 3.47e-04 | 3256.2ms/step | 18.4 samples/s +Step 19290/50462 | Loss: 2.6635 | LR: 3.47e-04 | 3200.0ms/step | 18.8 samples/s +Step 19300/50462 | Loss: 2.6304 | LR: 3.47e-04 | 3200.0ms/step | 18.7 samples/s +Step 19310/50462 | Loss: 2.6267 | LR: 3.47e-04 | 3199.8ms/step | 18.8 samples/s +Step 19320/50462 | Loss: 2.6003 | LR: 3.47e-04 | 3200.1ms/step | 18.7 samples/s +Step 19330/50462 | Loss: 2.6314 | LR: 3.47e-04 | 3199.9ms/step | 18.8 samples/s +Step 19340/50462 | Loss: 2.6181 | LR: 3.46e-04 | 3202.2ms/step | 18.7 samples/s +Step 19350/50462 | Loss: 2.6622 | LR: 3.46e-04 | 3202.1ms/step | 18.7 samples/s +Step 19360/50462 | Loss: 2.6539 | LR: 3.46e-04 | 3202.0ms/step | 18.7 samples/s +Step 19370/50462 | Loss: 2.6219 | LR: 3.46e-04 | 3199.6ms/step | 18.8 samples/s +Step 19380/50462 | Loss: 2.6462 | LR: 3.46e-04 | 3200.4ms/step | 18.7 samples/s +Step 19390/50462 | Loss: 2.6464 | LR: 3.46e-04 | 3199.8ms/step | 18.8 samples/s +Step 19400/50462 | Loss: 2.6221 | LR: 3.46e-04 | 3200.1ms/step | 18.7 samples/s +Step 19410/50462 | Loss: 2.6320 | LR: 3.45e-04 | 3199.9ms/step | 18.8 samples/s +Step 19420/50462 | Loss: 2.6162 | LR: 3.45e-04 | 3199.9ms/step | 18.8 samples/s +Step 19430/50462 | Loss: 2.6692 | LR: 3.45e-04 | 3200.7ms/step | 18.7 samples/s +Step 19440/50462 | Loss: 2.6363 | LR: 3.45e-04 | 3202.0ms/step | 18.7 samples/s +Step 19450/50462 | Loss: 2.6430 | LR: 3.45e-04 | 3201.3ms/step | 18.7 samples/s +Step 19460/50462 | Loss: 2.6339 | LR: 3.45e-04 | 3199.7ms/step | 18.8 samples/s +Step 19470/50462 | Loss: 2.6668 | LR: 3.45e-04 | 3200.4ms/step | 18.7 samples/s +Step 19480/50462 | Loss: 2.6279 | LR: 3.44e-04 | 3200.1ms/step | 18.7 samples/s +Step 19490/50462 | Loss: 2.6588 | LR: 3.44e-04 | 3199.8ms/step | 18.8 samples/s +Step 19500/50462 | Loss: 2.6423 | LR: 3.44e-04 | 3254.1ms/step | 18.4 samples/s +Validation at step 19500: Loss = 2.6520, Perplexity = 14.18 +Step 19510/50462 | Loss: 2.6430 | LR: 3.44e-04 | 4877.7ms/step | 12.3 samples/s +Step 19520/50462 | Loss: 2.6136 | LR: 3.44e-04 | 3200.1ms/step | 18.7 samples/s +Step 19530/50462 | Loss: 2.6401 | LR: 3.44e-04 | 3256.0ms/step | 18.4 samples/s +Step 19540/50462 | Loss: 2.6509 | LR: 3.44e-04 | 3200.2ms/step | 18.7 samples/s +Step 19550/50462 | Loss: 2.6447 | LR: 3.43e-04 | 3199.8ms/step | 18.8 samples/s +Step 19560/50462 | Loss: 2.6501 | LR: 3.43e-04 | 3200.3ms/step | 18.7 samples/s +Step 19570/50462 | Loss: 2.5971 | LR: 3.43e-04 | 3201.6ms/step | 18.7 samples/s +Step 19580/50462 | Loss: 2.6218 | LR: 3.43e-04 | 3200.2ms/step | 18.7 samples/s +Step 19590/50462 | Loss: 2.6510 | LR: 3.43e-04 | 3199.9ms/step | 18.8 samples/s +Step 19600/50462 | Loss: 2.6120 | LR: 3.43e-04 | 3202.1ms/step | 18.7 samples/s +Step 19610/50462 | Loss: 2.6660 | LR: 3.42e-04 | 3254.1ms/step | 18.4 samples/s +Step 19620/50462 | Loss: 2.6152 | LR: 3.42e-04 | 3199.2ms/step | 18.8 samples/s +Step 19630/50462 | Loss: 2.6173 | LR: 3.42e-04 | 3200.7ms/step | 18.7 samples/s +Step 19640/50462 | Loss: 2.6558 | LR: 3.42e-04 | 3200.0ms/step | 18.8 samples/s +Step 19650/50462 | Loss: 2.6244 | LR: 3.42e-04 | 3199.9ms/step | 18.8 samples/s +Step 19660/50462 | Loss: 2.6179 | LR: 3.42e-04 | 3199.9ms/step | 18.8 samples/s +Step 19670/50462 | Loss: 2.6627 | LR: 3.42e-04 | 3200.2ms/step | 18.7 samples/s +Step 19680/50462 | Loss: 2.6389 | LR: 3.41e-04 | 3200.0ms/step | 18.8 samples/s +Step 19690/50462 | Loss: 2.6457 | LR: 3.41e-04 | 3199.9ms/step | 18.8 samples/s +Step 19700/50462 | Loss: 2.6422 | LR: 3.41e-04 | 3202.0ms/step | 18.7 samples/s +Step 19710/50462 | Loss: 2.6474 | LR: 3.41e-04 | 3200.1ms/step | 18.7 samples/s +Step 19720/50462 | Loss: 2.6204 | LR: 3.41e-04 | 3200.1ms/step | 18.7 samples/s +Step 19730/50462 | Loss: 2.6266 | LR: 3.41e-04 | 3199.8ms/step | 18.8 samples/s +Step 19740/50462 | Loss: 2.6758 | LR: 3.41e-04 | 3199.9ms/step | 18.8 samples/s +Step 19750/50462 | Loss: 2.6510 | LR: 3.40e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 19750: Loss = 2.6483, Perplexity = 14.13 +Step 19760/50462 | Loss: 2.6394 | LR: 3.40e-04 | 4875.5ms/step | 12.3 samples/s +Step 19770/50462 | Loss: 2.6678 | LR: 3.40e-04 | 3202.5ms/step | 18.7 samples/s +Step 19780/50462 | Loss: 2.6614 | LR: 3.40e-04 | 3200.0ms/step | 18.8 samples/s +Step 19790/50462 | Loss: 2.5925 | LR: 3.40e-04 | 3200.0ms/step | 18.7 samples/s +Step 19800/50462 | Loss: 2.6511 | LR: 3.40e-04 | 3199.9ms/step | 18.8 samples/s +Step 19810/50462 | Loss: 2.6360 | LR: 3.40e-04 | 3200.2ms/step | 18.7 samples/s +Step 19820/50462 | Loss: 2.6404 | LR: 3.39e-04 | 3199.6ms/step | 18.8 samples/s +Step 19830/50462 | Loss: 2.6555 | LR: 3.39e-04 | 3200.4ms/step | 18.7 samples/s +Step 19840/50462 | Loss: 2.6237 | LR: 3.39e-04 | 3252.0ms/step | 18.5 samples/s +Step 19850/50462 | Loss: 2.6207 | LR: 3.39e-04 | 3199.7ms/step | 18.8 samples/s +Step 19860/50462 | Loss: 2.6304 | LR: 3.39e-04 | 3200.2ms/step | 18.7 samples/s +Step 19870/50462 | Loss: 2.6217 | LR: 3.39e-04 | 3251.7ms/step | 18.5 samples/s +Step 19880/50462 | Loss: 2.6348 | LR: 3.39e-04 | 3200.3ms/step | 18.7 samples/s +Step 19890/50462 | Loss: 2.6248 | LR: 3.38e-04 | 3200.1ms/step | 18.7 samples/s +Step 19900/50462 | Loss: 2.6415 | LR: 3.38e-04 | 3197.7ms/step | 18.8 samples/s +Step 19910/50462 | Loss: 2.6052 | LR: 3.38e-04 | 3200.0ms/step | 18.7 samples/s +Step 19920/50462 | Loss: 2.6273 | LR: 3.38e-04 | 3200.2ms/step | 18.7 samples/s +Step 19930/50462 | Loss: 2.6204 | LR: 3.38e-04 | 3200.0ms/step | 18.7 samples/s +Step 19940/50462 | Loss: 2.5995 | LR: 3.38e-04 | 3199.9ms/step | 18.8 samples/s +Step 19950/50462 | Loss: 2.6492 | LR: 3.37e-04 | 3256.1ms/step | 18.4 samples/s +Step 19960/50462 | Loss: 2.6389 | LR: 3.37e-04 | 3199.4ms/step | 18.8 samples/s +Step 19970/50462 | Loss: 2.6423 | LR: 3.37e-04 | 3200.7ms/step | 18.7 samples/s +Step 19980/50462 | Loss: 2.5994 | LR: 3.37e-04 | 3199.7ms/step | 18.8 samples/s +Step 19990/50462 | Loss: 2.6200 | LR: 3.37e-04 | 3200.0ms/step | 18.7 samples/s +Step 20000/50462 | Loss: 2.6157 | LR: 3.37e-04 | 3200.0ms/step | 18.7 samples/s +Validation at step 20000: Loss = 2.6460, Perplexity = 14.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-20000 +Step 20010/50462 | Loss: 2.6233 | LR: 3.37e-04 | 5797.7ms/step | 10.3 samples/s +Step 20020/50462 | Loss: 2.6288 | LR: 3.36e-04 | 3192.8ms/step | 18.8 samples/s +Step 20030/50462 | Loss: 2.6335 | LR: 3.36e-04 | 3199.6ms/step | 18.8 samples/s +Step 20040/50462 | Loss: 2.6564 | LR: 3.36e-04 | 3199.9ms/step | 18.8 samples/s +Step 20050/50462 | Loss: 2.6493 | LR: 3.36e-04 | 3199.7ms/step | 18.8 samples/s +Step 20060/50462 | Loss: 2.6117 | LR: 3.36e-04 | 3202.2ms/step | 18.7 samples/s +Step 20070/50462 | Loss: 2.6190 | LR: 3.36e-04 | 3200.2ms/step | 18.7 samples/s +Step 20080/50462 | Loss: 2.6183 | LR: 3.36e-04 | 3199.9ms/step | 18.8 samples/s +Step 20090/50462 | Loss: 2.6419 | LR: 3.35e-04 | 3200.0ms/step | 18.8 samples/s +Step 20100/50462 | Loss: 2.6573 | LR: 3.35e-04 | 3200.0ms/step | 18.7 samples/s +Step 20110/50462 | Loss: 2.6094 | LR: 3.35e-04 | 3200.1ms/step | 18.7 samples/s +Step 20120/50462 | Loss: 2.6344 | LR: 3.35e-04 | 3199.9ms/step | 18.8 samples/s +Step 20130/50462 | Loss: 2.6598 | LR: 3.35e-04 | 3200.3ms/step | 18.7 samples/s +Step 20140/50462 | Loss: 2.5985 | LR: 3.35e-04 | 3199.6ms/step | 18.8 samples/s +Step 20150/50462 | Loss: 2.6427 | LR: 3.35e-04 | 3202.2ms/step | 18.7 samples/s +Step 20160/50462 | Loss: 2.6230 | LR: 3.34e-04 | 3199.9ms/step | 18.8 samples/s +Step 20170/50462 | Loss: 2.6682 | LR: 3.34e-04 | 3200.7ms/step | 18.7 samples/s +Step 20180/50462 | Loss: 2.5999 | LR: 3.34e-04 | 3253.3ms/step | 18.4 samples/s +Step 20190/50462 | Loss: 2.6308 | LR: 3.34e-04 | 3199.9ms/step | 18.8 samples/s +Step 20200/50462 | Loss: 2.6369 | LR: 3.34e-04 | 3199.9ms/step | 18.8 samples/s +Step 20210/50462 | Loss: 2.6555 | LR: 3.34e-04 | 3256.1ms/step | 18.4 samples/s +Step 20220/50462 | Loss: 2.6499 | LR: 3.33e-04 | 3200.2ms/step | 18.7 samples/s +Step 20230/50462 | Loss: 2.6284 | LR: 3.33e-04 | 3199.7ms/step | 18.8 samples/s +Step 20240/50462 | Loss: 2.6242 | LR: 3.33e-04 | 3200.0ms/step | 18.7 samples/s +Step 20250/50462 | Loss: 2.6256 | LR: 3.33e-04 | 3202.2ms/step | 18.7 samples/s +Validation at step 20250: Loss = 2.6434, Perplexity = 14.06 +Step 20260/50462 | Loss: 2.6514 | LR: 3.33e-04 | 4878.0ms/step | 12.3 samples/s +Step 20270/50462 | Loss: 2.6773 | LR: 3.33e-04 | 3199.9ms/step | 18.8 samples/s +Step 20280/50462 | Loss: 2.5937 | LR: 3.33e-04 | 3202.8ms/step | 18.7 samples/s +Step 20290/50462 | Loss: 2.6580 | LR: 3.32e-04 | 3257.7ms/step | 18.4 samples/s +Step 20300/50462 | Loss: 2.6344 | LR: 3.32e-04 | 3201.6ms/step | 18.7 samples/s +Step 20310/50462 | Loss: 2.6317 | LR: 3.32e-04 | 3199.9ms/step | 18.8 samples/s +Step 20320/50462 | Loss: 2.6264 | LR: 3.32e-04 | 3202.0ms/step | 18.7 samples/s +Step 20330/50462 | Loss: 2.6573 | LR: 3.32e-04 | 3199.9ms/step | 18.8 samples/s +Step 20340/50462 | Loss: 2.6176 | LR: 3.32e-04 | 3200.1ms/step | 18.7 samples/s +Step 20350/50462 | Loss: 2.6128 | LR: 3.32e-04 | 3202.1ms/step | 18.7 samples/s +Step 20360/50462 | Loss: 2.6536 | LR: 3.31e-04 | 3199.9ms/step | 18.8 samples/s +Step 20370/50462 | Loss: 2.6380 | LR: 3.31e-04 | 3200.0ms/step | 18.7 samples/s +Step 20380/50462 | Loss: 2.6566 | LR: 3.31e-04 | 3200.0ms/step | 18.8 samples/s +Step 20390/50462 | Loss: 2.6205 | LR: 3.31e-04 | 3200.0ms/step | 18.8 samples/s +Step 20400/50462 | Loss: 2.6246 | LR: 3.31e-04 | 3203.9ms/step | 18.7 samples/s +Step 20410/50462 | Loss: 2.6194 | LR: 3.31e-04 | 3199.8ms/step | 18.8 samples/s +Step 20420/50462 | Loss: 2.6375 | LR: 3.30e-04 | 3200.3ms/step | 18.7 samples/s +Step 20430/50462 | Loss: 2.6178 | LR: 3.30e-04 | 3200.2ms/step | 18.7 samples/s +Step 20440/50462 | Loss: 2.6299 | LR: 3.30e-04 | 3199.9ms/step | 18.8 samples/s +Step 20450/50462 | Loss: 2.6373 | LR: 3.30e-04 | 3199.7ms/step | 18.8 samples/s +Step 20460/50462 | Loss: 2.6344 | LR: 3.30e-04 | 3200.2ms/step | 18.7 samples/s +Step 20470/50462 | Loss: 2.6548 | LR: 3.30e-04 | 3199.9ms/step | 18.8 samples/s +Step 20480/50462 | Loss: 2.6503 | LR: 3.30e-04 | 3204.1ms/step | 18.7 samples/s +Step 20490/50462 | Loss: 2.6394 | LR: 3.29e-04 | 3199.5ms/step | 18.8 samples/s +Step 20500/50462 | Loss: 2.6105 | LR: 3.29e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 20500: Loss = 2.6411, Perplexity = 14.03 +Step 20510/50462 | Loss: 2.6268 | LR: 3.29e-04 | 4876.2ms/step | 12.3 samples/s +Step 20520/50462 | Loss: 2.6282 | LR: 3.29e-04 | 3254.0ms/step | 18.4 samples/s +Step 20530/50462 | Loss: 2.5990 | LR: 3.29e-04 | 3199.9ms/step | 18.8 samples/s +Step 20540/50462 | Loss: 2.6282 | LR: 3.29e-04 | 3200.2ms/step | 18.7 samples/s +Step 20550/50462 | Loss: 2.6465 | LR: 3.29e-04 | 3254.0ms/step | 18.4 samples/s +Step 20560/50462 | Loss: 2.6655 | LR: 3.28e-04 | 3199.7ms/step | 18.8 samples/s +Step 20570/50462 | Loss: 2.6565 | LR: 3.28e-04 | 3200.1ms/step | 18.7 samples/s +Step 20580/50462 | Loss: 2.6370 | LR: 3.28e-04 | 3200.3ms/step | 18.7 samples/s +Step 20590/50462 | Loss: 2.6580 | LR: 3.28e-04 | 3199.8ms/step | 18.8 samples/s +Step 20600/50462 | Loss: 2.6089 | LR: 3.28e-04 | 3200.0ms/step | 18.8 samples/s +Step 20610/50462 | Loss: 2.6116 | LR: 3.28e-04 | 3200.1ms/step | 18.7 samples/s +Step 20620/50462 | Loss: 2.6105 | LR: 3.27e-04 | 3200.2ms/step | 18.7 samples/s +Step 20630/50462 | Loss: 2.6269 | LR: 3.27e-04 | 3253.8ms/step | 18.4 samples/s +Step 20640/50462 | Loss: 2.6325 | LR: 3.27e-04 | 3199.8ms/step | 18.8 samples/s +Step 20650/50462 | Loss: 2.6098 | LR: 3.27e-04 | 3202.4ms/step | 18.7 samples/s +Step 20660/50462 | Loss: 2.6472 | LR: 3.27e-04 | 3199.8ms/step | 18.8 samples/s +Step 20670/50462 | Loss: 2.6626 | LR: 3.27e-04 | 3200.0ms/step | 18.7 samples/s +Step 20680/50462 | Loss: 2.6100 | LR: 3.27e-04 | 3199.9ms/step | 18.8 samples/s +Step 20690/50462 | Loss: 2.6240 | LR: 3.26e-04 | 3200.3ms/step | 18.7 samples/s +Step 20700/50462 | Loss: 2.5935 | LR: 3.26e-04 | 3200.0ms/step | 18.7 samples/s +Step 20710/50462 | Loss: 2.6398 | LR: 3.26e-04 | 3199.7ms/step | 18.8 samples/s +Step 20720/50462 | Loss: 2.6451 | LR: 3.26e-04 | 3200.1ms/step | 18.7 samples/s +Step 20730/50462 | Loss: 2.6026 | LR: 3.26e-04 | 3202.0ms/step | 18.7 samples/s +Step 20740/50462 | Loss: 2.6146 | LR: 3.26e-04 | 3202.6ms/step | 18.7 samples/s +Step 20750/50462 | Loss: 2.6273 | LR: 3.26e-04 | 3201.5ms/step | 18.7 samples/s +Validation at step 20750: Loss = 2.6381, Perplexity = 13.99 +Step 20760/50462 | Loss: 2.6368 | LR: 3.25e-04 | 4875.9ms/step | 12.3 samples/s +Step 20770/50462 | Loss: 2.6383 | LR: 3.25e-04 | 3199.9ms/step | 18.8 samples/s +Step 20780/50462 | Loss: 2.6478 | LR: 3.25e-04 | 3202.0ms/step | 18.7 samples/s +Step 20790/50462 | Loss: 2.5976 | LR: 3.25e-04 | 3200.0ms/step | 18.7 samples/s +Step 20800/50462 | Loss: 2.6399 | LR: 3.25e-04 | 3200.0ms/step | 18.8 samples/s +Step 20810/50462 | Loss: 2.6349 | LR: 3.25e-04 | 3199.9ms/step | 18.8 samples/s +Step 20820/50462 | Loss: 2.6429 | LR: 3.24e-04 | 3200.1ms/step | 18.7 samples/s +Step 20830/50462 | Loss: 2.6165 | LR: 3.24e-04 | 3199.8ms/step | 18.8 samples/s +Step 20840/50462 | Loss: 2.6460 | LR: 3.24e-04 | 3202.0ms/step | 18.7 samples/s +Step 20850/50462 | Loss: 2.6383 | LR: 3.24e-04 | 3200.3ms/step | 18.7 samples/s +Step 20860/50462 | Loss: 2.5996 | LR: 3.24e-04 | 3253.4ms/step | 18.4 samples/s +Step 20870/50462 | Loss: 2.6101 | LR: 3.24e-04 | 3200.3ms/step | 18.7 samples/s +Step 20880/50462 | Loss: 2.6321 | LR: 3.24e-04 | 3200.1ms/step | 18.7 samples/s +Step 20890/50462 | Loss: 2.6402 | LR: 3.23e-04 | 3256.0ms/step | 18.4 samples/s +Step 20900/50462 | Loss: 2.6089 | LR: 3.23e-04 | 3200.0ms/step | 18.8 samples/s +Step 20910/50462 | Loss: 2.6161 | LR: 3.23e-04 | 3202.1ms/step | 18.7 samples/s +Step 20920/50462 | Loss: 2.6380 | LR: 3.23e-04 | 3199.7ms/step | 18.8 samples/s +Step 20930/50462 | Loss: 2.6550 | LR: 3.23e-04 | 3200.1ms/step | 18.7 samples/s +Step 20940/50462 | Loss: 2.6283 | LR: 3.23e-04 | 3204.0ms/step | 18.7 samples/s +Step 20950/50462 | Loss: 2.6464 | LR: 3.23e-04 | 3200.4ms/step | 18.7 samples/s +Step 20960/50462 | Loss: 2.6416 | LR: 3.22e-04 | 3200.1ms/step | 18.7 samples/s +Step 20970/50462 | Loss: 2.6030 | LR: 3.22e-04 | 3258.3ms/step | 18.4 samples/s +Step 20980/50462 | Loss: 2.6170 | LR: 3.22e-04 | 3201.0ms/step | 18.7 samples/s +Step 20990/50462 | Loss: 2.6290 | LR: 3.22e-04 | 3200.9ms/step | 18.7 samples/s +Step 21000/50462 | Loss: 2.6447 | LR: 3.22e-04 | 3203.4ms/step | 18.7 samples/s +Validation at step 21000: Loss = 2.6364, Perplexity = 13.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-21000 +Step 21010/50462 | Loss: 2.6429 | LR: 3.22e-04 | 5773.4ms/step | 10.4 samples/s +Step 21020/50462 | Loss: 2.6498 | LR: 3.21e-04 | 3183.9ms/step | 18.8 samples/s +Step 21030/50462 | Loss: 2.5993 | LR: 3.21e-04 | 3196.4ms/step | 18.8 samples/s +Step 21040/50462 | Loss: 2.6229 | LR: 3.21e-04 | 3200.1ms/step | 18.7 samples/s +Step 21050/50462 | Loss: 2.6071 | LR: 3.21e-04 | 3200.0ms/step | 18.7 samples/s +Step 21060/50462 | Loss: 2.6156 | LR: 3.21e-04 | 3200.0ms/step | 18.8 samples/s +Step 21070/50462 | Loss: 2.6200 | LR: 3.21e-04 | 3202.2ms/step | 18.7 samples/s +Step 21080/50462 | Loss: 2.6230 | LR: 3.21e-04 | 3200.4ms/step | 18.7 samples/s +Step 21090/50462 | Loss: 2.6447 | LR: 3.20e-04 | 3200.0ms/step | 18.8 samples/s +Step 21100/50462 | Loss: 2.6600 | LR: 3.20e-04 | 3201.4ms/step | 18.7 samples/s +Step 21110/50462 | Loss: 2.6323 | LR: 3.20e-04 | 3202.5ms/step | 18.7 samples/s +Step 21120/50462 | Loss: 2.6042 | LR: 3.20e-04 | 3203.5ms/step | 18.7 samples/s +Step 21130/50462 | Loss: 2.6315 | LR: 3.20e-04 | 3199.9ms/step | 18.8 samples/s +Step 21140/50462 | Loss: 2.6313 | LR: 3.20e-04 | 3202.1ms/step | 18.7 samples/s +Step 21150/50462 | Loss: 2.6042 | LR: 3.19e-04 | 3201.7ms/step | 18.7 samples/s +Step 21160/50462 | Loss: 2.5938 | LR: 3.19e-04 | 3200.3ms/step | 18.7 samples/s +Step 21170/50462 | Loss: 2.6287 | LR: 3.19e-04 | 3199.9ms/step | 18.8 samples/s +Step 21180/50462 | Loss: 2.6435 | LR: 3.19e-04 | 3200.0ms/step | 18.7 samples/s +Step 21190/50462 | Loss: 2.6516 | LR: 3.19e-04 | 3200.1ms/step | 18.7 samples/s +Step 21200/50462 | Loss: 2.6362 | LR: 3.19e-04 | 3254.0ms/step | 18.4 samples/s +Step 21210/50462 | Loss: 2.6472 | LR: 3.19e-04 | 3199.7ms/step | 18.8 samples/s +Step 21220/50462 | Loss: 2.6430 | LR: 3.18e-04 | 3200.1ms/step | 18.7 samples/s +Step 21230/50462 | Loss: 2.5974 | LR: 3.18e-04 | 3256.0ms/step | 18.4 samples/s +Step 21240/50462 | Loss: 2.6416 | LR: 3.18e-04 | 3199.9ms/step | 18.8 samples/s +Step 21250/50462 | Loss: 2.6188 | LR: 3.18e-04 | 3200.0ms/step | 18.8 samples/s +Validation at step 21250: Loss = 2.6329, Perplexity = 13.91 +Step 21260/50462 | Loss: 2.6068 | LR: 3.18e-04 | 4882.1ms/step | 12.3 samples/s +Step 21270/50462 | Loss: 2.6109 | LR: 3.18e-04 | 3200.1ms/step | 18.7 samples/s +Step 21280/50462 | Loss: 2.6522 | LR: 3.17e-04 | 3199.9ms/step | 18.8 samples/s +Step 21290/50462 | Loss: 2.6363 | LR: 3.17e-04 | 3199.9ms/step | 18.8 samples/s +Step 21300/50462 | Loss: 2.6162 | LR: 3.17e-04 | 3255.8ms/step | 18.4 samples/s +Step 21310/50462 | Loss: 2.6309 | LR: 3.17e-04 | 3202.2ms/step | 18.7 samples/s +Step 21320/50462 | Loss: 2.6122 | LR: 3.17e-04 | 3201.6ms/step | 18.7 samples/s +Step 21330/50462 | Loss: 2.6235 | LR: 3.17e-04 | 3202.2ms/step | 18.7 samples/s +Step 21340/50462 | Loss: 2.6259 | LR: 3.17e-04 | 3204.2ms/step | 18.7 samples/s +Step 21350/50462 | Loss: 2.5970 | LR: 3.16e-04 | 3199.9ms/step | 18.8 samples/s +Step 21360/50462 | Loss: 2.6138 | LR: 3.16e-04 | 3200.1ms/step | 18.7 samples/s +Step 21370/50462 | Loss: 2.6348 | LR: 3.16e-04 | 3199.9ms/step | 18.8 samples/s +Step 21380/50462 | Loss: 2.6132 | LR: 3.16e-04 | 3200.5ms/step | 18.7 samples/s +Step 21390/50462 | Loss: 2.6525 | LR: 3.16e-04 | 3199.6ms/step | 18.8 samples/s +Step 21400/50462 | Loss: 2.6380 | LR: 3.16e-04 | 3199.8ms/step | 18.8 samples/s +Step 21410/50462 | Loss: 2.6402 | LR: 3.16e-04 | 3200.1ms/step | 18.7 samples/s +Step 21420/50462 | Loss: 2.6127 | LR: 3.15e-04 | 3199.9ms/step | 18.8 samples/s +Step 21430/50462 | Loss: 2.6245 | LR: 3.15e-04 | 3200.2ms/step | 18.7 samples/s +Step 21440/50462 | Loss: 2.6137 | LR: 3.15e-04 | 3201.9ms/step | 18.7 samples/s +Step 21450/50462 | Loss: 2.6142 | LR: 3.15e-04 | 3200.5ms/step | 18.7 samples/s +Step 21460/50462 | Loss: 2.6070 | LR: 3.15e-04 | 3201.6ms/step | 18.7 samples/s +Step 21470/50462 | Loss: 2.6193 | LR: 3.15e-04 | 3200.8ms/step | 18.7 samples/s +Step 21480/50462 | Loss: 2.6236 | LR: 3.14e-04 | 3201.2ms/step | 18.7 samples/s +Step 21490/50462 | Loss: 2.6246 | LR: 3.14e-04 | 3200.1ms/step | 18.7 samples/s +Step 21500/50462 | Loss: 2.5995 | LR: 3.14e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 21500: Loss = 2.6307, Perplexity = 13.88 +Step 21510/50462 | Loss: 2.6227 | LR: 3.14e-04 | 4875.9ms/step | 12.3 samples/s +Step 21520/50462 | Loss: 2.5867 | LR: 3.14e-04 | 3200.1ms/step | 18.7 samples/s +Step 21530/50462 | Loss: 2.6414 | LR: 3.14e-04 | 3200.2ms/step | 18.7 samples/s +Step 21540/50462 | Loss: 2.6007 | LR: 3.14e-04 | 3258.0ms/step | 18.4 samples/s +Step 21550/50462 | Loss: 2.6169 | LR: 3.13e-04 | 3202.3ms/step | 18.7 samples/s +Step 21560/50462 | Loss: 2.6008 | LR: 3.13e-04 | 3199.5ms/step | 18.8 samples/s +Step 21570/50462 | Loss: 2.6220 | LR: 3.13e-04 | 3254.0ms/step | 18.4 samples/s +Step 21580/50462 | Loss: 2.5990 | LR: 3.13e-04 | 3200.1ms/step | 18.7 samples/s +Step 21590/50462 | Loss: 2.6138 | LR: 3.13e-04 | 3199.9ms/step | 18.8 samples/s +Step 21600/50462 | Loss: 2.5871 | LR: 3.13e-04 | 3200.0ms/step | 18.7 samples/s +Step 21610/50462 | Loss: 2.6285 | LR: 3.12e-04 | 3202.2ms/step | 18.7 samples/s +Step 21620/50462 | Loss: 2.6296 | LR: 3.12e-04 | 3202.1ms/step | 18.7 samples/s +Step 21630/50462 | Loss: 2.6172 | LR: 3.12e-04 | 3200.0ms/step | 18.7 samples/s +Step 21640/50462 | Loss: 2.5821 | LR: 3.12e-04 | 3255.9ms/step | 18.4 samples/s +Step 21650/50462 | Loss: 2.6431 | LR: 3.12e-04 | 3199.9ms/step | 18.8 samples/s +Step 21660/50462 | Loss: 2.5851 | LR: 3.12e-04 | 3202.0ms/step | 18.7 samples/s +Step 21670/50462 | Loss: 2.5887 | LR: 3.12e-04 | 3200.1ms/step | 18.7 samples/s +Step 21680/50462 | Loss: 2.6193 | LR: 3.11e-04 | 3200.2ms/step | 18.7 samples/s +Step 21690/50462 | Loss: 2.5963 | LR: 3.11e-04 | 3199.4ms/step | 18.8 samples/s +Step 21700/50462 | Loss: 2.6414 | LR: 3.11e-04 | 3200.3ms/step | 18.7 samples/s +Step 21710/50462 | Loss: 2.6486 | LR: 3.11e-04 | 3200.0ms/step | 18.7 samples/s +Step 21720/50462 | Loss: 2.5948 | LR: 3.11e-04 | 3200.1ms/step | 18.7 samples/s +Step 21730/50462 | Loss: 2.6229 | LR: 3.11e-04 | 3199.8ms/step | 18.8 samples/s +Step 21740/50462 | Loss: 2.5961 | LR: 3.10e-04 | 3200.0ms/step | 18.7 samples/s +Step 21750/50462 | Loss: 2.5891 | LR: 3.10e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 21750: Loss = 2.6285, Perplexity = 13.85 +Step 21760/50462 | Loss: 2.6031 | LR: 3.10e-04 | 4879.7ms/step | 12.3 samples/s +Step 21770/50462 | Loss: 2.5833 | LR: 3.10e-04 | 3200.1ms/step | 18.7 samples/s +Step 21780/50462 | Loss: 2.5812 | LR: 3.10e-04 | 3200.0ms/step | 18.7 samples/s +Step 21790/50462 | Loss: 2.5997 | LR: 3.10e-04 | 3200.1ms/step | 18.7 samples/s +Step 21800/50462 | Loss: 2.6254 | LR: 3.10e-04 | 3199.5ms/step | 18.8 samples/s +Step 21810/50462 | Loss: 2.6452 | LR: 3.09e-04 | 3200.4ms/step | 18.7 samples/s +Step 21820/50462 | Loss: 2.5976 | LR: 3.09e-04 | 3200.2ms/step | 18.7 samples/s +Step 21830/50462 | Loss: 2.6193 | LR: 3.09e-04 | 3199.7ms/step | 18.8 samples/s +Step 21840/50462 | Loss: 2.6138 | LR: 3.09e-04 | 3200.2ms/step | 18.7 samples/s +Step 21850/50462 | Loss: 2.6014 | LR: 3.09e-04 | 3200.0ms/step | 18.7 samples/s +Step 21860/50462 | Loss: 2.6256 | LR: 3.09e-04 | 3200.0ms/step | 18.8 samples/s +Step 21870/50462 | Loss: 2.6077 | LR: 3.08e-04 | 3199.9ms/step | 18.8 samples/s +Step 21880/50462 | Loss: 2.6130 | LR: 3.08e-04 | 3256.0ms/step | 18.4 samples/s +Step 21890/50462 | Loss: 2.6239 | LR: 3.08e-04 | 3202.1ms/step | 18.7 samples/s +Step 21900/50462 | Loss: 2.5967 | LR: 3.08e-04 | 3201.8ms/step | 18.7 samples/s +Step 21910/50462 | Loss: 2.5980 | LR: 3.08e-04 | 3254.3ms/step | 18.4 samples/s +Step 21920/50462 | Loss: 2.6082 | LR: 3.08e-04 | 3200.5ms/step | 18.7 samples/s +Step 21930/50462 | Loss: 2.6281 | LR: 3.08e-04 | 3201.3ms/step | 18.7 samples/s +Step 21940/50462 | Loss: 2.6384 | LR: 3.07e-04 | 3199.9ms/step | 18.8 samples/s +Step 21950/50462 | Loss: 2.6162 | LR: 3.07e-04 | 3200.2ms/step | 18.7 samples/s +Step 21960/50462 | Loss: 2.6102 | LR: 3.07e-04 | 3200.1ms/step | 18.7 samples/s +Step 21970/50462 | Loss: 2.6074 | LR: 3.07e-04 | 3199.9ms/step | 18.8 samples/s +Step 21980/50462 | Loss: 2.5969 | LR: 3.07e-04 | 3254.0ms/step | 18.4 samples/s +Step 21990/50462 | Loss: 2.6096 | LR: 3.07e-04 | 3199.9ms/step | 18.8 samples/s +Step 22000/50462 | Loss: 2.6336 | LR: 3.06e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 22000: Loss = 2.6255, Perplexity = 13.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-22000 +Step 22010/50462 | Loss: 2.5973 | LR: 3.06e-04 | 5773.1ms/step | 10.4 samples/s +Step 22020/50462 | Loss: 2.6036 | LR: 3.06e-04 | 3186.5ms/step | 18.8 samples/s +Step 22030/50462 | Loss: 2.6073 | LR: 3.06e-04 | 3200.1ms/step | 18.7 samples/s +Step 22040/50462 | Loss: 2.6308 | LR: 3.06e-04 | 3200.0ms/step | 18.8 samples/s +Step 22050/50462 | Loss: 2.6164 | LR: 3.06e-04 | 3200.1ms/step | 18.7 samples/s +Step 22060/50462 | Loss: 2.5950 | LR: 3.06e-04 | 3199.9ms/step | 18.8 samples/s +Step 22070/50462 | Loss: 2.6325 | LR: 3.05e-04 | 3200.2ms/step | 18.7 samples/s +Step 22080/50462 | Loss: 2.5969 | LR: 3.05e-04 | 3200.0ms/step | 18.7 samples/s +Step 22090/50462 | Loss: 2.6163 | LR: 3.05e-04 | 3200.0ms/step | 18.7 samples/s +Step 22100/50462 | Loss: 2.6058 | LR: 3.05e-04 | 3199.9ms/step | 18.8 samples/s +Step 22110/50462 | Loss: 2.6231 | LR: 3.05e-04 | 3200.2ms/step | 18.7 samples/s +Step 22120/50462 | Loss: 2.6058 | LR: 3.05e-04 | 3199.9ms/step | 18.8 samples/s +Step 22130/50462 | Loss: 2.6177 | LR: 3.04e-04 | 3199.9ms/step | 18.8 samples/s +Step 22140/50462 | Loss: 2.5942 | LR: 3.04e-04 | 3200.0ms/step | 18.7 samples/s +Step 22150/50462 | Loss: 2.6214 | LR: 3.04e-04 | 3200.3ms/step | 18.7 samples/s +Step 22160/50462 | Loss: 2.6443 | LR: 3.04e-04 | 3199.9ms/step | 18.8 samples/s +Step 22170/50462 | Loss: 2.5999 | LR: 3.04e-04 | 3200.0ms/step | 18.7 samples/s +Step 22180/50462 | Loss: 2.6214 | LR: 3.04e-04 | 3199.9ms/step | 18.8 samples/s +Step 22190/50462 | Loss: 2.6291 | LR: 3.04e-04 | 3200.0ms/step | 18.7 samples/s +Step 22200/50462 | Loss: 2.6208 | LR: 3.03e-04 | 3200.1ms/step | 18.7 samples/s +Step 22210/50462 | Loss: 2.6391 | LR: 3.03e-04 | 3199.8ms/step | 18.8 samples/s +Step 22220/50462 | Loss: 2.6288 | LR: 3.03e-04 | 3254.1ms/step | 18.4 samples/s +Step 22230/50462 | Loss: 2.5985 | LR: 3.03e-04 | 3199.8ms/step | 18.8 samples/s +Step 22240/50462 | Loss: 2.6031 | LR: 3.03e-04 | 3200.1ms/step | 18.7 samples/s +Step 22250/50462 | Loss: 2.6002 | LR: 3.03e-04 | 3254.1ms/step | 18.4 samples/s +Validation at step 22250: Loss = 2.6234, Perplexity = 13.78 +Step 22260/50462 | Loss: 2.6095 | LR: 3.02e-04 | 4876.0ms/step | 12.3 samples/s +Step 22270/50462 | Loss: 2.6431 | LR: 3.02e-04 | 3199.9ms/step | 18.8 samples/s +Step 22280/50462 | Loss: 2.5808 | LR: 3.02e-04 | 3200.0ms/step | 18.8 samples/s +Step 22290/50462 | Loss: 2.6183 | LR: 3.02e-04 | 3200.0ms/step | 18.8 samples/s +Step 22300/50462 | Loss: 2.5900 | LR: 3.02e-04 | 3200.0ms/step | 18.7 samples/s +Step 22310/50462 | Loss: 2.6256 | LR: 3.02e-04 | 3200.2ms/step | 18.7 samples/s +Step 22320/50462 | Loss: 2.5909 | LR: 3.02e-04 | 3253.8ms/step | 18.4 samples/s +Step 22330/50462 | Loss: 2.6366 | LR: 3.01e-04 | 3199.9ms/step | 18.8 samples/s +Step 22340/50462 | Loss: 2.6088 | LR: 3.01e-04 | 3200.1ms/step | 18.7 samples/s +Step 22350/50462 | Loss: 2.6222 | LR: 3.01e-04 | 3200.0ms/step | 18.7 samples/s +Step 22360/50462 | Loss: 2.6100 | LR: 3.01e-04 | 3200.1ms/step | 18.7 samples/s +Step 22370/50462 | Loss: 2.6117 | LR: 3.01e-04 | 3200.0ms/step | 18.8 samples/s +Step 22380/50462 | Loss: 2.5876 | LR: 3.01e-04 | 3199.9ms/step | 18.8 samples/s +Step 22390/50462 | Loss: 2.6250 | LR: 3.00e-04 | 3202.1ms/step | 18.7 samples/s +Step 22400/50462 | Loss: 2.5960 | LR: 3.00e-04 | 3199.7ms/step | 18.8 samples/s +Step 22410/50462 | Loss: 2.6165 | LR: 3.00e-04 | 3200.3ms/step | 18.7 samples/s +Step 22420/50462 | Loss: 2.5993 | LR: 3.00e-04 | 3200.1ms/step | 18.7 samples/s +Step 22430/50462 | Loss: 2.6177 | LR: 3.00e-04 | 3199.6ms/step | 18.8 samples/s +Step 22440/50462 | Loss: 2.6276 | LR: 3.00e-04 | 3200.2ms/step | 18.7 samples/s +Step 22450/50462 | Loss: 2.6098 | LR: 3.00e-04 | 3200.1ms/step | 18.7 samples/s +Step 22460/50462 | Loss: 2.5788 | LR: 2.99e-04 | 3199.9ms/step | 18.8 samples/s +Step 22470/50462 | Loss: 2.6013 | LR: 2.99e-04 | 3199.9ms/step | 18.8 samples/s +Step 22480/50462 | Loss: 2.6085 | LR: 2.99e-04 | 3200.2ms/step | 18.7 samples/s +Step 22490/50462 | Loss: 2.5949 | LR: 2.99e-04 | 3199.8ms/step | 18.8 samples/s +Step 22500/50462 | Loss: 2.5951 | LR: 2.99e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 22500: Loss = 2.6215, Perplexity = 13.76 +Step 22510/50462 | Loss: 2.6160 | LR: 2.99e-04 | 4876.2ms/step | 12.3 samples/s +Step 22520/50462 | Loss: 2.6241 | LR: 2.98e-04 | 3199.6ms/step | 18.8 samples/s +Step 22530/50462 | Loss: 2.5951 | LR: 2.98e-04 | 3200.2ms/step | 18.7 samples/s +Step 22540/50462 | Loss: 2.6236 | LR: 2.98e-04 | 3199.9ms/step | 18.8 samples/s +Step 22550/50462 | Loss: 2.5984 | LR: 2.98e-04 | 3200.1ms/step | 18.7 samples/s +Step 22560/50462 | Loss: 2.6069 | LR: 2.98e-04 | 3251.8ms/step | 18.5 samples/s +Step 22570/50462 | Loss: 2.6167 | LR: 2.98e-04 | 3200.1ms/step | 18.7 samples/s +Step 22580/50462 | Loss: 2.5672 | LR: 2.98e-04 | 3200.2ms/step | 18.7 samples/s +Step 22590/50462 | Loss: 2.6387 | LR: 2.97e-04 | 3251.5ms/step | 18.5 samples/s +Step 22600/50462 | Loss: 2.6070 | LR: 2.97e-04 | 3200.3ms/step | 18.7 samples/s +Step 22610/50462 | Loss: 2.6270 | LR: 2.97e-04 | 3200.1ms/step | 18.7 samples/s +Step 22620/50462 | Loss: 2.5883 | LR: 2.97e-04 | 3199.9ms/step | 18.8 samples/s +Step 22630/50462 | Loss: 2.5827 | LR: 2.97e-04 | 3199.7ms/step | 18.8 samples/s +Step 22640/50462 | Loss: 2.5920 | LR: 2.97e-04 | 3200.4ms/step | 18.7 samples/s +Step 22650/50462 | Loss: 2.6496 | LR: 2.96e-04 | 3253.2ms/step | 18.4 samples/s +Step 22660/50462 | Loss: 2.6105 | LR: 2.96e-04 | 3200.6ms/step | 18.7 samples/s +Step 22670/50462 | Loss: 2.5930 | LR: 2.96e-04 | 3200.3ms/step | 18.7 samples/s +Step 22680/50462 | Loss: 2.6231 | LR: 2.96e-04 | 3199.6ms/step | 18.8 samples/s +Step 22690/50462 | Loss: 2.6006 | LR: 2.96e-04 | 3200.3ms/step | 18.7 samples/s +Step 22700/50462 | Loss: 2.6176 | LR: 2.96e-04 | 3197.4ms/step | 18.8 samples/s +Step 22710/50462 | Loss: 2.5966 | LR: 2.95e-04 | 3200.4ms/step | 18.7 samples/s +Step 22720/50462 | Loss: 2.6391 | LR: 2.95e-04 | 3200.0ms/step | 18.8 samples/s +Step 22730/50462 | Loss: 2.5951 | LR: 2.95e-04 | 3200.0ms/step | 18.7 samples/s +Step 22740/50462 | Loss: 2.6177 | LR: 2.95e-04 | 3199.9ms/step | 18.8 samples/s +Step 22750/50462 | Loss: 2.6300 | LR: 2.95e-04 | 3200.0ms/step | 18.7 samples/s +Validation at step 22750: Loss = 2.6190, Perplexity = 13.72 +Step 22760/50462 | Loss: 2.6221 | LR: 2.95e-04 | 4876.1ms/step | 12.3 samples/s +Step 22770/50462 | Loss: 2.6100 | LR: 2.95e-04 | 3199.9ms/step | 18.8 samples/s +Step 22780/50462 | Loss: 2.6048 | LR: 2.94e-04 | 3200.1ms/step | 18.7 samples/s +Step 22790/50462 | Loss: 2.6028 | LR: 2.94e-04 | 3199.9ms/step | 18.8 samples/s +Step 22800/50462 | Loss: 2.6281 | LR: 2.94e-04 | 3200.1ms/step | 18.7 samples/s +Step 22810/50462 | Loss: 2.6052 | LR: 2.94e-04 | 3199.9ms/step | 18.8 samples/s +Step 22820/50462 | Loss: 2.5998 | LR: 2.94e-04 | 3200.1ms/step | 18.7 samples/s +Step 22830/50462 | Loss: 2.6077 | LR: 2.94e-04 | 3199.9ms/step | 18.8 samples/s +Step 22840/50462 | Loss: 2.5895 | LR: 2.93e-04 | 3200.0ms/step | 18.7 samples/s +Step 22850/50462 | Loss: 2.5846 | LR: 2.93e-04 | 3199.8ms/step | 18.8 samples/s +Step 22860/50462 | Loss: 2.6124 | LR: 2.93e-04 | 3199.7ms/step | 18.8 samples/s +Step 22870/50462 | Loss: 2.6244 | LR: 2.93e-04 | 3200.5ms/step | 18.7 samples/s +Step 22880/50462 | Loss: 2.6277 | LR: 2.93e-04 | 3199.9ms/step | 18.8 samples/s +Step 22890/50462 | Loss: 2.5865 | LR: 2.93e-04 | 3200.0ms/step | 18.7 samples/s +Step 22900/50462 | Loss: 2.5893 | LR: 2.93e-04 | 3252.0ms/step | 18.4 samples/s +Step 22910/50462 | Loss: 2.6075 | LR: 2.92e-04 | 3200.0ms/step | 18.8 samples/s +Step 22920/50462 | Loss: 2.5704 | LR: 2.92e-04 | 3199.8ms/step | 18.8 samples/s +Step 22930/50462 | Loss: 2.6096 | LR: 2.92e-04 | 3254.2ms/step | 18.4 samples/s +Step 22940/50462 | Loss: 2.5860 | LR: 2.92e-04 | 3199.9ms/step | 18.8 samples/s +Step 22950/50462 | Loss: 2.5858 | LR: 2.92e-04 | 3200.0ms/step | 18.7 samples/s +Step 22960/50462 | Loss: 2.5893 | LR: 2.92e-04 | 3200.1ms/step | 18.7 samples/s +Step 22970/50462 | Loss: 2.6038 | LR: 2.91e-04 | 3199.9ms/step | 18.8 samples/s +Step 22980/50462 | Loss: 2.6051 | LR: 2.91e-04 | 3200.0ms/step | 18.7 samples/s +Step 22990/50462 | Loss: 2.6178 | LR: 2.91e-04 | 3255.9ms/step | 18.4 samples/s +Step 23000/50462 | Loss: 2.5877 | LR: 2.91e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 23000: Loss = 2.6167, Perplexity = 13.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-23000 +Step 23010/50462 | Loss: 2.6116 | LR: 2.91e-04 | 5802.6ms/step | 10.3 samples/s +Step 23020/50462 | Loss: 2.5719 | LR: 2.91e-04 | 3185.8ms/step | 18.8 samples/s +Step 23030/50462 | Loss: 2.6017 | LR: 2.91e-04 | 3189.8ms/step | 18.8 samples/s +Step 23040/50462 | Loss: 2.6037 | LR: 2.90e-04 | 3195.7ms/step | 18.8 samples/s +Step 23050/50462 | Loss: 2.5983 | LR: 2.90e-04 | 3199.9ms/step | 18.8 samples/s +Step 23060/50462 | Loss: 2.5604 | LR: 2.90e-04 | 3200.0ms/step | 18.7 samples/s +Step 23070/50462 | Loss: 2.5936 | LR: 2.90e-04 | 3200.0ms/step | 18.7 samples/s +Step 23080/50462 | Loss: 2.6069 | LR: 2.90e-04 | 3200.0ms/step | 18.8 samples/s +Step 23090/50462 | Loss: 2.5827 | LR: 2.90e-04 | 3199.9ms/step | 18.8 samples/s +Step 23100/50462 | Loss: 2.5844 | LR: 2.89e-04 | 3200.1ms/step | 18.7 samples/s +Step 23110/50462 | Loss: 2.5894 | LR: 2.89e-04 | 3200.1ms/step | 18.7 samples/s +Step 23120/50462 | Loss: 2.6381 | LR: 2.89e-04 | 3199.8ms/step | 18.8 samples/s +Step 23130/50462 | Loss: 2.5994 | LR: 2.89e-04 | 3200.2ms/step | 18.7 samples/s +Step 23140/50462 | Loss: 2.6026 | LR: 2.89e-04 | 3199.8ms/step | 18.8 samples/s +Step 23150/50462 | Loss: 2.6270 | LR: 2.89e-04 | 3199.9ms/step | 18.8 samples/s +Step 23160/50462 | Loss: 2.5978 | LR: 2.88e-04 | 3200.3ms/step | 18.7 samples/s +Step 23170/50462 | Loss: 2.6146 | LR: 2.88e-04 | 3199.9ms/step | 18.8 samples/s +Step 23180/50462 | Loss: 2.5919 | LR: 2.88e-04 | 3200.0ms/step | 18.8 samples/s +Step 23190/50462 | Loss: 2.6185 | LR: 2.88e-04 | 3200.1ms/step | 18.7 samples/s +Step 23200/50462 | Loss: 2.6099 | LR: 2.88e-04 | 3199.9ms/step | 18.8 samples/s +Step 23210/50462 | Loss: 2.5963 | LR: 2.88e-04 | 3199.9ms/step | 18.8 samples/s +Step 23220/50462 | Loss: 2.6086 | LR: 2.88e-04 | 3200.0ms/step | 18.7 samples/s +Step 23230/50462 | Loss: 2.6180 | LR: 2.87e-04 | 3200.2ms/step | 18.7 samples/s +Step 23240/50462 | Loss: 2.5749 | LR: 2.87e-04 | 3251.8ms/step | 18.5 samples/s +Step 23250/50462 | Loss: 2.6116 | LR: 2.87e-04 | 3200.0ms/step | 18.7 samples/s +Validation at step 23250: Loss = 2.6142, Perplexity = 13.66 +Step 23260/50462 | Loss: 2.6223 | LR: 2.87e-04 | 4874.1ms/step | 12.3 samples/s +Step 23270/50462 | Loss: 2.5990 | LR: 2.87e-04 | 3252.1ms/step | 18.4 samples/s +Step 23280/50462 | Loss: 2.5913 | LR: 2.87e-04 | 3199.9ms/step | 18.8 samples/s +Step 23290/50462 | Loss: 2.6126 | LR: 2.86e-04 | 3200.0ms/step | 18.8 samples/s +Step 23300/50462 | Loss: 2.5973 | LR: 2.86e-04 | 3200.1ms/step | 18.7 samples/s +Step 23310/50462 | Loss: 2.6215 | LR: 2.86e-04 | 3199.9ms/step | 18.8 samples/s +Step 23320/50462 | Loss: 2.6201 | LR: 2.86e-04 | 3200.0ms/step | 18.7 samples/s +Step 23330/50462 | Loss: 2.6070 | LR: 2.86e-04 | 3256.1ms/step | 18.4 samples/s +Step 23340/50462 | Loss: 2.6054 | LR: 2.86e-04 | 3199.9ms/step | 18.8 samples/s +Step 23350/50462 | Loss: 2.6012 | LR: 2.86e-04 | 3199.8ms/step | 18.8 samples/s +Step 23360/50462 | Loss: 2.5586 | LR: 2.85e-04 | 3201.9ms/step | 18.7 samples/s +Step 23370/50462 | Loss: 2.6106 | LR: 2.85e-04 | 3200.2ms/step | 18.7 samples/s +Step 23380/50462 | Loss: 2.5949 | LR: 2.85e-04 | 3199.9ms/step | 18.8 samples/s +Step 23390/50462 | Loss: 2.5868 | LR: 2.85e-04 | 3199.9ms/step | 18.8 samples/s +Step 23400/50462 | Loss: 2.6193 | LR: 2.85e-04 | 3200.4ms/step | 18.7 samples/s +Step 23410/50462 | Loss: 2.5840 | LR: 2.85e-04 | 3199.8ms/step | 18.8 samples/s +Step 23420/50462 | Loss: 2.6137 | LR: 2.84e-04 | 3200.3ms/step | 18.7 samples/s +Step 23430/50462 | Loss: 2.5923 | LR: 2.84e-04 | 3199.9ms/step | 18.8 samples/s +Step 23440/50462 | Loss: 2.5944 | LR: 2.84e-04 | 3199.9ms/step | 18.8 samples/s +Step 23450/50462 | Loss: 2.5856 | LR: 2.84e-04 | 3199.8ms/step | 18.8 samples/s +Step 23460/50462 | Loss: 2.6328 | LR: 2.84e-04 | 3200.2ms/step | 18.7 samples/s +Step 23470/50462 | Loss: 2.6390 | LR: 2.84e-04 | 3199.9ms/step | 18.8 samples/s +Step 23480/50462 | Loss: 2.6281 | LR: 2.83e-04 | 3199.9ms/step | 18.8 samples/s +Step 23490/50462 | Loss: 2.5788 | LR: 2.83e-04 | 3200.3ms/step | 18.7 samples/s +Step 23500/50462 | Loss: 2.6237 | LR: 2.83e-04 | 3199.7ms/step | 18.8 samples/s +Validation at step 23500: Loss = 2.6123, Perplexity = 13.63 +Step 23510/50462 | Loss: 2.5830 | LR: 2.83e-04 | 4876.0ms/step | 12.3 samples/s +Step 23520/50462 | Loss: 2.6025 | LR: 2.83e-04 | 3200.2ms/step | 18.7 samples/s +Step 23530/50462 | Loss: 2.6065 | LR: 2.83e-04 | 3200.6ms/step | 18.7 samples/s +Step 23540/50462 | Loss: 2.6166 | LR: 2.83e-04 | 3201.2ms/step | 18.7 samples/s +Step 23550/50462 | Loss: 2.5749 | LR: 2.82e-04 | 3199.9ms/step | 18.8 samples/s +Step 23560/50462 | Loss: 2.5850 | LR: 2.82e-04 | 3200.2ms/step | 18.7 samples/s +Step 23570/50462 | Loss: 2.6094 | LR: 2.82e-04 | 3201.3ms/step | 18.7 samples/s +Step 23580/50462 | Loss: 2.5696 | LR: 2.82e-04 | 3250.5ms/step | 18.5 samples/s +Step 23590/50462 | Loss: 2.5896 | LR: 2.82e-04 | 3200.0ms/step | 18.7 samples/s +Step 23600/50462 | Loss: 2.6197 | LR: 2.82e-04 | 3200.1ms/step | 18.7 samples/s +Step 23610/50462 | Loss: 2.6080 | LR: 2.81e-04 | 3254.0ms/step | 18.4 samples/s +Step 23620/50462 | Loss: 2.6204 | LR: 2.81e-04 | 3200.0ms/step | 18.7 samples/s +Step 23630/50462 | Loss: 2.5889 | LR: 2.81e-04 | 3199.9ms/step | 18.8 samples/s +Step 23640/50462 | Loss: 2.6101 | LR: 2.81e-04 | 3200.2ms/step | 18.7 samples/s +Step 23650/50462 | Loss: 2.6078 | LR: 2.81e-04 | 3199.9ms/step | 18.8 samples/s +Step 23660/50462 | Loss: 2.5778 | LR: 2.81e-04 | 3195.5ms/step | 18.8 samples/s +Step 23670/50462 | Loss: 2.6064 | LR: 2.81e-04 | 3251.8ms/step | 18.5 samples/s +Step 23680/50462 | Loss: 2.5839 | LR: 2.80e-04 | 3200.4ms/step | 18.7 samples/s +Step 23690/50462 | Loss: 2.5832 | LR: 2.80e-04 | 3200.2ms/step | 18.7 samples/s +Step 23700/50462 | Loss: 2.5930 | LR: 2.80e-04 | 3200.1ms/step | 18.7 samples/s +Step 23710/50462 | Loss: 2.6268 | LR: 2.80e-04 | 3199.7ms/step | 18.8 samples/s +Step 23720/50462 | Loss: 2.5508 | LR: 2.80e-04 | 3200.2ms/step | 18.7 samples/s +Step 23730/50462 | Loss: 2.5881 | LR: 2.80e-04 | 3200.0ms/step | 18.8 samples/s +Step 23740/50462 | Loss: 2.6121 | LR: 2.79e-04 | 3200.2ms/step | 18.7 samples/s +Step 23750/50462 | Loss: 2.5837 | LR: 2.79e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 23750: Loss = 2.6101, Perplexity = 13.60 +Step 23760/50462 | Loss: 2.6306 | LR: 2.79e-04 | 4871.9ms/step | 12.3 samples/s +Step 23770/50462 | Loss: 2.5803 | LR: 2.79e-04 | 3200.0ms/step | 18.8 samples/s +Step 23780/50462 | Loss: 2.5769 | LR: 2.79e-04 | 3201.9ms/step | 18.7 samples/s +Step 23790/50462 | Loss: 2.5622 | LR: 2.79e-04 | 3202.4ms/step | 18.7 samples/s +Step 23800/50462 | Loss: 2.5946 | LR: 2.78e-04 | 3199.8ms/step | 18.8 samples/s +Step 23810/50462 | Loss: 2.5907 | LR: 2.78e-04 | 3200.2ms/step | 18.7 samples/s +Step 23820/50462 | Loss: 2.6016 | LR: 2.78e-04 | 3201.8ms/step | 18.7 samples/s +Step 23830/50462 | Loss: 2.5662 | LR: 2.78e-04 | 3200.2ms/step | 18.7 samples/s +Step 23840/50462 | Loss: 2.5974 | LR: 2.78e-04 | 3200.0ms/step | 18.7 samples/s +Step 23850/50462 | Loss: 2.6108 | LR: 2.78e-04 | 3199.8ms/step | 18.8 samples/s +Step 23860/50462 | Loss: 2.6184 | LR: 2.78e-04 | 3200.2ms/step | 18.7 samples/s +Step 23870/50462 | Loss: 2.6116 | LR: 2.77e-04 | 3200.2ms/step | 18.7 samples/s +Step 23880/50462 | Loss: 2.6199 | LR: 2.77e-04 | 3199.7ms/step | 18.8 samples/s +Step 23890/50462 | Loss: 2.5968 | LR: 2.77e-04 | 3200.1ms/step | 18.7 samples/s +Step 23900/50462 | Loss: 2.6041 | LR: 2.77e-04 | 3199.9ms/step | 18.8 samples/s +Step 23910/50462 | Loss: 2.5885 | LR: 2.77e-04 | 3199.9ms/step | 18.8 samples/s +Step 23920/50462 | Loss: 2.5840 | LR: 2.77e-04 | 3252.2ms/step | 18.4 samples/s +Step 23930/50462 | Loss: 2.6000 | LR: 2.76e-04 | 3195.5ms/step | 18.8 samples/s +Step 23940/50462 | Loss: 2.5935 | LR: 2.76e-04 | 3200.5ms/step | 18.7 samples/s +Step 23950/50462 | Loss: 2.5941 | LR: 2.76e-04 | 3255.4ms/step | 18.4 samples/s +Step 23960/50462 | Loss: 2.5945 | LR: 2.76e-04 | 3200.7ms/step | 18.7 samples/s +Step 23970/50462 | Loss: 2.5635 | LR: 2.76e-04 | 3199.8ms/step | 18.8 samples/s +Step 23980/50462 | Loss: 2.6269 | LR: 2.76e-04 | 3199.9ms/step | 18.8 samples/s +Step 23990/50462 | Loss: 2.6024 | LR: 2.75e-04 | 3200.2ms/step | 18.7 samples/s +Step 24000/50462 | Loss: 2.6027 | LR: 2.75e-04 | 3254.4ms/step | 18.4 samples/s +Validation at step 24000: Loss = 2.6079, Perplexity = 13.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-24000 +Step 24010/50462 | Loss: 2.5922 | LR: 2.75e-04 | 5765.5ms/step | 10.4 samples/s +Step 24020/50462 | Loss: 2.5917 | LR: 2.75e-04 | 3185.9ms/step | 18.8 samples/s +Step 24030/50462 | Loss: 2.5685 | LR: 2.75e-04 | 3200.1ms/step | 18.7 samples/s +Step 24040/50462 | Loss: 2.6147 | LR: 2.75e-04 | 3200.2ms/step | 18.7 samples/s +Step 24050/50462 | Loss: 2.5758 | LR: 2.75e-04 | 3199.9ms/step | 18.8 samples/s +Step 24060/50462 | Loss: 2.5989 | LR: 2.74e-04 | 3200.0ms/step | 18.8 samples/s +Step 24070/50462 | Loss: 2.6116 | LR: 2.74e-04 | 3200.0ms/step | 18.8 samples/s +Step 24080/50462 | Loss: 2.6144 | LR: 2.74e-04 | 3199.9ms/step | 18.8 samples/s +Step 24090/50462 | Loss: 2.5938 | LR: 2.74e-04 | 3200.0ms/step | 18.8 samples/s +Step 24100/50462 | Loss: 2.5715 | LR: 2.74e-04 | 3200.2ms/step | 18.7 samples/s +Step 24110/50462 | Loss: 2.5949 | LR: 2.74e-04 | 3199.9ms/step | 18.8 samples/s +Step 24120/50462 | Loss: 2.6081 | LR: 2.73e-04 | 3200.0ms/step | 18.7 samples/s +Step 24130/50462 | Loss: 2.6019 | LR: 2.73e-04 | 3199.9ms/step | 18.8 samples/s +Step 24140/50462 | Loss: 2.5991 | LR: 2.73e-04 | 3200.9ms/step | 18.7 samples/s +Step 24150/50462 | Loss: 2.5928 | LR: 2.73e-04 | 3199.1ms/step | 18.8 samples/s +Step 24160/50462 | Loss: 2.6078 | LR: 2.73e-04 | 3202.2ms/step | 18.7 samples/s +Step 24170/50462 | Loss: 2.5695 | LR: 2.73e-04 | 3199.9ms/step | 18.8 samples/s +Step 24180/50462 | Loss: 2.5874 | LR: 2.72e-04 | 3199.9ms/step | 18.8 samples/s +Step 24190/50462 | Loss: 2.5936 | LR: 2.72e-04 | 3200.2ms/step | 18.7 samples/s +Step 24200/50462 | Loss: 2.5754 | LR: 2.72e-04 | 3199.9ms/step | 18.8 samples/s +Step 24210/50462 | Loss: 2.5914 | LR: 2.72e-04 | 3199.9ms/step | 18.8 samples/s +Step 24220/50462 | Loss: 2.5617 | LR: 2.72e-04 | 3200.1ms/step | 18.7 samples/s +Step 24230/50462 | Loss: 2.6112 | LR: 2.72e-04 | 3200.0ms/step | 18.8 samples/s +Step 24240/50462 | Loss: 2.6003 | LR: 2.72e-04 | 3200.0ms/step | 18.8 samples/s +Step 24250/50462 | Loss: 2.5865 | LR: 2.71e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 24250: Loss = 2.6058, Perplexity = 13.54 +Step 24260/50462 | Loss: 2.5931 | LR: 2.71e-04 | 4928.1ms/step | 12.2 samples/s +Step 24270/50462 | Loss: 2.5947 | LR: 2.71e-04 | 3199.8ms/step | 18.8 samples/s +Step 24280/50462 | Loss: 2.6042 | LR: 2.71e-04 | 3199.9ms/step | 18.8 samples/s +Step 24290/50462 | Loss: 2.5842 | LR: 2.71e-04 | 3251.7ms/step | 18.5 samples/s +Step 24300/50462 | Loss: 2.5772 | LR: 2.71e-04 | 3204.4ms/step | 18.7 samples/s +Step 24310/50462 | Loss: 2.5971 | LR: 2.70e-04 | 3199.8ms/step | 18.8 samples/s +Step 24320/50462 | Loss: 2.5947 | LR: 2.70e-04 | 3200.1ms/step | 18.7 samples/s +Step 24330/50462 | Loss: 2.6169 | LR: 2.70e-04 | 3200.0ms/step | 18.8 samples/s +Step 24340/50462 | Loss: 2.5615 | LR: 2.70e-04 | 3254.2ms/step | 18.4 samples/s +Step 24350/50462 | Loss: 2.5982 | LR: 2.70e-04 | 3193.8ms/step | 18.8 samples/s +Step 24360/50462 | Loss: 2.5879 | LR: 2.70e-04 | 3199.9ms/step | 18.8 samples/s +Step 24370/50462 | Loss: 2.5921 | LR: 2.70e-04 | 3200.0ms/step | 18.8 samples/s +Step 24380/50462 | Loss: 2.6162 | LR: 2.69e-04 | 3200.2ms/step | 18.7 samples/s +Step 24390/50462 | Loss: 2.5753 | LR: 2.69e-04 | 3200.1ms/step | 18.7 samples/s +Step 24400/50462 | Loss: 2.5977 | LR: 2.69e-04 | 3200.0ms/step | 18.8 samples/s +Step 24410/50462 | Loss: 2.5950 | LR: 2.69e-04 | 3199.8ms/step | 18.8 samples/s +Step 24420/50462 | Loss: 2.5921 | LR: 2.69e-04 | 3200.0ms/step | 18.7 samples/s +Step 24430/50462 | Loss: 2.5697 | LR: 2.69e-04 | 3200.1ms/step | 18.7 samples/s +Step 24440/50462 | Loss: 2.6322 | LR: 2.68e-04 | 3199.9ms/step | 18.8 samples/s +Step 24450/50462 | Loss: 2.6041 | LR: 2.68e-04 | 3200.0ms/step | 18.7 samples/s +Step 24460/50462 | Loss: 2.5792 | LR: 2.68e-04 | 3200.5ms/step | 18.7 samples/s +Step 24470/50462 | Loss: 2.5737 | LR: 2.68e-04 | 3199.4ms/step | 18.8 samples/s +Step 24480/50462 | Loss: 2.5863 | LR: 2.68e-04 | 3199.7ms/step | 18.8 samples/s +Step 24490/50462 | Loss: 2.6109 | LR: 2.68e-04 | 3200.4ms/step | 18.7 samples/s +Step 24500/50462 | Loss: 2.6039 | LR: 2.67e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 24500: Loss = 2.6034, Perplexity = 13.51 +Step 24510/50462 | Loss: 2.5898 | LR: 2.67e-04 | 4875.8ms/step | 12.3 samples/s +Step 24520/50462 | Loss: 2.6143 | LR: 2.67e-04 | 3200.0ms/step | 18.8 samples/s +Step 24530/50462 | Loss: 2.5927 | LR: 2.67e-04 | 3200.1ms/step | 18.7 samples/s +Step 24540/50462 | Loss: 2.6183 | LR: 2.67e-04 | 3199.8ms/step | 18.8 samples/s +Step 24550/50462 | Loss: 2.5914 | LR: 2.67e-04 | 3200.1ms/step | 18.7 samples/s +Step 24560/50462 | Loss: 2.6412 | LR: 2.67e-04 | 3200.0ms/step | 18.8 samples/s +Step 24570/50462 | Loss: 2.5932 | LR: 2.66e-04 | 3200.1ms/step | 18.7 samples/s +Step 24580/50462 | Loss: 2.6031 | LR: 2.66e-04 | 3199.8ms/step | 18.8 samples/s +Step 24590/50462 | Loss: 2.6067 | LR: 2.66e-04 | 3200.0ms/step | 18.7 samples/s +Step 24600/50462 | Loss: 2.6097 | LR: 2.66e-04 | 3252.1ms/step | 18.4 samples/s +Step 24610/50462 | Loss: 2.5750 | LR: 2.66e-04 | 3199.8ms/step | 18.8 samples/s +Step 24620/50462 | Loss: 2.6187 | LR: 2.66e-04 | 3200.0ms/step | 18.7 samples/s +Step 24630/50462 | Loss: 2.5656 | LR: 2.65e-04 | 3253.7ms/step | 18.4 samples/s +Step 24640/50462 | Loss: 2.6076 | LR: 2.65e-04 | 3200.5ms/step | 18.7 samples/s +Step 24650/50462 | Loss: 2.5670 | LR: 2.65e-04 | 3197.3ms/step | 18.8 samples/s +Step 24660/50462 | Loss: 2.6075 | LR: 2.65e-04 | 3184.1ms/step | 18.8 samples/s +Step 24670/50462 | Loss: 2.5964 | LR: 2.65e-04 | 3196.6ms/step | 18.8 samples/s +Step 24680/50462 | Loss: 2.6004 | LR: 2.65e-04 | 3254.2ms/step | 18.4 samples/s +Step 24690/50462 | Loss: 2.5935 | LR: 2.64e-04 | 3199.8ms/step | 18.8 samples/s +Step 24700/50462 | Loss: 2.6159 | LR: 2.64e-04 | 3200.1ms/step | 18.7 samples/s +Step 24710/50462 | Loss: 2.5896 | LR: 2.64e-04 | 3199.9ms/step | 18.8 samples/s +Step 24720/50462 | Loss: 2.5663 | LR: 2.64e-04 | 3199.9ms/step | 18.8 samples/s +Step 24730/50462 | Loss: 2.5809 | LR: 2.64e-04 | 3200.0ms/step | 18.7 samples/s +Step 24740/50462 | Loss: 2.6070 | LR: 2.64e-04 | 3200.1ms/step | 18.7 samples/s +Step 24750/50462 | Loss: 2.5840 | LR: 2.64e-04 | 3200.0ms/step | 18.7 samples/s +Validation at step 24750: Loss = 2.6010, Perplexity = 13.48 +Step 24760/50462 | Loss: 2.6049 | LR: 2.63e-04 | 4875.8ms/step | 12.3 samples/s +Step 24770/50462 | Loss: 2.5720 | LR: 2.63e-04 | 3200.1ms/step | 18.7 samples/s +Step 24780/50462 | Loss: 2.5765 | LR: 2.63e-04 | 3200.2ms/step | 18.7 samples/s +Step 24790/50462 | Loss: 2.5875 | LR: 2.63e-04 | 3199.6ms/step | 18.8 samples/s +Step 24800/50462 | Loss: 2.6127 | LR: 2.63e-04 | 3200.5ms/step | 18.7 samples/s +Step 24810/50462 | Loss: 2.6045 | LR: 2.63e-04 | 3200.0ms/step | 18.8 samples/s +Step 24820/50462 | Loss: 2.6024 | LR: 2.62e-04 | 3199.7ms/step | 18.8 samples/s +Step 24830/50462 | Loss: 2.5936 | LR: 2.62e-04 | 3200.2ms/step | 18.7 samples/s +Step 24840/50462 | Loss: 2.5678 | LR: 2.62e-04 | 3199.7ms/step | 18.8 samples/s +Step 24850/50462 | Loss: 2.5821 | LR: 2.62e-04 | 3200.3ms/step | 18.7 samples/s +Step 24860/50462 | Loss: 2.6103 | LR: 2.62e-04 | 3199.7ms/step | 18.8 samples/s +Step 24870/50462 | Loss: 2.5988 | LR: 2.62e-04 | 3200.3ms/step | 18.7 samples/s +Step 24880/50462 | Loss: 2.5433 | LR: 2.61e-04 | 3199.9ms/step | 18.8 samples/s +Step 24890/50462 | Loss: 2.5580 | LR: 2.61e-04 | 3199.9ms/step | 18.8 samples/s +Step 24900/50462 | Loss: 2.6132 | LR: 2.61e-04 | 3199.6ms/step | 18.8 samples/s +Step 24910/50462 | Loss: 2.5542 | LR: 2.61e-04 | 3200.3ms/step | 18.7 samples/s +Step 24920/50462 | Loss: 2.5616 | LR: 2.61e-04 | 3200.0ms/step | 18.7 samples/s +Step 24930/50462 | Loss: 2.5763 | LR: 2.61e-04 | 3200.1ms/step | 18.7 samples/s +Step 24940/50462 | Loss: 2.6083 | LR: 2.61e-04 | 3251.8ms/step | 18.5 samples/s +Step 24950/50462 | Loss: 2.6017 | LR: 2.60e-04 | 3200.2ms/step | 18.7 samples/s +Step 24960/50462 | Loss: 2.5593 | LR: 2.60e-04 | 3199.8ms/step | 18.8 samples/s +Step 24970/50462 | Loss: 2.5619 | LR: 2.60e-04 | 3251.7ms/step | 18.5 samples/s +Step 24980/50462 | Loss: 2.5972 | LR: 2.60e-04 | 3200.5ms/step | 18.7 samples/s +Step 24990/50462 | Loss: 2.5763 | LR: 2.60e-04 | 3199.6ms/step | 18.8 samples/s +Step 25000/50462 | Loss: 2.5963 | LR: 2.60e-04 | 3200.4ms/step | 18.7 samples/s +Validation at step 25000: Loss = 2.5989, Perplexity = 13.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-25000 +Step 25010/50462 | Loss: 2.5791 | LR: 2.59e-04 | 5827.4ms/step | 10.3 samples/s +Step 25020/50462 | Loss: 2.5855 | LR: 2.59e-04 | 3192.5ms/step | 18.8 samples/s +Step 25030/50462 | Loss: 2.5753 | LR: 2.59e-04 | 3198.0ms/step | 18.8 samples/s +Step 25040/50462 | Loss: 2.5966 | LR: 2.59e-04 | 3200.0ms/step | 18.7 samples/s +Step 25050/50462 | Loss: 2.6069 | LR: 2.59e-04 | 3199.6ms/step | 18.8 samples/s +Step 25060/50462 | Loss: 2.5868 | LR: 2.59e-04 | 3200.4ms/step | 18.7 samples/s +Step 25070/50462 | Loss: 2.5700 | LR: 2.58e-04 | 3199.9ms/step | 18.8 samples/s +Step 25080/50462 | Loss: 2.5606 | LR: 2.58e-04 | 3202.1ms/step | 18.7 samples/s +Step 25090/50462 | Loss: 2.5820 | LR: 2.58e-04 | 3199.8ms/step | 18.8 samples/s +Step 25100/50462 | Loss: 2.5895 | LR: 2.58e-04 | 3200.1ms/step | 18.7 samples/s +Step 25110/50462 | Loss: 2.5807 | LR: 2.58e-04 | 3199.9ms/step | 18.8 samples/s +Step 25120/50462 | Loss: 2.6056 | LR: 2.58e-04 | 3200.1ms/step | 18.7 samples/s +Step 25130/50462 | Loss: 2.6089 | LR: 2.58e-04 | 3199.9ms/step | 18.8 samples/s +Step 25140/50462 | Loss: 2.5937 | LR: 2.57e-04 | 3200.3ms/step | 18.7 samples/s +Step 25150/50462 | Loss: 2.5628 | LR: 2.57e-04 | 3199.9ms/step | 18.8 samples/s +Step 25160/50462 | Loss: 2.5647 | LR: 2.57e-04 | 3199.9ms/step | 18.8 samples/s +Step 25170/50462 | Loss: 2.5797 | LR: 2.57e-04 | 3199.9ms/step | 18.8 samples/s +Step 25180/50462 | Loss: 2.5826 | LR: 2.57e-04 | 3200.4ms/step | 18.7 samples/s +Step 25190/50462 | Loss: 2.5774 | LR: 2.57e-04 | 3201.6ms/step | 18.7 samples/s +Step 25200/50462 | Loss: 2.6109 | LR: 2.56e-04 | 3200.0ms/step | 18.7 samples/s +Step 25210/50462 | Loss: 2.5909 | LR: 2.56e-04 | 3200.3ms/step | 18.7 samples/s +Step 25220/50462 | Loss: 2.6285 | LR: 2.56e-04 | 3199.8ms/step | 18.8 samples/s +Step 25230/50462 | Loss: 2.6102 | LR: 2.56e-04 | 3199.9ms/step | 18.8 samples/s +Step 25240/50462 | Loss: 2.5549 | LR: 2.56e-04 | 3200.2ms/step | 18.7 samples/s +Step 25250/50462 | Loss: 2.5915 | LR: 2.56e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 25250: Loss = 2.5967, Perplexity = 13.42 +Step 25260/50462 | Loss: 2.5651 | LR: 2.55e-04 | 4879.9ms/step | 12.3 samples/s +Step 25270/50462 | Loss: 2.5897 | LR: 2.55e-04 | 3200.0ms/step | 18.8 samples/s +Step 25280/50462 | Loss: 2.6182 | LR: 2.55e-04 | 3260.0ms/step | 18.4 samples/s +Step 25290/50462 | Loss: 2.5880 | LR: 2.55e-04 | 3196.2ms/step | 18.8 samples/s +Step 25300/50462 | Loss: 2.5727 | LR: 2.55e-04 | 3199.9ms/step | 18.8 samples/s +Step 25310/50462 | Loss: 2.5616 | LR: 2.55e-04 | 3200.1ms/step | 18.7 samples/s +Step 25320/50462 | Loss: 2.5946 | LR: 2.55e-04 | 3248.2ms/step | 18.5 samples/s +Step 25330/50462 | Loss: 2.5683 | LR: 2.54e-04 | 3197.9ms/step | 18.8 samples/s +Step 25340/50462 | Loss: 2.6020 | LR: 2.54e-04 | 3199.9ms/step | 18.8 samples/s +Step 25350/50462 | Loss: 2.5429 | LR: 2.54e-04 | 3263.9ms/step | 18.4 samples/s +Step 25360/50462 | Loss: 2.5927 | LR: 2.54e-04 | 3200.3ms/step | 18.7 samples/s +Step 25370/50462 | Loss: 2.5951 | LR: 2.54e-04 | 3199.6ms/step | 18.8 samples/s +Step 25380/50462 | Loss: 2.6348 | LR: 2.54e-04 | 3200.0ms/step | 18.7 samples/s +Step 25390/50462 | Loss: 2.5960 | LR: 2.53e-04 | 3200.0ms/step | 18.7 samples/s +Step 25400/50462 | Loss: 2.5720 | LR: 2.53e-04 | 3200.0ms/step | 18.7 samples/s +Step 25410/50462 | Loss: 2.5646 | LR: 2.53e-04 | 3199.9ms/step | 18.8 samples/s +Step 25420/50462 | Loss: 2.5906 | LR: 2.53e-04 | 3200.1ms/step | 18.7 samples/s +Step 25430/50462 | Loss: 2.5797 | LR: 2.53e-04 | 3200.2ms/step | 18.7 samples/s +Step 25440/50462 | Loss: 2.5723 | LR: 2.53e-04 | 3199.8ms/step | 18.8 samples/s +Step 25450/50462 | Loss: 2.5772 | LR: 2.52e-04 | 3199.9ms/step | 18.8 samples/s +Step 25460/50462 | Loss: 2.5874 | LR: 2.52e-04 | 3200.2ms/step | 18.7 samples/s +Step 25470/50462 | Loss: 2.5859 | LR: 2.52e-04 | 3200.1ms/step | 18.7 samples/s +Step 25480/50462 | Loss: 2.5727 | LR: 2.52e-04 | 3199.8ms/step | 18.8 samples/s +Step 25490/50462 | Loss: 2.5753 | LR: 2.52e-04 | 3200.0ms/step | 18.7 samples/s +Step 25500/50462 | Loss: 2.5764 | LR: 2.52e-04 | 3200.0ms/step | 18.7 samples/s +Validation at step 25500: Loss = 2.5945, Perplexity = 13.39 +Step 25510/50462 | Loss: 2.5713 | LR: 2.52e-04 | 4875.9ms/step | 12.3 samples/s +Step 25520/50462 | Loss: 2.5793 | LR: 2.51e-04 | 3200.2ms/step | 18.7 samples/s +Step 25530/50462 | Loss: 2.6072 | LR: 2.51e-04 | 3199.9ms/step | 18.8 samples/s +Step 25540/50462 | Loss: 2.6028 | LR: 2.51e-04 | 3200.1ms/step | 18.7 samples/s +Step 25550/50462 | Loss: 2.5954 | LR: 2.51e-04 | 3200.1ms/step | 18.7 samples/s +Step 25560/50462 | Loss: 2.5933 | LR: 2.51e-04 | 3199.9ms/step | 18.8 samples/s +Step 25570/50462 | Loss: 2.5978 | LR: 2.51e-04 | 3200.2ms/step | 18.7 samples/s +Step 25580/50462 | Loss: 2.6207 | LR: 2.50e-04 | 3199.7ms/step | 18.8 samples/s +Step 25590/50462 | Loss: 2.6105 | LR: 2.50e-04 | 3200.2ms/step | 18.7 samples/s +Step 25600/50462 | Loss: 2.5780 | LR: 2.50e-04 | 3199.7ms/step | 18.8 samples/s +Step 25610/50462 | Loss: 2.5944 | LR: 2.50e-04 | 3200.2ms/step | 18.7 samples/s +Step 25620/50462 | Loss: 2.5821 | LR: 2.50e-04 | 3251.8ms/step | 18.5 samples/s +Step 25630/50462 | Loss: 2.5758 | LR: 2.50e-04 | 3200.1ms/step | 18.7 samples/s +Step 25640/50462 | Loss: 2.6084 | LR: 2.49e-04 | 3200.3ms/step | 18.7 samples/s +Step 25650/50462 | Loss: 2.5720 | LR: 2.49e-04 | 3200.0ms/step | 18.7 samples/s +Step 25660/50462 | Loss: 2.5481 | LR: 2.49e-04 | 3251.6ms/step | 18.5 samples/s +Step 25670/50462 | Loss: 2.5754 | LR: 2.49e-04 | 3200.2ms/step | 18.7 samples/s +Step 25680/50462 | Loss: 2.5938 | LR: 2.49e-04 | 3200.1ms/step | 18.7 samples/s +Step 25690/50462 | Loss: 2.5958 | LR: 2.49e-04 | 3253.9ms/step | 18.4 samples/s +Step 25700/50462 | Loss: 2.5856 | LR: 2.49e-04 | 3200.1ms/step | 18.7 samples/s +Step 25710/50462 | Loss: 2.5611 | LR: 2.48e-04 | 3199.8ms/step | 18.8 samples/s +Step 25720/50462 | Loss: 2.5631 | LR: 2.48e-04 | 3200.1ms/step | 18.7 samples/s +Step 25730/50462 | Loss: 2.5957 | LR: 2.48e-04 | 3200.1ms/step | 18.7 samples/s +Step 25740/50462 | Loss: 2.5904 | LR: 2.48e-04 | 3199.6ms/step | 18.8 samples/s +Step 25750/50462 | Loss: 2.6004 | LR: 2.48e-04 | 3200.4ms/step | 18.7 samples/s +Validation at step 25750: Loss = 2.5928, Perplexity = 13.37 +Step 25760/50462 | Loss: 2.5765 | LR: 2.48e-04 | 4875.8ms/step | 12.3 samples/s +Step 25770/50462 | Loss: 2.5808 | LR: 2.47e-04 | 3200.0ms/step | 18.7 samples/s +Step 25780/50462 | Loss: 2.6082 | LR: 2.47e-04 | 3200.2ms/step | 18.7 samples/s +Step 25790/50462 | Loss: 2.5785 | LR: 2.47e-04 | 3199.9ms/step | 18.8 samples/s +Step 25800/50462 | Loss: 2.5676 | LR: 2.47e-04 | 3200.0ms/step | 18.8 samples/s +Step 25810/50462 | Loss: 2.5783 | LR: 2.47e-04 | 3200.0ms/step | 18.7 samples/s +Step 25820/50462 | Loss: 2.5808 | LR: 2.47e-04 | 3199.9ms/step | 18.8 samples/s +Step 25830/50462 | Loss: 2.5937 | LR: 2.46e-04 | 3200.2ms/step | 18.7 samples/s +Step 25840/50462 | Loss: 2.6126 | LR: 2.46e-04 | 3199.8ms/step | 18.8 samples/s +Step 25850/50462 | Loss: 2.5823 | LR: 2.46e-04 | 3200.2ms/step | 18.7 samples/s +Step 25860/50462 | Loss: 2.5723 | LR: 2.46e-04 | 3199.9ms/step | 18.8 samples/s +Step 25870/50462 | Loss: 2.5881 | LR: 2.46e-04 | 3200.1ms/step | 18.7 samples/s +Step 25880/50462 | Loss: 2.5924 | LR: 2.46e-04 | 3201.8ms/step | 18.7 samples/s +Step 25890/50462 | Loss: 2.5642 | LR: 2.46e-04 | 3199.9ms/step | 18.8 samples/s +Step 25900/50462 | Loss: 2.5546 | LR: 2.45e-04 | 3200.4ms/step | 18.7 samples/s +Step 25910/50462 | Loss: 2.5570 | LR: 2.45e-04 | 3199.9ms/step | 18.8 samples/s +Step 25920/50462 | Loss: 2.5898 | LR: 2.45e-04 | 3200.0ms/step | 18.7 samples/s +Step 25930/50462 | Loss: 2.5797 | LR: 2.45e-04 | 3199.9ms/step | 18.8 samples/s +Step 25940/50462 | Loss: 2.5755 | LR: 2.45e-04 | 3200.1ms/step | 18.7 samples/s +Step 25950/50462 | Loss: 2.6019 | LR: 2.45e-04 | 3200.3ms/step | 18.7 samples/s +Step 25960/50462 | Loss: 2.5866 | LR: 2.44e-04 | 3253.6ms/step | 18.4 samples/s +Step 25970/50462 | Loss: 2.5814 | LR: 2.44e-04 | 3199.5ms/step | 18.8 samples/s +Step 25980/50462 | Loss: 2.5659 | LR: 2.44e-04 | 3200.5ms/step | 18.7 samples/s +Step 25990/50462 | Loss: 2.5691 | LR: 2.44e-04 | 3200.0ms/step | 18.8 samples/s +Step 26000/50462 | Loss: 2.5674 | LR: 2.44e-04 | 3252.0ms/step | 18.5 samples/s +Validation at step 26000: Loss = 2.5908, Perplexity = 13.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-26000 +Step 26010/50462 | Loss: 2.6015 | LR: 2.44e-04 | 5796.0ms/step | 10.4 samples/s +Step 26020/50462 | Loss: 2.6011 | LR: 2.43e-04 | 3244.7ms/step | 18.5 samples/s +Step 26030/50462 | Loss: 2.5859 | LR: 2.43e-04 | 3192.7ms/step | 18.8 samples/s +Step 26040/50462 | Loss: 2.5664 | LR: 2.43e-04 | 3198.6ms/step | 18.8 samples/s +Step 26050/50462 | Loss: 2.5655 | LR: 2.43e-04 | 3199.9ms/step | 18.8 samples/s +Step 26060/50462 | Loss: 2.5697 | LR: 2.43e-04 | 3200.0ms/step | 18.8 samples/s +Step 26070/50462 | Loss: 2.6022 | LR: 2.43e-04 | 3199.2ms/step | 18.8 samples/s +Step 26080/50462 | Loss: 2.5540 | LR: 2.43e-04 | 3200.9ms/step | 18.7 samples/s +Step 26090/50462 | Loss: 2.5923 | LR: 2.42e-04 | 3199.8ms/step | 18.8 samples/s +Step 26100/50462 | Loss: 2.5939 | LR: 2.42e-04 | 3199.5ms/step | 18.8 samples/s +Step 26110/50462 | Loss: 2.5959 | LR: 2.42e-04 | 3202.7ms/step | 18.7 samples/s +Step 26120/50462 | Loss: 2.6003 | LR: 2.42e-04 | 3199.9ms/step | 18.8 samples/s +Step 26130/50462 | Loss: 2.5710 | LR: 2.42e-04 | 3200.2ms/step | 18.7 samples/s +Step 26140/50462 | Loss: 2.5841 | LR: 2.42e-04 | 3199.9ms/step | 18.8 samples/s +Step 26150/50462 | Loss: 2.5794 | LR: 2.41e-04 | 3199.9ms/step | 18.8 samples/s +Step 26160/50462 | Loss: 2.5714 | LR: 2.41e-04 | 3200.0ms/step | 18.7 samples/s +Step 26170/50462 | Loss: 2.5891 | LR: 2.41e-04 | 3200.0ms/step | 18.7 samples/s +Step 26180/50462 | Loss: 2.5797 | LR: 2.41e-04 | 3200.0ms/step | 18.7 samples/s +Step 26190/50462 | Loss: 2.5678 | LR: 2.41e-04 | 3199.9ms/step | 18.8 samples/s +Step 26200/50462 | Loss: 2.5787 | LR: 2.41e-04 | 3199.8ms/step | 18.8 samples/s +Step 26210/50462 | Loss: 2.5635 | LR: 2.40e-04 | 3200.2ms/step | 18.7 samples/s +Step 26220/50462 | Loss: 2.5714 | LR: 2.40e-04 | 3199.7ms/step | 18.8 samples/s +Step 26230/50462 | Loss: 2.5763 | LR: 2.40e-04 | 3200.3ms/step | 18.7 samples/s +Step 26240/50462 | Loss: 2.5851 | LR: 2.40e-04 | 3199.9ms/step | 18.8 samples/s +Step 26250/50462 | Loss: 2.5953 | LR: 2.40e-04 | 3200.3ms/step | 18.7 samples/s +Validation at step 26250: Loss = 2.5888, Perplexity = 13.31 +Step 26260/50462 | Loss: 2.5812 | LR: 2.40e-04 | 4877.7ms/step | 12.3 samples/s +Step 26270/50462 | Loss: 2.5477 | LR: 2.40e-04 | 3199.8ms/step | 18.8 samples/s +Step 26280/50462 | Loss: 2.5698 | LR: 2.39e-04 | 3200.3ms/step | 18.7 samples/s +Step 26290/50462 | Loss: 2.5796 | LR: 2.39e-04 | 3199.9ms/step | 18.8 samples/s +Step 26300/50462 | Loss: 2.5808 | LR: 2.39e-04 | 3251.4ms/step | 18.5 samples/s +Step 26310/50462 | Loss: 2.5716 | LR: 2.39e-04 | 3200.8ms/step | 18.7 samples/s +Step 26320/50462 | Loss: 2.5830 | LR: 2.39e-04 | 3200.0ms/step | 18.8 samples/s +Step 26330/50462 | Loss: 2.5808 | LR: 2.39e-04 | 3199.8ms/step | 18.8 samples/s +Step 26340/50462 | Loss: 2.5965 | LR: 2.38e-04 | 3254.1ms/step | 18.4 samples/s +Step 26350/50462 | Loss: 2.5960 | LR: 2.38e-04 | 3199.7ms/step | 18.8 samples/s +Step 26360/50462 | Loss: 2.5892 | LR: 2.38e-04 | 3252.3ms/step | 18.4 samples/s +Step 26370/50462 | Loss: 2.5635 | LR: 2.38e-04 | 3194.0ms/step | 18.8 samples/s +Step 26380/50462 | Loss: 2.5360 | LR: 2.38e-04 | 3198.0ms/step | 18.8 samples/s +Step 26390/50462 | Loss: 2.5741 | LR: 2.38e-04 | 3199.9ms/step | 18.8 samples/s +Step 26400/50462 | Loss: 2.5778 | LR: 2.37e-04 | 3199.9ms/step | 18.8 samples/s +Step 26410/50462 | Loss: 2.5881 | LR: 2.37e-04 | 3200.4ms/step | 18.7 samples/s +Step 26420/50462 | Loss: 2.5752 | LR: 2.37e-04 | 3199.7ms/step | 18.8 samples/s +Step 26430/50462 | Loss: 2.5723 | LR: 2.37e-04 | 3200.1ms/step | 18.7 samples/s +Step 26440/50462 | Loss: 2.5926 | LR: 2.37e-04 | 3200.1ms/step | 18.7 samples/s +Step 26450/50462 | Loss: 2.5892 | LR: 2.37e-04 | 3199.9ms/step | 18.8 samples/s +Step 26460/50462 | Loss: 2.5769 | LR: 2.37e-04 | 3200.1ms/step | 18.7 samples/s +Step 26470/50462 | Loss: 2.6140 | LR: 2.36e-04 | 3200.0ms/step | 18.7 samples/s +Step 26480/50462 | Loss: 2.5630 | LR: 2.36e-04 | 3200.0ms/step | 18.7 samples/s +Step 26490/50462 | Loss: 2.5710 | LR: 2.36e-04 | 3199.9ms/step | 18.8 samples/s +Step 26500/50462 | Loss: 2.5775 | LR: 2.36e-04 | 3200.0ms/step | 18.7 samples/s +Validation at step 26500: Loss = 2.5868, Perplexity = 13.29 +Step 26510/50462 | Loss: 2.5850 | LR: 2.36e-04 | 4876.3ms/step | 12.3 samples/s +Step 26520/50462 | Loss: 2.5848 | LR: 2.36e-04 | 3199.8ms/step | 18.8 samples/s +Step 26530/50462 | Loss: 2.5722 | LR: 2.35e-04 | 3200.1ms/step | 18.7 samples/s +Step 26540/50462 | Loss: 2.5535 | LR: 2.35e-04 | 3200.0ms/step | 18.7 samples/s +Step 26550/50462 | Loss: 2.6011 | LR: 2.35e-04 | 3199.9ms/step | 18.8 samples/s +Step 26560/50462 | Loss: 2.5961 | LR: 2.35e-04 | 3200.0ms/step | 18.8 samples/s +Step 26570/50462 | Loss: 2.5768 | LR: 2.35e-04 | 3199.9ms/step | 18.8 samples/s +Step 26580/50462 | Loss: 2.5756 | LR: 2.35e-04 | 3200.0ms/step | 18.7 samples/s +Step 26590/50462 | Loss: 2.5583 | LR: 2.34e-04 | 3200.1ms/step | 18.7 samples/s +Step 26600/50462 | Loss: 2.5581 | LR: 2.34e-04 | 3199.4ms/step | 18.8 samples/s +Step 26610/50462 | Loss: 2.6046 | LR: 2.34e-04 | 3200.4ms/step | 18.7 samples/s +Step 26620/50462 | Loss: 2.5730 | LR: 2.34e-04 | 3200.0ms/step | 18.8 samples/s +Step 26630/50462 | Loss: 2.5784 | LR: 2.34e-04 | 3199.7ms/step | 18.8 samples/s +Step 26640/50462 | Loss: 2.5832 | LR: 2.34e-04 | 3252.2ms/step | 18.4 samples/s +Step 26650/50462 | Loss: 2.5695 | LR: 2.34e-04 | 3200.4ms/step | 18.7 samples/s +Step 26660/50462 | Loss: 2.5447 | LR: 2.33e-04 | 3198.5ms/step | 18.8 samples/s +Step 26670/50462 | Loss: 2.5416 | LR: 2.33e-04 | 3192.0ms/step | 18.8 samples/s +Step 26680/50462 | Loss: 2.5380 | LR: 2.33e-04 | 3249.3ms/step | 18.5 samples/s +Step 26690/50462 | Loss: 2.5619 | LR: 2.33e-04 | 3199.9ms/step | 18.8 samples/s +Step 26700/50462 | Loss: 2.5766 | LR: 2.33e-04 | 3252.1ms/step | 18.4 samples/s +Step 26710/50462 | Loss: 2.5811 | LR: 2.33e-04 | 3191.7ms/step | 18.8 samples/s +Step 26720/50462 | Loss: 2.5266 | LR: 2.32e-04 | 3200.2ms/step | 18.7 samples/s +Step 26730/50462 | Loss: 2.5587 | LR: 2.32e-04 | 3198.1ms/step | 18.8 samples/s +Step 26740/50462 | Loss: 2.5755 | LR: 2.32e-04 | 3199.9ms/step | 18.8 samples/s +Step 26750/50462 | Loss: 2.5870 | LR: 2.32e-04 | 3198.0ms/step | 18.8 samples/s +Validation at step 26750: Loss = 2.5854, Perplexity = 13.27 +Step 26760/50462 | Loss: 2.5825 | LR: 2.32e-04 | 4876.0ms/step | 12.3 samples/s +Step 26770/50462 | Loss: 2.5547 | LR: 2.32e-04 | 3197.4ms/step | 18.8 samples/s +Step 26780/50462 | Loss: 2.5459 | LR: 2.31e-04 | 3176.1ms/step | 18.9 samples/s +Step 26790/50462 | Loss: 2.6104 | LR: 2.31e-04 | 3194.4ms/step | 18.8 samples/s +Step 26800/50462 | Loss: 2.5981 | LR: 2.31e-04 | 3197.3ms/step | 18.8 samples/s +Step 26810/50462 | Loss: 2.6096 | LR: 2.31e-04 | 3200.7ms/step | 18.7 samples/s +Step 26820/50462 | Loss: 2.5635 | LR: 2.31e-04 | 3194.4ms/step | 18.8 samples/s +Step 26830/50462 | Loss: 2.6102 | LR: 2.31e-04 | 3188.6ms/step | 18.8 samples/s +Step 26840/50462 | Loss: 2.5348 | LR: 2.31e-04 | 3194.3ms/step | 18.8 samples/s +Step 26850/50462 | Loss: 2.6001 | LR: 2.30e-04 | 3183.9ms/step | 18.8 samples/s +Step 26860/50462 | Loss: 2.5821 | LR: 2.30e-04 | 3193.0ms/step | 18.8 samples/s +Step 26870/50462 | Loss: 2.5626 | LR: 2.30e-04 | 3195.2ms/step | 18.8 samples/s +Step 26880/50462 | Loss: 2.5542 | LR: 2.30e-04 | 3190.7ms/step | 18.8 samples/s +Step 26890/50462 | Loss: 2.5626 | LR: 2.30e-04 | 3191.9ms/step | 18.8 samples/s +Step 26900/50462 | Loss: 2.5694 | LR: 2.30e-04 | 3199.1ms/step | 18.8 samples/s +Step 26910/50462 | Loss: 2.5826 | LR: 2.29e-04 | 3193.0ms/step | 18.8 samples/s +Step 26920/50462 | Loss: 2.6104 | LR: 2.29e-04 | 3198.0ms/step | 18.8 samples/s +Step 26930/50462 | Loss: 2.5662 | LR: 2.29e-04 | 3199.7ms/step | 18.8 samples/s +Step 26940/50462 | Loss: 2.5699 | LR: 2.29e-04 | 3198.0ms/step | 18.8 samples/s +Step 26950/50462 | Loss: 2.5681 | LR: 2.29e-04 | 3199.7ms/step | 18.8 samples/s +Step 26960/50462 | Loss: 2.5707 | LR: 2.29e-04 | 3198.7ms/step | 18.8 samples/s +Step 26970/50462 | Loss: 2.5785 | LR: 2.28e-04 | 3195.2ms/step | 18.8 samples/s +Step 26980/50462 | Loss: 2.5424 | LR: 2.28e-04 | 3252.0ms/step | 18.5 samples/s +Step 26990/50462 | Loss: 2.5841 | LR: 2.28e-04 | 3197.9ms/step | 18.8 samples/s +Step 27000/50462 | Loss: 2.5894 | LR: 2.28e-04 | 3200.4ms/step | 18.7 samples/s +Validation at step 27000: Loss = 2.5830, Perplexity = 13.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-27000 +Step 27010/50462 | Loss: 2.5513 | LR: 2.28e-04 | 5776.1ms/step | 10.4 samples/s +Step 27020/50462 | Loss: 2.5874 | LR: 2.28e-04 | 3256.8ms/step | 18.4 samples/s +Step 27030/50462 | Loss: 2.5973 | LR: 2.28e-04 | 3249.2ms/step | 18.5 samples/s +Step 27040/50462 | Loss: 2.5834 | LR: 2.27e-04 | 3194.9ms/step | 18.8 samples/s +Step 27050/50462 | Loss: 2.5759 | LR: 2.27e-04 | 3199.1ms/step | 18.8 samples/s +Step 27060/50462 | Loss: 2.5691 | LR: 2.27e-04 | 3198.7ms/step | 18.8 samples/s +Step 27070/50462 | Loss: 2.5701 | LR: 2.27e-04 | 3196.8ms/step | 18.8 samples/s +Step 27080/50462 | Loss: 2.5785 | LR: 2.27e-04 | 3189.0ms/step | 18.8 samples/s +Step 27090/50462 | Loss: 2.5617 | LR: 2.27e-04 | 3199.5ms/step | 18.8 samples/s +Step 27100/50462 | Loss: 2.5786 | LR: 2.26e-04 | 3202.1ms/step | 18.7 samples/s +Step 27110/50462 | Loss: 2.5690 | LR: 2.26e-04 | 3200.1ms/step | 18.7 samples/s +Step 27120/50462 | Loss: 2.5791 | LR: 2.26e-04 | 3197.8ms/step | 18.8 samples/s +Step 27130/50462 | Loss: 2.5923 | LR: 2.26e-04 | 3198.1ms/step | 18.8 samples/s +Step 27140/50462 | Loss: 2.5548 | LR: 2.26e-04 | 3193.2ms/step | 18.8 samples/s +Step 27150/50462 | Loss: 2.5615 | LR: 2.26e-04 | 3194.2ms/step | 18.8 samples/s +Step 27160/50462 | Loss: 2.5700 | LR: 2.25e-04 | 3200.5ms/step | 18.7 samples/s +Step 27170/50462 | Loss: 2.5989 | LR: 2.25e-04 | 3184.2ms/step | 18.8 samples/s +Step 27180/50462 | Loss: 2.5823 | LR: 2.25e-04 | 3183.5ms/step | 18.8 samples/s +Step 27190/50462 | Loss: 2.5893 | LR: 2.25e-04 | 3196.1ms/step | 18.8 samples/s +Step 27200/50462 | Loss: 2.5343 | LR: 2.25e-04 | 3200.1ms/step | 18.7 samples/s +Step 27210/50462 | Loss: 2.5730 | LR: 2.25e-04 | 3200.2ms/step | 18.7 samples/s +Step 27220/50462 | Loss: 2.5590 | LR: 2.25e-04 | 3198.5ms/step | 18.8 samples/s +Step 27230/50462 | Loss: 2.6037 | LR: 2.24e-04 | 3198.8ms/step | 18.8 samples/s +Step 27240/50462 | Loss: 2.5748 | LR: 2.24e-04 | 3200.5ms/step | 18.7 samples/s +Step 27250/50462 | Loss: 2.5885 | LR: 2.24e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 27250: Loss = 2.5816, Perplexity = 13.22 +Step 27260/50462 | Loss: 2.5756 | LR: 2.24e-04 | 4858.3ms/step | 12.4 samples/s +Step 27270/50462 | Loss: 2.5800 | LR: 2.24e-04 | 3183.6ms/step | 18.8 samples/s +Step 27280/50462 | Loss: 2.5532 | LR: 2.24e-04 | 3196.3ms/step | 18.8 samples/s +Step 27290/50462 | Loss: 2.5739 | LR: 2.23e-04 | 3196.1ms/step | 18.8 samples/s +Step 27300/50462 | Loss: 2.5361 | LR: 2.23e-04 | 3198.0ms/step | 18.8 samples/s +Step 27310/50462 | Loss: 2.5288 | LR: 2.23e-04 | 3191.8ms/step | 18.8 samples/s +Step 27320/50462 | Loss: 2.5721 | LR: 2.23e-04 | 3240.5ms/step | 18.5 samples/s +Step 27330/50462 | Loss: 2.6003 | LR: 2.23e-04 | 3193.7ms/step | 18.8 samples/s +Step 27340/50462 | Loss: 2.5612 | LR: 2.23e-04 | 3189.8ms/step | 18.8 samples/s +Step 27350/50462 | Loss: 2.5506 | LR: 2.23e-04 | 3192.0ms/step | 18.8 samples/s +Step 27360/50462 | Loss: 2.5211 | LR: 2.22e-04 | 3236.5ms/step | 18.5 samples/s +Step 27370/50462 | Loss: 2.5604 | LR: 2.22e-04 | 3239.0ms/step | 18.5 samples/s +Step 27380/50462 | Loss: 2.5532 | LR: 2.22e-04 | 3187.6ms/step | 18.8 samples/s +Step 27390/50462 | Loss: 2.5688 | LR: 2.22e-04 | 3184.2ms/step | 18.8 samples/s +Step 27400/50462 | Loss: 2.5846 | LR: 2.22e-04 | 3194.7ms/step | 18.8 samples/s +Step 27410/50462 | Loss: 2.5498 | LR: 2.22e-04 | 3191.4ms/step | 18.8 samples/s +Step 27420/50462 | Loss: 2.5511 | LR: 2.21e-04 | 3188.0ms/step | 18.8 samples/s +Step 27430/50462 | Loss: 2.5751 | LR: 2.21e-04 | 3200.6ms/step | 18.7 samples/s +Step 27440/50462 | Loss: 2.5596 | LR: 2.21e-04 | 3198.7ms/step | 18.8 samples/s +Step 27450/50462 | Loss: 2.5567 | LR: 2.21e-04 | 3192.1ms/step | 18.8 samples/s +Step 27460/50462 | Loss: 2.5932 | LR: 2.21e-04 | 3185.1ms/step | 18.8 samples/s +Step 27470/50462 | Loss: 2.5479 | LR: 2.21e-04 | 3185.4ms/step | 18.8 samples/s +Step 27480/50462 | Loss: 2.5651 | LR: 2.20e-04 | 3182.3ms/step | 18.9 samples/s +Step 27490/50462 | Loss: 2.5370 | LR: 2.20e-04 | 3186.6ms/step | 18.8 samples/s +Step 27500/50462 | Loss: 2.5732 | LR: 2.20e-04 | 3190.0ms/step | 18.8 samples/s +Validation at step 27500: Loss = 2.5794, Perplexity = 13.19 +Step 27510/50462 | Loss: 2.5663 | LR: 2.20e-04 | 4867.7ms/step | 12.3 samples/s +Step 27520/50462 | Loss: 2.5701 | LR: 2.20e-04 | 3191.4ms/step | 18.8 samples/s +Step 27530/50462 | Loss: 2.5920 | LR: 2.20e-04 | 3200.8ms/step | 18.7 samples/s +Step 27540/50462 | Loss: 2.5869 | LR: 2.20e-04 | 3199.9ms/step | 18.8 samples/s +Step 27550/50462 | Loss: 2.5653 | LR: 2.19e-04 | 3200.0ms/step | 18.7 samples/s +Step 27560/50462 | Loss: 2.5753 | LR: 2.19e-04 | 3198.1ms/step | 18.8 samples/s +Step 27570/50462 | Loss: 2.5688 | LR: 2.19e-04 | 3199.3ms/step | 18.8 samples/s +Step 27580/50462 | Loss: 2.5524 | LR: 2.19e-04 | 3194.2ms/step | 18.8 samples/s +Step 27590/50462 | Loss: 2.5769 | LR: 2.19e-04 | 3194.0ms/step | 18.8 samples/s +Step 27600/50462 | Loss: 2.5727 | LR: 2.19e-04 | 3198.5ms/step | 18.8 samples/s +Step 27610/50462 | Loss: 2.5583 | LR: 2.18e-04 | 3200.1ms/step | 18.7 samples/s +Step 27620/50462 | Loss: 2.5791 | LR: 2.18e-04 | 3199.9ms/step | 18.8 samples/s +Step 27630/50462 | Loss: 2.5577 | LR: 2.18e-04 | 3198.4ms/step | 18.8 samples/s +Step 27640/50462 | Loss: 2.5597 | LR: 2.18e-04 | 3197.6ms/step | 18.8 samples/s +Step 27650/50462 | Loss: 2.5476 | LR: 2.18e-04 | 3192.9ms/step | 18.8 samples/s +Step 27660/50462 | Loss: 2.5612 | LR: 2.18e-04 | 3247.1ms/step | 18.5 samples/s +Step 27670/50462 | Loss: 2.5677 | LR: 2.17e-04 | 3183.4ms/step | 18.8 samples/s +Step 27680/50462 | Loss: 2.5600 | LR: 2.17e-04 | 3188.6ms/step | 18.8 samples/s +Step 27690/50462 | Loss: 2.5497 | LR: 2.17e-04 | 3191.8ms/step | 18.8 samples/s +Step 27700/50462 | Loss: 2.5647 | LR: 2.17e-04 | 3250.7ms/step | 18.5 samples/s +Step 27710/50462 | Loss: 2.5632 | LR: 2.17e-04 | 3251.6ms/step | 18.5 samples/s +Step 27720/50462 | Loss: 2.5799 | LR: 2.17e-04 | 3200.1ms/step | 18.7 samples/s +Step 27730/50462 | Loss: 2.5796 | LR: 2.17e-04 | 3184.5ms/step | 18.8 samples/s +Step 27740/50462 | Loss: 2.5642 | LR: 2.16e-04 | 3186.1ms/step | 18.8 samples/s +Step 27750/50462 | Loss: 2.5731 | LR: 2.16e-04 | 3188.6ms/step | 18.8 samples/s +Validation at step 27750: Loss = 2.5778, Perplexity = 13.17 +Step 27760/50462 | Loss: 2.5596 | LR: 2.16e-04 | 4864.5ms/step | 12.3 samples/s +Step 27770/50462 | Loss: 2.5452 | LR: 2.16e-04 | 3186.2ms/step | 18.8 samples/s +Step 27780/50462 | Loss: 2.5823 | LR: 2.16e-04 | 3199.1ms/step | 18.8 samples/s +Step 27790/50462 | Loss: 2.5344 | LR: 2.16e-04 | 3201.6ms/step | 18.7 samples/s +Step 27800/50462 | Loss: 2.5671 | LR: 2.15e-04 | 3200.6ms/step | 18.7 samples/s +Step 27810/50462 | Loss: 2.5738 | LR: 2.15e-04 | 3184.1ms/step | 18.8 samples/s +Step 27820/50462 | Loss: 2.5594 | LR: 2.15e-04 | 3191.0ms/step | 18.8 samples/s +Step 27830/50462 | Loss: 2.5724 | LR: 2.15e-04 | 3197.3ms/step | 18.8 samples/s +Step 27840/50462 | Loss: 2.5520 | LR: 2.15e-04 | 3200.2ms/step | 18.7 samples/s +Step 27850/50462 | Loss: 2.5588 | LR: 2.15e-04 | 3197.8ms/step | 18.8 samples/s +Step 27860/50462 | Loss: 2.5623 | LR: 2.15e-04 | 3192.2ms/step | 18.8 samples/s +Step 27870/50462 | Loss: 2.5198 | LR: 2.14e-04 | 3193.1ms/step | 18.8 samples/s +Step 27880/50462 | Loss: 2.5778 | LR: 2.14e-04 | 3198.4ms/step | 18.8 samples/s +Step 27890/50462 | Loss: 2.5479 | LR: 2.14e-04 | 3191.7ms/step | 18.8 samples/s +Step 27900/50462 | Loss: 2.5467 | LR: 2.14e-04 | 3200.7ms/step | 18.7 samples/s +Step 27910/50462 | Loss: 2.6162 | LR: 2.14e-04 | 3189.7ms/step | 18.8 samples/s +Step 27920/50462 | Loss: 2.5862 | LR: 2.14e-04 | 3194.8ms/step | 18.8 samples/s +Step 27930/50462 | Loss: 2.5557 | LR: 2.13e-04 | 3197.3ms/step | 18.8 samples/s +Step 27940/50462 | Loss: 2.5433 | LR: 2.13e-04 | 3198.0ms/step | 18.8 samples/s +Step 27950/50462 | Loss: 2.5897 | LR: 2.13e-04 | 3183.2ms/step | 18.8 samples/s +Step 27960/50462 | Loss: 2.5693 | LR: 2.13e-04 | 3196.1ms/step | 18.8 samples/s +Step 27970/50462 | Loss: 2.5744 | LR: 2.13e-04 | 3195.3ms/step | 18.8 samples/s +Step 27980/50462 | Loss: 2.5674 | LR: 2.13e-04 | 3184.7ms/step | 18.8 samples/s +Step 27990/50462 | Loss: 2.5740 | LR: 2.12e-04 | 3194.8ms/step | 18.8 samples/s +Step 28000/50462 | Loss: 2.5860 | LR: 2.12e-04 | 3247.5ms/step | 18.5 samples/s +Validation at step 28000: Loss = 2.5760, Perplexity = 13.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-28000 +Step 28010/50462 | Loss: 2.5778 | LR: 2.12e-04 | 5779.3ms/step | 10.4 samples/s +Step 28020/50462 | Loss: 2.5856 | LR: 2.12e-04 | 3170.5ms/step | 18.9 samples/s +Step 28030/50462 | Loss: 2.5881 | LR: 2.12e-04 | 3186.7ms/step | 18.8 samples/s +Step 28040/50462 | Loss: 2.5844 | LR: 2.12e-04 | 3305.7ms/step | 18.2 samples/s +Step 28050/50462 | Loss: 2.5480 | LR: 2.12e-04 | 3183.7ms/step | 18.8 samples/s +Step 28060/50462 | Loss: 2.5907 | LR: 2.11e-04 | 3183.4ms/step | 18.8 samples/s +Step 28070/50462 | Loss: 2.5643 | LR: 2.11e-04 | 3196.8ms/step | 18.8 samples/s +Step 28080/50462 | Loss: 2.5551 | LR: 2.11e-04 | 3198.5ms/step | 18.8 samples/s +Step 28090/50462 | Loss: 2.5706 | LR: 2.11e-04 | 3191.9ms/step | 18.8 samples/s +Step 28100/50462 | Loss: 2.5331 | LR: 2.11e-04 | 3196.7ms/step | 18.8 samples/s +Step 28110/50462 | Loss: 2.5684 | LR: 2.11e-04 | 3191.2ms/step | 18.8 samples/s +Step 28120/50462 | Loss: 2.5543 | LR: 2.10e-04 | 3200.0ms/step | 18.7 samples/s +Step 28130/50462 | Loss: 2.5854 | LR: 2.10e-04 | 3185.7ms/step | 18.8 samples/s +Step 28140/50462 | Loss: 2.5552 | LR: 2.10e-04 | 3182.6ms/step | 18.9 samples/s +Step 28150/50462 | Loss: 2.5777 | LR: 2.10e-04 | 3193.4ms/step | 18.8 samples/s +Step 28160/50462 | Loss: 2.5666 | LR: 2.10e-04 | 3185.7ms/step | 18.8 samples/s +Step 28170/50462 | Loss: 2.5443 | LR: 2.10e-04 | 3198.6ms/step | 18.8 samples/s +Step 28180/50462 | Loss: 2.5782 | LR: 2.10e-04 | 3200.1ms/step | 18.7 samples/s +Step 28190/50462 | Loss: 2.5662 | LR: 2.09e-04 | 3193.7ms/step | 18.8 samples/s +Step 28200/50462 | Loss: 2.5730 | LR: 2.09e-04 | 3186.4ms/step | 18.8 samples/s +Step 28210/50462 | Loss: 2.5603 | LR: 2.09e-04 | 3199.3ms/step | 18.8 samples/s +Step 28220/50462 | Loss: 2.5779 | LR: 2.09e-04 | 3179.9ms/step | 18.9 samples/s +Step 28230/50462 | Loss: 2.5484 | LR: 2.09e-04 | 3177.9ms/step | 18.9 samples/s +Step 28240/50462 | Loss: 2.5535 | LR: 2.09e-04 | 3181.2ms/step | 18.9 samples/s +Step 28250/50462 | Loss: 2.5717 | LR: 2.08e-04 | 3177.5ms/step | 18.9 samples/s +Validation at step 28250: Loss = 2.5747, Perplexity = 13.13 +Step 28260/50462 | Loss: 2.5473 | LR: 2.08e-04 | 4869.9ms/step | 12.3 samples/s +Step 28270/50462 | Loss: 2.5879 | LR: 2.08e-04 | 3187.5ms/step | 18.8 samples/s +Step 28280/50462 | Loss: 2.5388 | LR: 2.08e-04 | 3198.1ms/step | 18.8 samples/s +Step 28290/50462 | Loss: 2.5582 | LR: 2.08e-04 | 3193.8ms/step | 18.8 samples/s +Step 28300/50462 | Loss: 2.5479 | LR: 2.08e-04 | 3189.4ms/step | 18.8 samples/s +Step 28310/50462 | Loss: 2.5809 | LR: 2.07e-04 | 3193.7ms/step | 18.8 samples/s +Step 28320/50462 | Loss: 2.5719 | LR: 2.07e-04 | 3191.0ms/step | 18.8 samples/s +Step 28330/50462 | Loss: 2.5800 | LR: 2.07e-04 | 3185.3ms/step | 18.8 samples/s +Step 28340/50462 | Loss: 2.5621 | LR: 2.07e-04 | 3225.2ms/step | 18.6 samples/s +Step 28350/50462 | Loss: 2.5498 | LR: 2.07e-04 | 3185.6ms/step | 18.8 samples/s +Step 28360/50462 | Loss: 2.5524 | LR: 2.07e-04 | 3194.4ms/step | 18.8 samples/s +Step 28370/50462 | Loss: 2.5470 | LR: 2.07e-04 | 3198.9ms/step | 18.8 samples/s +Step 28380/50462 | Loss: 2.5566 | LR: 2.06e-04 | 3297.2ms/step | 18.2 samples/s +Step 28390/50462 | Loss: 2.5510 | LR: 2.06e-04 | 3197.7ms/step | 18.8 samples/s +Step 28400/50462 | Loss: 2.5829 | LR: 2.06e-04 | 3188.5ms/step | 18.8 samples/s +Step 28410/50462 | Loss: 2.5615 | LR: 2.06e-04 | 3199.9ms/step | 18.8 samples/s +Step 28420/50462 | Loss: 2.5834 | LR: 2.06e-04 | 3200.1ms/step | 18.7 samples/s +Step 28430/50462 | Loss: 2.5283 | LR: 2.06e-04 | 3196.0ms/step | 18.8 samples/s +Step 28440/50462 | Loss: 2.5681 | LR: 2.05e-04 | 3196.0ms/step | 18.8 samples/s +Step 28450/50462 | Loss: 2.5365 | LR: 2.05e-04 | 3197.8ms/step | 18.8 samples/s +Step 28460/50462 | Loss: 2.5661 | LR: 2.05e-04 | 3189.6ms/step | 18.8 samples/s +Step 28470/50462 | Loss: 2.5683 | LR: 2.05e-04 | 3198.2ms/step | 18.8 samples/s +Step 28480/50462 | Loss: 2.5597 | LR: 2.05e-04 | 3199.0ms/step | 18.8 samples/s +Step 28490/50462 | Loss: 2.5534 | LR: 2.05e-04 | 3197.4ms/step | 18.8 samples/s +Step 28500/50462 | Loss: 2.5632 | LR: 2.05e-04 | 3197.8ms/step | 18.8 samples/s +Validation at step 28500: Loss = 2.5726, Perplexity = 13.10 +Step 28510/50462 | Loss: 2.5506 | LR: 2.04e-04 | 4871.5ms/step | 12.3 samples/s +Step 28520/50462 | Loss: 2.5642 | LR: 2.04e-04 | 3182.5ms/step | 18.9 samples/s +Step 28530/50462 | Loss: 2.5144 | LR: 2.04e-04 | 3188.0ms/step | 18.8 samples/s +Step 28540/50462 | Loss: 2.5576 | LR: 2.04e-04 | 3192.2ms/step | 18.8 samples/s +Step 28550/50462 | Loss: 2.6097 | LR: 2.04e-04 | 3198.0ms/step | 18.8 samples/s +Step 28560/50462 | Loss: 2.5492 | LR: 2.04e-04 | 3189.6ms/step | 18.8 samples/s +Step 28570/50462 | Loss: 2.5654 | LR: 2.03e-04 | 3185.7ms/step | 18.8 samples/s +Step 28580/50462 | Loss: 2.5606 | LR: 2.03e-04 | 3192.7ms/step | 18.8 samples/s +Step 28590/50462 | Loss: 2.5787 | LR: 2.03e-04 | 3190.0ms/step | 18.8 samples/s +Step 28600/50462 | Loss: 2.5464 | LR: 2.03e-04 | 3196.1ms/step | 18.8 samples/s +Step 28610/50462 | Loss: 2.5637 | LR: 2.03e-04 | 3199.9ms/step | 18.8 samples/s +Step 28620/50462 | Loss: 2.5428 | LR: 2.03e-04 | 3189.1ms/step | 18.8 samples/s +Step 28630/50462 | Loss: 2.5567 | LR: 2.03e-04 | 3189.0ms/step | 18.8 samples/s +Step 28640/50462 | Loss: 2.5700 | LR: 2.02e-04 | 3195.8ms/step | 18.8 samples/s +Step 28650/50462 | Loss: 2.5590 | LR: 2.02e-04 | 3197.6ms/step | 18.8 samples/s +Step 28660/50462 | Loss: 2.5603 | LR: 2.02e-04 | 3200.2ms/step | 18.7 samples/s +Step 28670/50462 | Loss: 2.5595 | LR: 2.02e-04 | 3190.6ms/step | 18.8 samples/s +Step 28680/50462 | Loss: 2.5730 | LR: 2.02e-04 | 3243.6ms/step | 18.5 samples/s +Step 28690/50462 | Loss: 2.5764 | LR: 2.02e-04 | 3191.4ms/step | 18.8 samples/s +Step 28700/50462 | Loss: 2.5570 | LR: 2.01e-04 | 3197.9ms/step | 18.8 samples/s +Step 28710/50462 | Loss: 2.5407 | LR: 2.01e-04 | 3180.6ms/step | 18.9 samples/s +Step 28720/50462 | Loss: 2.5741 | LR: 2.01e-04 | 3292.8ms/step | 18.2 samples/s +Step 28730/50462 | Loss: 2.5463 | LR: 2.01e-04 | 3177.3ms/step | 18.9 samples/s +Step 28740/50462 | Loss: 2.5640 | LR: 2.01e-04 | 3179.4ms/step | 18.9 samples/s +Step 28750/50462 | Loss: 2.5500 | LR: 2.01e-04 | 3182.0ms/step | 18.9 samples/s +Validation at step 28750: Loss = 2.5708, Perplexity = 13.08 +Step 28760/50462 | Loss: 2.6019 | LR: 2.00e-04 | 4873.5ms/step | 12.3 samples/s +Step 28770/50462 | Loss: 2.5913 | LR: 2.00e-04 | 3193.2ms/step | 18.8 samples/s +Step 28780/50462 | Loss: 2.5628 | LR: 2.00e-04 | 3194.0ms/step | 18.8 samples/s +Step 28790/50462 | Loss: 2.5343 | LR: 2.00e-04 | 3177.9ms/step | 18.9 samples/s +Step 28800/50462 | Loss: 2.5972 | LR: 2.00e-04 | 3186.0ms/step | 18.8 samples/s +Step 28810/50462 | Loss: 2.5329 | LR: 2.00e-04 | 3184.0ms/step | 18.8 samples/s +Step 28820/50462 | Loss: 2.5469 | LR: 2.00e-04 | 3193.9ms/step | 18.8 samples/s +Step 28830/50462 | Loss: 2.5536 | LR: 1.99e-04 | 3200.1ms/step | 18.7 samples/s +Step 28840/50462 | Loss: 2.5629 | LR: 1.99e-04 | 3198.2ms/step | 18.8 samples/s +Step 28850/50462 | Loss: 2.5507 | LR: 1.99e-04 | 3200.0ms/step | 18.8 samples/s +Step 28860/50462 | Loss: 2.5443 | LR: 1.99e-04 | 3200.0ms/step | 18.8 samples/s +Step 28870/50462 | Loss: 2.5659 | LR: 1.99e-04 | 3193.8ms/step | 18.8 samples/s +Step 28880/50462 | Loss: 2.5870 | LR: 1.99e-04 | 3188.2ms/step | 18.8 samples/s +Step 28890/50462 | Loss: 2.5391 | LR: 1.98e-04 | 3197.9ms/step | 18.8 samples/s +Step 28900/50462 | Loss: 2.5541 | LR: 1.98e-04 | 3194.2ms/step | 18.8 samples/s +Step 28910/50462 | Loss: 2.5603 | LR: 1.98e-04 | 3197.4ms/step | 18.8 samples/s +Step 28920/50462 | Loss: 2.5498 | LR: 1.98e-04 | 3200.4ms/step | 18.7 samples/s +Step 28930/50462 | Loss: 2.5763 | LR: 1.98e-04 | 3198.7ms/step | 18.8 samples/s +Step 28940/50462 | Loss: 2.5514 | LR: 1.98e-04 | 3185.3ms/step | 18.8 samples/s +Step 28950/50462 | Loss: 2.5605 | LR: 1.98e-04 | 3178.1ms/step | 18.9 samples/s +Step 28960/50462 | Loss: 2.5632 | LR: 1.97e-04 | 3186.1ms/step | 18.8 samples/s +Step 28970/50462 | Loss: 2.5666 | LR: 1.97e-04 | 3189.4ms/step | 18.8 samples/s +Step 28980/50462 | Loss: 2.5576 | LR: 1.97e-04 | 3185.7ms/step | 18.8 samples/s +Step 28990/50462 | Loss: 2.5520 | LR: 1.97e-04 | 3188.4ms/step | 18.8 samples/s +Step 29000/50462 | Loss: 2.5571 | LR: 1.97e-04 | 3198.5ms/step | 18.8 samples/s +Validation at step 29000: Loss = 2.5693, Perplexity = 13.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-29000 +Step 29010/50462 | Loss: 2.5645 | LR: 1.97e-04 | 5770.5ms/step | 10.4 samples/s +Step 29020/50462 | Loss: 2.5894 | LR: 1.96e-04 | 3236.2ms/step | 18.5 samples/s +Step 29030/50462 | Loss: 2.5613 | LR: 1.96e-04 | 3184.5ms/step | 18.8 samples/s +Step 29040/50462 | Loss: 2.5388 | LR: 1.96e-04 | 3186.1ms/step | 18.8 samples/s +Step 29050/50462 | Loss: 2.5486 | LR: 1.96e-04 | 3200.8ms/step | 18.7 samples/s +Step 29060/50462 | Loss: 2.5790 | LR: 1.96e-04 | 3306.0ms/step | 18.1 samples/s +Step 29070/50462 | Loss: 2.5602 | LR: 1.96e-04 | 3199.9ms/step | 18.8 samples/s +Step 29080/50462 | Loss: 2.5701 | LR: 1.96e-04 | 3200.0ms/step | 18.7 samples/s +Step 29090/50462 | Loss: 2.5672 | LR: 1.95e-04 | 3195.4ms/step | 18.8 samples/s +Step 29100/50462 | Loss: 2.5277 | LR: 1.95e-04 | 3196.7ms/step | 18.8 samples/s +Step 29110/50462 | Loss: 2.5673 | LR: 1.95e-04 | 3190.0ms/step | 18.8 samples/s +Step 29120/50462 | Loss: 2.5747 | LR: 1.95e-04 | 3191.9ms/step | 18.8 samples/s +Step 29130/50462 | Loss: 2.5564 | LR: 1.95e-04 | 3193.4ms/step | 18.8 samples/s +Step 29140/50462 | Loss: 2.5542 | LR: 1.95e-04 | 3200.7ms/step | 18.7 samples/s +Step 29150/50462 | Loss: 2.5864 | LR: 1.94e-04 | 3197.2ms/step | 18.8 samples/s +Step 29160/50462 | Loss: 2.5739 | LR: 1.94e-04 | 3200.3ms/step | 18.7 samples/s +Step 29170/50462 | Loss: 2.5476 | LR: 1.94e-04 | 3192.6ms/step | 18.8 samples/s +Step 29180/50462 | Loss: 2.5480 | LR: 1.94e-04 | 3197.9ms/step | 18.8 samples/s +Step 29190/50462 | Loss: 2.5621 | LR: 1.94e-04 | 3199.9ms/step | 18.8 samples/s +Step 29200/50462 | Loss: 2.5388 | LR: 1.94e-04 | 3196.6ms/step | 18.8 samples/s +Step 29210/50462 | Loss: 2.5702 | LR: 1.94e-04 | 3197.3ms/step | 18.8 samples/s +Step 29220/50462 | Loss: 2.5813 | LR: 1.93e-04 | 3190.0ms/step | 18.8 samples/s +Step 29230/50462 | Loss: 2.5466 | LR: 1.93e-04 | 3191.0ms/step | 18.8 samples/s +Step 29240/50462 | Loss: 2.5666 | LR: 1.93e-04 | 3185.9ms/step | 18.8 samples/s +Step 29250/50462 | Loss: 2.5908 | LR: 1.93e-04 | 3193.0ms/step | 18.8 samples/s +Validation at step 29250: Loss = 2.5679, Perplexity = 13.04 +Step 29260/50462 | Loss: 2.5595 | LR: 1.93e-04 | 4874.3ms/step | 12.3 samples/s +Step 29270/50462 | Loss: 2.5620 | LR: 1.93e-04 | 3199.6ms/step | 18.8 samples/s +Step 29280/50462 | Loss: 2.5514 | LR: 1.92e-04 | 3200.1ms/step | 18.7 samples/s +Step 29290/50462 | Loss: 2.5753 | LR: 1.92e-04 | 3199.5ms/step | 18.8 samples/s +Step 29300/50462 | Loss: 2.5601 | LR: 1.92e-04 | 3194.6ms/step | 18.8 samples/s +Step 29310/50462 | Loss: 2.5619 | LR: 1.92e-04 | 3194.8ms/step | 18.8 samples/s +Step 29320/50462 | Loss: 2.5620 | LR: 1.92e-04 | 3179.3ms/step | 18.9 samples/s +Step 29330/50462 | Loss: 2.5490 | LR: 1.92e-04 | 3190.0ms/step | 18.8 samples/s +Step 29340/50462 | Loss: 2.5715 | LR: 1.92e-04 | 3195.7ms/step | 18.8 samples/s +Step 29350/50462 | Loss: 2.5820 | LR: 1.91e-04 | 3198.2ms/step | 18.8 samples/s +Step 29360/50462 | Loss: 2.5348 | LR: 1.91e-04 | 3244.0ms/step | 18.5 samples/s +Step 29370/50462 | Loss: 2.5824 | LR: 1.91e-04 | 3188.7ms/step | 18.8 samples/s +Step 29380/50462 | Loss: 2.5584 | LR: 1.91e-04 | 3188.2ms/step | 18.8 samples/s +Step 29390/50462 | Loss: 2.5510 | LR: 1.91e-04 | 3183.3ms/step | 18.8 samples/s +Step 29400/50462 | Loss: 2.5807 | LR: 1.91e-04 | 3286.3ms/step | 18.3 samples/s +Step 29410/50462 | Loss: 2.5582 | LR: 1.90e-04 | 3187.8ms/step | 18.8 samples/s +Step 29420/50462 | Loss: 2.5557 | LR: 1.90e-04 | 3193.9ms/step | 18.8 samples/s +Step 29430/50462 | Loss: 2.5258 | LR: 1.90e-04 | 3198.1ms/step | 18.8 samples/s +Step 29440/50462 | Loss: 2.5512 | LR: 1.90e-04 | 3199.8ms/step | 18.8 samples/s +Step 29450/50462 | Loss: 2.5359 | LR: 1.90e-04 | 3199.9ms/step | 18.8 samples/s +Step 29460/50462 | Loss: 2.5756 | LR: 1.90e-04 | 3192.3ms/step | 18.8 samples/s +Step 29470/50462 | Loss: 2.5649 | LR: 1.90e-04 | 3189.2ms/step | 18.8 samples/s +Step 29480/50462 | Loss: 2.5473 | LR: 1.89e-04 | 3196.1ms/step | 18.8 samples/s +Step 29490/50462 | Loss: 2.5290 | LR: 1.89e-04 | 3200.6ms/step | 18.7 samples/s +Step 29500/50462 | Loss: 2.5431 | LR: 1.89e-04 | 3199.6ms/step | 18.8 samples/s +Validation at step 29500: Loss = 2.5667, Perplexity = 13.02 +Step 29510/50462 | Loss: 2.5453 | LR: 1.89e-04 | 4877.9ms/step | 12.3 samples/s +Step 29520/50462 | Loss: 2.5689 | LR: 1.89e-04 | 3202.5ms/step | 18.7 samples/s +Step 29530/50462 | Loss: 2.5743 | LR: 1.89e-04 | 3195.6ms/step | 18.8 samples/s +Step 29540/50462 | Loss: 2.5529 | LR: 1.88e-04 | 3193.9ms/step | 18.8 samples/s +Step 29550/50462 | Loss: 2.5232 | LR: 1.88e-04 | 3192.2ms/step | 18.8 samples/s +Step 29560/50462 | Loss: 2.5505 | LR: 1.88e-04 | 3194.5ms/step | 18.8 samples/s +Step 29570/50462 | Loss: 2.5420 | LR: 1.88e-04 | 3186.2ms/step | 18.8 samples/s +Step 29580/50462 | Loss: 2.5965 | LR: 1.88e-04 | 3184.2ms/step | 18.8 samples/s +Step 29590/50462 | Loss: 2.5426 | LR: 1.88e-04 | 3191.0ms/step | 18.8 samples/s +Step 29600/50462 | Loss: 2.5737 | LR: 1.88e-04 | 3194.5ms/step | 18.8 samples/s +Step 29610/50462 | Loss: 2.5764 | LR: 1.87e-04 | 3193.3ms/step | 18.8 samples/s +Step 29620/50462 | Loss: 2.5497 | LR: 1.87e-04 | 3180.5ms/step | 18.9 samples/s +Step 29630/50462 | Loss: 2.5409 | LR: 1.87e-04 | 3186.1ms/step | 18.8 samples/s +Step 29640/50462 | Loss: 2.5570 | LR: 1.87e-04 | 3178.0ms/step | 18.9 samples/s +Step 29650/50462 | Loss: 2.5703 | LR: 1.87e-04 | 3184.0ms/step | 18.8 samples/s +Step 29660/50462 | Loss: 2.5220 | LR: 1.87e-04 | 3184.6ms/step | 18.8 samples/s +Step 29670/50462 | Loss: 2.5583 | LR: 1.86e-04 | 3179.6ms/step | 18.9 samples/s +Step 29680/50462 | Loss: 2.5605 | LR: 1.86e-04 | 3176.2ms/step | 18.9 samples/s +Step 29690/50462 | Loss: 2.5360 | LR: 1.86e-04 | 3188.3ms/step | 18.8 samples/s +Step 29700/50462 | Loss: 2.5695 | LR: 1.86e-04 | 3247.5ms/step | 18.5 samples/s +Step 29710/50462 | Loss: 2.5823 | LR: 1.86e-04 | 3180.5ms/step | 18.9 samples/s +Step 29720/50462 | Loss: 2.5604 | LR: 1.86e-04 | 3182.1ms/step | 18.9 samples/s +Step 29730/50462 | Loss: 2.5566 | LR: 1.86e-04 | 3243.4ms/step | 18.5 samples/s +Step 29740/50462 | Loss: 2.5680 | LR: 1.85e-04 | 3240.4ms/step | 18.5 samples/s +Step 29750/50462 | Loss: 2.5591 | LR: 1.85e-04 | 3184.1ms/step | 18.8 samples/s +Validation at step 29750: Loss = 2.5656, Perplexity = 13.01 +Step 29760/50462 | Loss: 2.5624 | LR: 1.85e-04 | 4861.5ms/step | 12.3 samples/s +Step 29770/50462 | Loss: 2.5598 | LR: 1.85e-04 | 3184.1ms/step | 18.8 samples/s +Step 29780/50462 | Loss: 2.5451 | LR: 1.85e-04 | 3189.3ms/step | 18.8 samples/s +Step 29790/50462 | Loss: 2.5563 | LR: 1.85e-04 | 3188.6ms/step | 18.8 samples/s +Step 29800/50462 | Loss: 2.5287 | LR: 1.85e-04 | 3191.7ms/step | 18.8 samples/s +Step 29810/50462 | Loss: 2.5239 | LR: 1.84e-04 | 3184.5ms/step | 18.8 samples/s +Step 29820/50462 | Loss: 2.5470 | LR: 1.84e-04 | 3175.3ms/step | 18.9 samples/s +Step 29830/50462 | Loss: 2.5626 | LR: 1.84e-04 | 3187.2ms/step | 18.8 samples/s +Step 29840/50462 | Loss: 2.5431 | LR: 1.84e-04 | 3188.2ms/step | 18.8 samples/s +Step 29850/50462 | Loss: 2.5449 | LR: 1.84e-04 | 3189.5ms/step | 18.8 samples/s +Step 29860/50462 | Loss: 2.5503 | LR: 1.84e-04 | 3179.1ms/step | 18.9 samples/s +Step 29870/50462 | Loss: 2.5554 | LR: 1.83e-04 | 3181.5ms/step | 18.9 samples/s +Step 29880/50462 | Loss: 2.5601 | LR: 1.83e-04 | 3186.8ms/step | 18.8 samples/s +Step 29890/50462 | Loss: 2.5582 | LR: 1.83e-04 | 3184.5ms/step | 18.8 samples/s +Step 29900/50462 | Loss: 2.5552 | LR: 1.83e-04 | 3185.4ms/step | 18.8 samples/s +Step 29910/50462 | Loss: 2.5382 | LR: 1.83e-04 | 3183.5ms/step | 18.8 samples/s +Step 29920/50462 | Loss: 2.5487 | LR: 1.83e-04 | 3178.1ms/step | 18.9 samples/s +Step 29930/50462 | Loss: 2.5732 | LR: 1.83e-04 | 3179.6ms/step | 18.9 samples/s +Step 29940/50462 | Loss: 2.5493 | LR: 1.82e-04 | 3186.5ms/step | 18.8 samples/s +Step 29950/50462 | Loss: 2.5890 | LR: 1.82e-04 | 3189.2ms/step | 18.8 samples/s +Step 29960/50462 | Loss: 2.5660 | LR: 1.82e-04 | 3191.4ms/step | 18.8 samples/s +Step 29970/50462 | Loss: 2.5834 | LR: 1.82e-04 | 3186.6ms/step | 18.8 samples/s +Step 29980/50462 | Loss: 2.5307 | LR: 1.82e-04 | 3188.8ms/step | 18.8 samples/s +Step 29990/50462 | Loss: 2.5256 | LR: 1.82e-04 | 3188.0ms/step | 18.8 samples/s +Step 30000/50462 | Loss: 2.5041 | LR: 1.81e-04 | 3188.4ms/step | 18.8 samples/s +Validation at step 30000: Loss = 2.5642, Perplexity = 12.99 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-30000 +Step 30010/50462 | Loss: 2.5211 | LR: 1.81e-04 | 5820.9ms/step | 10.3 samples/s +Step 30020/50462 | Loss: 2.5350 | LR: 1.81e-04 | 3177.7ms/step | 18.9 samples/s +Step 30030/50462 | Loss: 2.5535 | LR: 1.81e-04 | 3185.1ms/step | 18.8 samples/s +Step 30040/50462 | Loss: 2.6109 | LR: 1.81e-04 | 3254.5ms/step | 18.4 samples/s +Step 30050/50462 | Loss: 2.5607 | LR: 1.81e-04 | 3186.3ms/step | 18.8 samples/s +Step 30060/50462 | Loss: 2.5619 | LR: 1.81e-04 | 3236.4ms/step | 18.5 samples/s +Step 30070/50462 | Loss: 2.5680 | LR: 1.80e-04 | 3264.5ms/step | 18.4 samples/s +Step 30080/50462 | Loss: 2.5593 | LR: 1.80e-04 | 3188.0ms/step | 18.8 samples/s +Step 30090/50462 | Loss: 2.5647 | LR: 1.80e-04 | 3188.2ms/step | 18.8 samples/s +Step 30100/50462 | Loss: 2.5909 | LR: 1.80e-04 | 3188.3ms/step | 18.8 samples/s +Step 30110/50462 | Loss: 2.5832 | LR: 1.80e-04 | 3193.7ms/step | 18.8 samples/s +Step 30120/50462 | Loss: 2.5589 | LR: 1.80e-04 | 3188.2ms/step | 18.8 samples/s +Step 30130/50462 | Loss: 2.5194 | LR: 1.79e-04 | 3186.5ms/step | 18.8 samples/s +Step 30140/50462 | Loss: 2.5725 | LR: 1.79e-04 | 3187.7ms/step | 18.8 samples/s +Step 30150/50462 | Loss: 2.5650 | LR: 1.79e-04 | 3189.6ms/step | 18.8 samples/s +Step 30160/50462 | Loss: 2.5624 | LR: 1.79e-04 | 3200.2ms/step | 18.7 samples/s +Step 30170/50462 | Loss: 2.5414 | LR: 1.79e-04 | 3198.1ms/step | 18.8 samples/s +Step 30180/50462 | Loss: 2.5291 | LR: 1.79e-04 | 3199.7ms/step | 18.8 samples/s +Step 30190/50462 | Loss: 2.5261 | LR: 1.79e-04 | 3199.8ms/step | 18.8 samples/s +Step 30200/50462 | Loss: 2.5534 | LR: 1.78e-04 | 3196.7ms/step | 18.8 samples/s +Step 30210/50462 | Loss: 2.5765 | LR: 1.78e-04 | 3195.6ms/step | 18.8 samples/s +Step 30220/50462 | Loss: 2.5369 | LR: 1.78e-04 | 3192.8ms/step | 18.8 samples/s +Step 30230/50462 | Loss: 2.5516 | LR: 1.78e-04 | 3193.4ms/step | 18.8 samples/s +Step 30240/50462 | Loss: 2.5698 | LR: 1.78e-04 | 3200.1ms/step | 18.7 samples/s +Step 30250/50462 | Loss: 2.5331 | LR: 1.78e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 30250: Loss = 2.5630, Perplexity = 12.98 +Step 30260/50462 | Loss: 2.5533 | LR: 1.78e-04 | 4872.1ms/step | 12.3 samples/s +Step 30270/50462 | Loss: 2.5684 | LR: 1.77e-04 | 3199.9ms/step | 18.8 samples/s +Step 30280/50462 | Loss: 2.5634 | LR: 1.77e-04 | 3199.9ms/step | 18.8 samples/s +Step 30290/50462 | Loss: 2.5571 | LR: 1.77e-04 | 3192.0ms/step | 18.8 samples/s +Step 30300/50462 | Loss: 2.5576 | LR: 1.77e-04 | 3198.0ms/step | 18.8 samples/s +Step 30310/50462 | Loss: 2.5385 | LR: 1.77e-04 | 3198.1ms/step | 18.8 samples/s +Step 30320/50462 | Loss: 2.5450 | LR: 1.77e-04 | 3190.5ms/step | 18.8 samples/s +Step 30330/50462 | Loss: 2.5614 | LR: 1.76e-04 | 3199.2ms/step | 18.8 samples/s +Step 30340/50462 | Loss: 2.5352 | LR: 1.76e-04 | 3200.4ms/step | 18.7 samples/s +Step 30350/50462 | Loss: 2.5836 | LR: 1.76e-04 | 3191.8ms/step | 18.8 samples/s +Step 30360/50462 | Loss: 2.5632 | LR: 1.76e-04 | 3199.6ms/step | 18.8 samples/s +Step 30370/50462 | Loss: 2.5575 | LR: 1.76e-04 | 3198.2ms/step | 18.8 samples/s +Step 30380/50462 | Loss: 2.5522 | LR: 1.76e-04 | 3251.8ms/step | 18.5 samples/s +Step 30390/50462 | Loss: 2.5385 | LR: 1.76e-04 | 3196.3ms/step | 18.8 samples/s +Step 30400/50462 | Loss: 2.5269 | LR: 1.75e-04 | 3250.6ms/step | 18.5 samples/s +Step 30410/50462 | Loss: 2.5470 | LR: 1.75e-04 | 3242.2ms/step | 18.5 samples/s +Step 30420/50462 | Loss: 2.5076 | LR: 1.75e-04 | 3192.9ms/step | 18.8 samples/s +Step 30430/50462 | Loss: 2.5627 | LR: 1.75e-04 | 3192.3ms/step | 18.8 samples/s +Step 30440/50462 | Loss: 2.5635 | LR: 1.75e-04 | 3196.2ms/step | 18.8 samples/s +Step 30450/50462 | Loss: 2.5734 | LR: 1.75e-04 | 3197.6ms/step | 18.8 samples/s +Step 30460/50462 | Loss: 2.5272 | LR: 1.75e-04 | 3198.4ms/step | 18.8 samples/s +Step 30470/50462 | Loss: 2.5519 | LR: 1.74e-04 | 3197.8ms/step | 18.8 samples/s +Step 30480/50462 | Loss: 2.5719 | LR: 1.74e-04 | 3198.0ms/step | 18.8 samples/s +Step 30490/50462 | Loss: 2.5663 | LR: 1.74e-04 | 3199.8ms/step | 18.8 samples/s +Step 30500/50462 | Loss: 2.5452 | LR: 1.74e-04 | 3200.2ms/step | 18.7 samples/s +Validation at step 30500: Loss = 2.5617, Perplexity = 12.96 +Step 30510/50462 | Loss: 2.5444 | LR: 1.74e-04 | 4869.7ms/step | 12.3 samples/s +Step 30520/50462 | Loss: 2.5458 | LR: 1.74e-04 | 3200.1ms/step | 18.7 samples/s +Step 30530/50462 | Loss: 2.5673 | LR: 1.73e-04 | 3200.0ms/step | 18.7 samples/s +Step 30540/50462 | Loss: 2.5238 | LR: 1.73e-04 | 3200.1ms/step | 18.7 samples/s +Step 30550/50462 | Loss: 2.5708 | LR: 1.73e-04 | 3197.5ms/step | 18.8 samples/s +Step 30560/50462 | Loss: 2.5239 | LR: 1.73e-04 | 3193.7ms/step | 18.8 samples/s +Step 30570/50462 | Loss: 2.5573 | LR: 1.73e-04 | 3189.1ms/step | 18.8 samples/s +Step 30580/50462 | Loss: 2.5572 | LR: 1.73e-04 | 3197.8ms/step | 18.8 samples/s +Step 30590/50462 | Loss: 2.5248 | LR: 1.73e-04 | 3196.3ms/step | 18.8 samples/s +Step 30600/50462 | Loss: 2.5297 | LR: 1.72e-04 | 3197.9ms/step | 18.8 samples/s +Step 30610/50462 | Loss: 2.5755 | LR: 1.72e-04 | 3198.0ms/step | 18.8 samples/s +Step 30620/50462 | Loss: 2.5385 | LR: 1.72e-04 | 3197.5ms/step | 18.8 samples/s +Step 30630/50462 | Loss: 2.5253 | LR: 1.72e-04 | 3197.7ms/step | 18.8 samples/s +Step 30640/50462 | Loss: 2.5289 | LR: 1.72e-04 | 3196.7ms/step | 18.8 samples/s +Step 30650/50462 | Loss: 2.5587 | LR: 1.72e-04 | 3189.3ms/step | 18.8 samples/s +Step 30660/50462 | Loss: 2.5158 | LR: 1.72e-04 | 3198.4ms/step | 18.8 samples/s +Step 30670/50462 | Loss: 2.5358 | LR: 1.71e-04 | 3197.1ms/step | 18.8 samples/s +Step 30680/50462 | Loss: 2.5369 | LR: 1.71e-04 | 3195.1ms/step | 18.8 samples/s +Step 30690/50462 | Loss: 2.5359 | LR: 1.71e-04 | 3198.5ms/step | 18.8 samples/s +Step 30700/50462 | Loss: 2.5649 | LR: 1.71e-04 | 3197.6ms/step | 18.8 samples/s +Step 30710/50462 | Loss: 2.5451 | LR: 1.71e-04 | 3194.7ms/step | 18.8 samples/s +Step 30720/50462 | Loss: 2.5752 | LR: 1.71e-04 | 3251.6ms/step | 18.5 samples/s +Step 30730/50462 | Loss: 2.5482 | LR: 1.70e-04 | 3195.3ms/step | 18.8 samples/s +Step 30740/50462 | Loss: 2.5451 | LR: 1.70e-04 | 3244.5ms/step | 18.5 samples/s +Step 30750/50462 | Loss: 2.5383 | LR: 1.70e-04 | 3245.5ms/step | 18.5 samples/s +Validation at step 30750: Loss = 2.5604, Perplexity = 12.94 +Step 30760/50462 | Loss: 2.5605 | LR: 1.70e-04 | 4871.9ms/step | 12.3 samples/s +Step 30770/50462 | Loss: 2.5538 | LR: 1.70e-04 | 3196.1ms/step | 18.8 samples/s +Step 30780/50462 | Loss: 2.5476 | LR: 1.70e-04 | 3197.8ms/step | 18.8 samples/s +Step 30790/50462 | Loss: 2.5603 | LR: 1.70e-04 | 3197.5ms/step | 18.8 samples/s +Step 30800/50462 | Loss: 2.5298 | LR: 1.69e-04 | 3201.3ms/step | 18.7 samples/s +Step 30810/50462 | Loss: 2.5275 | LR: 1.69e-04 | 3193.5ms/step | 18.8 samples/s +Step 30820/50462 | Loss: 2.5579 | LR: 1.69e-04 | 3198.0ms/step | 18.8 samples/s +Step 30830/50462 | Loss: 2.5550 | LR: 1.69e-04 | 3198.0ms/step | 18.8 samples/s +Step 30840/50462 | Loss: 2.5307 | LR: 1.69e-04 | 3198.5ms/step | 18.8 samples/s +Step 30850/50462 | Loss: 2.5516 | LR: 1.69e-04 | 3197.7ms/step | 18.8 samples/s +Step 30860/50462 | Loss: 2.5669 | LR: 1.69e-04 | 3195.8ms/step | 18.8 samples/s +Step 30870/50462 | Loss: 2.5367 | LR: 1.68e-04 | 3196.0ms/step | 18.8 samples/s +Step 30880/50462 | Loss: 2.5119 | LR: 1.68e-04 | 3195.9ms/step | 18.8 samples/s +Step 30890/50462 | Loss: 2.5365 | LR: 1.68e-04 | 3196.5ms/step | 18.8 samples/s +Step 30900/50462 | Loss: 2.5223 | LR: 1.68e-04 | 3197.5ms/step | 18.8 samples/s +Step 30910/50462 | Loss: 2.5240 | LR: 1.68e-04 | 3198.4ms/step | 18.8 samples/s +Step 30920/50462 | Loss: 2.5488 | LR: 1.68e-04 | 3197.8ms/step | 18.8 samples/s +Step 30930/50462 | Loss: 2.5286 | LR: 1.67e-04 | 3194.1ms/step | 18.8 samples/s +Step 30940/50462 | Loss: 2.5063 | LR: 1.67e-04 | 3199.5ms/step | 18.8 samples/s +Step 30950/50462 | Loss: 2.5690 | LR: 1.67e-04 | 3199.9ms/step | 18.8 samples/s +Step 30960/50462 | Loss: 2.5437 | LR: 1.67e-04 | 3198.2ms/step | 18.8 samples/s +Step 30970/50462 | Loss: 2.5664 | LR: 1.67e-04 | 3199.9ms/step | 18.8 samples/s +Step 30980/50462 | Loss: 2.5648 | LR: 1.67e-04 | 3198.0ms/step | 18.8 samples/s +Step 30990/50462 | Loss: 2.5440 | LR: 1.67e-04 | 3198.6ms/step | 18.8 samples/s +Step 31000/50462 | Loss: 2.5384 | LR: 1.66e-04 | 3197.1ms/step | 18.8 samples/s +Validation at step 31000: Loss = 2.5594, Perplexity = 12.93 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-31000 +Step 31010/50462 | Loss: 2.5792 | LR: 1.66e-04 | 5991.9ms/step | 10.0 samples/s +Step 31020/50462 | Loss: 2.5466 | LR: 1.66e-04 | 3196.2ms/step | 18.8 samples/s +Step 31030/50462 | Loss: 2.5378 | LR: 1.66e-04 | 3196.7ms/step | 18.8 samples/s +Step 31040/50462 | Loss: 2.5162 | LR: 1.66e-04 | 3200.1ms/step | 18.7 samples/s +Step 31050/50462 | Loss: 2.5416 | LR: 1.66e-04 | 3199.9ms/step | 18.8 samples/s +Step 31060/50462 | Loss: 2.5418 | LR: 1.66e-04 | 3251.1ms/step | 18.5 samples/s +Step 31070/50462 | Loss: 2.5301 | LR: 1.65e-04 | 3256.4ms/step | 18.4 samples/s +Step 31080/50462 | Loss: 2.5081 | LR: 1.65e-04 | 3192.3ms/step | 18.8 samples/s +Step 31090/50462 | Loss: 2.5319 | LR: 1.65e-04 | 3252.2ms/step | 18.4 samples/s +Step 31100/50462 | Loss: 2.5697 | LR: 1.65e-04 | 3199.5ms/step | 18.8 samples/s +Step 31110/50462 | Loss: 2.5339 | LR: 1.65e-04 | 3198.5ms/step | 18.8 samples/s +Step 31120/50462 | Loss: 2.5711 | LR: 1.65e-04 | 3197.8ms/step | 18.8 samples/s +Step 31130/50462 | Loss: 2.5636 | LR: 1.64e-04 | 3195.7ms/step | 18.8 samples/s +Step 31140/50462 | Loss: 2.5130 | LR: 1.64e-04 | 3198.4ms/step | 18.8 samples/s +Step 31150/50462 | Loss: 2.5739 | LR: 1.64e-04 | 3198.1ms/step | 18.8 samples/s +Step 31160/50462 | Loss: 2.5385 | LR: 1.64e-04 | 3200.0ms/step | 18.7 samples/s +Step 31170/50462 | Loss: 2.5567 | LR: 1.64e-04 | 3199.7ms/step | 18.8 samples/s +Step 31180/50462 | Loss: 2.5394 | LR: 1.64e-04 | 3199.6ms/step | 18.8 samples/s +Step 31190/50462 | Loss: 2.5506 | LR: 1.64e-04 | 3198.9ms/step | 18.8 samples/s +Step 31200/50462 | Loss: 2.5220 | LR: 1.63e-04 | 3199.7ms/step | 18.8 samples/s +Step 31210/50462 | Loss: 2.5432 | LR: 1.63e-04 | 3199.6ms/step | 18.8 samples/s +Step 31220/50462 | Loss: 2.5251 | LR: 1.63e-04 | 3200.6ms/step | 18.7 samples/s +Step 31230/50462 | Loss: 2.5425 | LR: 1.63e-04 | 3199.2ms/step | 18.8 samples/s +Step 31240/50462 | Loss: 2.5217 | LR: 1.63e-04 | 3200.5ms/step | 18.7 samples/s +Step 31250/50462 | Loss: 2.5349 | LR: 1.63e-04 | 3200.4ms/step | 18.7 samples/s +Validation at step 31250: Loss = 2.5579, Perplexity = 12.91 +Step 31260/50462 | Loss: 2.5253 | LR: 1.63e-04 | 4873.3ms/step | 12.3 samples/s +Step 31270/50462 | Loss: 2.5362 | LR: 1.62e-04 | 3198.4ms/step | 18.8 samples/s +Step 31280/50462 | Loss: 2.5711 | LR: 1.62e-04 | 3200.0ms/step | 18.7 samples/s +Step 31290/50462 | Loss: 2.5664 | LR: 1.62e-04 | 3195.5ms/step | 18.8 samples/s +Step 31300/50462 | Loss: 2.5490 | LR: 1.62e-04 | 3200.4ms/step | 18.7 samples/s +Step 31310/50462 | Loss: 2.5475 | LR: 1.62e-04 | 3200.1ms/step | 18.7 samples/s +Step 31320/50462 | Loss: 2.5619 | LR: 1.62e-04 | 3196.9ms/step | 18.8 samples/s +Step 31330/50462 | Loss: 2.5008 | LR: 1.62e-04 | 3196.4ms/step | 18.8 samples/s +Step 31340/50462 | Loss: 2.5508 | LR: 1.61e-04 | 3194.0ms/step | 18.8 samples/s +Step 31350/50462 | Loss: 2.5687 | LR: 1.61e-04 | 3196.4ms/step | 18.8 samples/s +Step 31360/50462 | Loss: 2.5610 | LR: 1.61e-04 | 3197.7ms/step | 18.8 samples/s +Step 31370/50462 | Loss: 2.5394 | LR: 1.61e-04 | 3197.2ms/step | 18.8 samples/s +Step 31380/50462 | Loss: 2.5210 | LR: 1.61e-04 | 3200.8ms/step | 18.7 samples/s +Step 31390/50462 | Loss: 2.5133 | LR: 1.61e-04 | 3200.3ms/step | 18.7 samples/s +Step 31400/50462 | Loss: 2.5376 | LR: 1.60e-04 | 3200.1ms/step | 18.7 samples/s +Step 31410/50462 | Loss: 2.5474 | LR: 1.60e-04 | 3301.5ms/step | 18.2 samples/s +Step 31420/50462 | Loss: 2.5531 | LR: 1.60e-04 | 3192.1ms/step | 18.8 samples/s +Step 31430/50462 | Loss: 2.5531 | LR: 1.60e-04 | 3246.3ms/step | 18.5 samples/s +Step 31440/50462 | Loss: 2.5724 | LR: 1.60e-04 | 3189.9ms/step | 18.8 samples/s +Step 31450/50462 | Loss: 2.5383 | LR: 1.60e-04 | 3186.3ms/step | 18.8 samples/s +Step 31460/50462 | Loss: 2.5513 | LR: 1.60e-04 | 3188.2ms/step | 18.8 samples/s +Step 31470/50462 | Loss: 2.5271 | LR: 1.59e-04 | 3193.4ms/step | 18.8 samples/s +Step 31480/50462 | Loss: 2.5679 | LR: 1.59e-04 | 3192.6ms/step | 18.8 samples/s +Step 31490/50462 | Loss: 2.5744 | LR: 1.59e-04 | 3196.7ms/step | 18.8 samples/s +Step 31500/50462 | Loss: 2.5447 | LR: 1.59e-04 | 3195.2ms/step | 18.8 samples/s +Validation at step 31500: Loss = 2.5563, Perplexity = 12.89 +Step 31510/50462 | Loss: 2.5607 | LR: 1.59e-04 | 4873.9ms/step | 12.3 samples/s +Step 31520/50462 | Loss: 2.5908 | LR: 1.59e-04 | 3198.0ms/step | 18.8 samples/s +Step 31530/50462 | Loss: 2.5169 | LR: 1.59e-04 | 3197.8ms/step | 18.8 samples/s +Step 31540/50462 | Loss: 2.5342 | LR: 1.58e-04 | 3200.3ms/step | 18.7 samples/s +Step 31550/50462 | Loss: 2.5281 | LR: 1.58e-04 | 3187.7ms/step | 18.8 samples/s +Step 31560/50462 | Loss: 2.5593 | LR: 1.58e-04 | 3195.8ms/step | 18.8 samples/s +Step 31570/50462 | Loss: 2.5352 | LR: 1.58e-04 | 3186.8ms/step | 18.8 samples/s +Step 31580/50462 | Loss: 2.5143 | LR: 1.58e-04 | 3194.3ms/step | 18.8 samples/s +Step 31590/50462 | Loss: 2.5439 | LR: 1.58e-04 | 3197.8ms/step | 18.8 samples/s +Step 31600/50462 | Loss: 2.5377 | LR: 1.58e-04 | 3195.4ms/step | 18.8 samples/s +Step 31610/50462 | Loss: 2.5186 | LR: 1.57e-04 | 3200.0ms/step | 18.7 samples/s +Step 31620/50462 | Loss: 2.5682 | LR: 1.57e-04 | 3199.9ms/step | 18.8 samples/s +Step 31630/50462 | Loss: 2.5400 | LR: 1.57e-04 | 3200.1ms/step | 18.7 samples/s +Step 31640/50462 | Loss: 2.5622 | LR: 1.57e-04 | 3200.0ms/step | 18.8 samples/s +Step 31650/50462 | Loss: 2.5491 | LR: 1.57e-04 | 3199.9ms/step | 18.8 samples/s +Step 31660/50462 | Loss: 2.5280 | LR: 1.57e-04 | 3200.2ms/step | 18.7 samples/s +Step 31670/50462 | Loss: 2.5496 | LR: 1.57e-04 | 3198.6ms/step | 18.8 samples/s +Step 31680/50462 | Loss: 2.5535 | LR: 1.56e-04 | 3196.8ms/step | 18.8 samples/s +Step 31690/50462 | Loss: 2.5505 | LR: 1.56e-04 | 3199.5ms/step | 18.8 samples/s +Step 31700/50462 | Loss: 2.5049 | LR: 1.56e-04 | 3199.3ms/step | 18.8 samples/s +Step 31710/50462 | Loss: 2.5127 | LR: 1.56e-04 | 3201.7ms/step | 18.7 samples/s +Step 31720/50462 | Loss: 2.5570 | LR: 1.56e-04 | 3200.2ms/step | 18.7 samples/s +Step 31730/50462 | Loss: 2.5636 | LR: 1.56e-04 | 3200.0ms/step | 18.7 samples/s +Step 31740/50462 | Loss: 2.5567 | LR: 1.55e-04 | 3201.7ms/step | 18.7 samples/s +Step 31750/50462 | Loss: 2.5490 | LR: 1.55e-04 | 3321.9ms/step | 18.1 samples/s +Validation at step 31750: Loss = 2.5555, Perplexity = 12.88 +Step 31760/50462 | Loss: 2.5400 | LR: 1.55e-04 | 4874.9ms/step | 12.3 samples/s +Step 31770/50462 | Loss: 2.5627 | LR: 1.55e-04 | 3251.1ms/step | 18.5 samples/s +Step 31780/50462 | Loss: 2.5319 | LR: 1.55e-04 | 3198.5ms/step | 18.8 samples/s +Step 31790/50462 | Loss: 2.5523 | LR: 1.55e-04 | 3201.9ms/step | 18.7 samples/s +Step 31800/50462 | Loss: 2.5444 | LR: 1.55e-04 | 3201.9ms/step | 18.7 samples/s +Step 31810/50462 | Loss: 2.5358 | LR: 1.54e-04 | 3200.1ms/step | 18.7 samples/s +Step 31820/50462 | Loss: 2.5513 | LR: 1.54e-04 | 3200.0ms/step | 18.8 samples/s +Step 31830/50462 | Loss: 2.5249 | LR: 1.54e-04 | 3199.6ms/step | 18.8 samples/s +Step 31840/50462 | Loss: 2.5443 | LR: 1.54e-04 | 3199.6ms/step | 18.8 samples/s +Step 31850/50462 | Loss: 2.5376 | LR: 1.54e-04 | 3199.7ms/step | 18.8 samples/s +Step 31860/50462 | Loss: 2.5291 | LR: 1.54e-04 | 3193.9ms/step | 18.8 samples/s +Step 31870/50462 | Loss: 2.5438 | LR: 1.54e-04 | 3192.2ms/step | 18.8 samples/s +Step 31880/50462 | Loss: 2.5442 | LR: 1.53e-04 | 3198.1ms/step | 18.8 samples/s +Step 31890/50462 | Loss: 2.5453 | LR: 1.53e-04 | 3200.7ms/step | 18.7 samples/s +Step 31900/50462 | Loss: 2.5646 | LR: 1.53e-04 | 3201.8ms/step | 18.7 samples/s +Step 31910/50462 | Loss: 2.5459 | LR: 1.53e-04 | 3200.2ms/step | 18.7 samples/s +Step 31920/50462 | Loss: 2.5085 | LR: 1.53e-04 | 3199.9ms/step | 18.8 samples/s +Step 31930/50462 | Loss: 2.5287 | LR: 1.53e-04 | 3200.1ms/step | 18.7 samples/s +Step 31940/50462 | Loss: 2.5578 | LR: 1.53e-04 | 3200.0ms/step | 18.7 samples/s +Step 31950/50462 | Loss: 2.5617 | LR: 1.52e-04 | 3199.8ms/step | 18.8 samples/s +Step 31960/50462 | Loss: 2.5533 | LR: 1.52e-04 | 3198.1ms/step | 18.8 samples/s +Step 31970/50462 | Loss: 2.5434 | LR: 1.52e-04 | 3196.0ms/step | 18.8 samples/s +Step 31980/50462 | Loss: 2.5659 | LR: 1.52e-04 | 3198.2ms/step | 18.8 samples/s +Step 31990/50462 | Loss: 2.5656 | LR: 1.52e-04 | 3197.7ms/step | 18.8 samples/s +Step 32000/50462 | Loss: 2.5903 | LR: 1.52e-04 | 3199.5ms/step | 18.8 samples/s +Validation at step 32000: Loss = 2.5544, Perplexity = 12.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-32000 +Step 32010/50462 | Loss: 2.5715 | LR: 1.52e-04 | 5922.9ms/step | 10.1 samples/s +Step 32020/50462 | Loss: 2.5246 | LR: 1.51e-04 | 3183.5ms/step | 18.8 samples/s +Step 32030/50462 | Loss: 2.5284 | LR: 1.51e-04 | 3194.2ms/step | 18.8 samples/s +Step 32040/50462 | Loss: 2.5808 | LR: 1.51e-04 | 3195.4ms/step | 18.8 samples/s +Step 32050/50462 | Loss: 2.5444 | LR: 1.51e-04 | 3194.5ms/step | 18.8 samples/s +Step 32060/50462 | Loss: 2.5290 | LR: 1.51e-04 | 3199.4ms/step | 18.8 samples/s +Step 32070/50462 | Loss: 2.5459 | LR: 1.51e-04 | 3195.1ms/step | 18.8 samples/s +Step 32080/50462 | Loss: 2.5538 | LR: 1.51e-04 | 3195.4ms/step | 18.8 samples/s +Step 32090/50462 | Loss: 2.5392 | LR: 1.50e-04 | 3292.3ms/step | 18.2 samples/s +Step 32100/50462 | Loss: 2.5347 | LR: 1.50e-04 | 3193.3ms/step | 18.8 samples/s +Step 32110/50462 | Loss: 2.5213 | LR: 1.50e-04 | 3250.3ms/step | 18.5 samples/s +Step 32120/50462 | Loss: 2.5492 | LR: 1.50e-04 | 3196.9ms/step | 18.8 samples/s +Step 32130/50462 | Loss: 2.5365 | LR: 1.50e-04 | 3198.7ms/step | 18.8 samples/s +Step 32140/50462 | Loss: 2.5567 | LR: 1.50e-04 | 3200.0ms/step | 18.7 samples/s +Step 32150/50462 | Loss: 2.5768 | LR: 1.50e-04 | 3200.5ms/step | 18.7 samples/s +Step 32160/50462 | Loss: 2.5400 | LR: 1.49e-04 | 3198.0ms/step | 18.8 samples/s +Step 32170/50462 | Loss: 2.5399 | LR: 1.49e-04 | 3197.9ms/step | 18.8 samples/s +Step 32180/50462 | Loss: 2.5270 | LR: 1.49e-04 | 3199.5ms/step | 18.8 samples/s +Step 32190/50462 | Loss: 2.5548 | LR: 1.49e-04 | 3198.6ms/step | 18.8 samples/s +Step 32200/50462 | Loss: 2.5361 | LR: 1.49e-04 | 3200.1ms/step | 18.7 samples/s +Step 32210/50462 | Loss: 2.5566 | LR: 1.49e-04 | 3197.7ms/step | 18.8 samples/s +Step 32220/50462 | Loss: 2.5502 | LR: 1.49e-04 | 3198.0ms/step | 18.8 samples/s +Step 32230/50462 | Loss: 2.5411 | LR: 1.48e-04 | 3200.0ms/step | 18.8 samples/s +Step 32240/50462 | Loss: 2.5296 | LR: 1.48e-04 | 3194.2ms/step | 18.8 samples/s +Step 32250/50462 | Loss: 2.5440 | LR: 1.48e-04 | 3188.0ms/step | 18.8 samples/s +Validation at step 32250: Loss = 2.5537, Perplexity = 12.85 +Step 32260/50462 | Loss: 2.5360 | LR: 1.48e-04 | 4865.9ms/step | 12.3 samples/s +Step 32270/50462 | Loss: 2.5534 | LR: 1.48e-04 | 3190.2ms/step | 18.8 samples/s +Step 32280/50462 | Loss: 2.5416 | LR: 1.48e-04 | 3199.8ms/step | 18.8 samples/s +Step 32290/50462 | Loss: 2.5543 | LR: 1.48e-04 | 3199.9ms/step | 18.8 samples/s +Step 32300/50462 | Loss: 2.5606 | LR: 1.47e-04 | 3192.1ms/step | 18.8 samples/s +Step 32310/50462 | Loss: 2.5813 | LR: 1.47e-04 | 3197.8ms/step | 18.8 samples/s +Step 32320/50462 | Loss: 2.5189 | LR: 1.47e-04 | 3200.2ms/step | 18.7 samples/s +Step 32330/50462 | Loss: 2.5642 | LR: 1.47e-04 | 3199.9ms/step | 18.8 samples/s +Step 32340/50462 | Loss: 2.5924 | LR: 1.47e-04 | 3200.2ms/step | 18.7 samples/s +Step 32350/50462 | Loss: 2.5288 | LR: 1.47e-04 | 3199.7ms/step | 18.8 samples/s +Step 32360/50462 | Loss: 2.5428 | LR: 1.47e-04 | 3198.8ms/step | 18.8 samples/s +Step 32370/50462 | Loss: 2.5519 | LR: 1.46e-04 | 3198.9ms/step | 18.8 samples/s +Step 32380/50462 | Loss: 2.5029 | LR: 1.46e-04 | 3198.5ms/step | 18.8 samples/s +Step 32390/50462 | Loss: 2.5323 | LR: 1.46e-04 | 3200.0ms/step | 18.8 samples/s +Step 32400/50462 | Loss: 2.5621 | LR: 1.46e-04 | 3200.0ms/step | 18.8 samples/s +Step 32410/50462 | Loss: 2.5567 | LR: 1.46e-04 | 3195.9ms/step | 18.8 samples/s +Step 32420/50462 | Loss: 2.5336 | LR: 1.46e-04 | 3250.0ms/step | 18.5 samples/s +Step 32430/50462 | Loss: 2.5293 | LR: 1.45e-04 | 3242.1ms/step | 18.5 samples/s +Step 32440/50462 | Loss: 2.5359 | LR: 1.45e-04 | 3190.9ms/step | 18.8 samples/s +Step 32450/50462 | Loss: 2.5530 | LR: 1.45e-04 | 3257.2ms/step | 18.4 samples/s +Step 32460/50462 | Loss: 2.5330 | LR: 1.45e-04 | 3194.0ms/step | 18.8 samples/s +Step 32470/50462 | Loss: 2.5563 | LR: 1.45e-04 | 3199.7ms/step | 18.8 samples/s +Step 32480/50462 | Loss: 2.5592 | LR: 1.45e-04 | 3195.5ms/step | 18.8 samples/s +Step 32490/50462 | Loss: 2.5372 | LR: 1.45e-04 | 3198.8ms/step | 18.8 samples/s +Step 32500/50462 | Loss: 2.5860 | LR: 1.44e-04 | 3199.6ms/step | 18.8 samples/s +Validation at step 32500: Loss = 2.5525, Perplexity = 12.84 +Step 32510/50462 | Loss: 2.5392 | LR: 1.44e-04 | 4873.7ms/step | 12.3 samples/s +Step 32520/50462 | Loss: 2.5391 | LR: 1.44e-04 | 3198.4ms/step | 18.8 samples/s +Step 32530/50462 | Loss: 2.5433 | LR: 1.44e-04 | 3198.3ms/step | 18.8 samples/s +Step 32540/50462 | Loss: 2.5385 | LR: 1.44e-04 | 3199.9ms/step | 18.8 samples/s +Step 32550/50462 | Loss: 2.5421 | LR: 1.44e-04 | 3199.3ms/step | 18.8 samples/s +Step 32560/50462 | Loss: 2.5050 | LR: 1.44e-04 | 3200.7ms/step | 18.7 samples/s +Step 32570/50462 | Loss: 2.5288 | LR: 1.43e-04 | 3199.8ms/step | 18.8 samples/s +Step 32580/50462 | Loss: 2.5252 | LR: 1.43e-04 | 3200.0ms/step | 18.7 samples/s +Step 32590/50462 | Loss: 2.5366 | LR: 1.43e-04 | 3200.1ms/step | 18.7 samples/s +Step 32600/50462 | Loss: 2.5403 | LR: 1.43e-04 | 3200.1ms/step | 18.7 samples/s +Step 32610/50462 | Loss: 2.5313 | LR: 1.43e-04 | 3197.9ms/step | 18.8 samples/s +Step 32620/50462 | Loss: 2.5483 | LR: 1.43e-04 | 3200.1ms/step | 18.7 samples/s +Step 32630/50462 | Loss: 2.5303 | LR: 1.43e-04 | 3199.8ms/step | 18.8 samples/s +Step 32640/50462 | Loss: 2.5373 | LR: 1.42e-04 | 3199.5ms/step | 18.8 samples/s +Step 32650/50462 | Loss: 2.5385 | LR: 1.42e-04 | 3199.2ms/step | 18.8 samples/s +Step 32660/50462 | Loss: 2.5534 | LR: 1.42e-04 | 3199.2ms/step | 18.8 samples/s +Step 32670/50462 | Loss: 2.5573 | LR: 1.42e-04 | 3198.2ms/step | 18.8 samples/s +Step 32680/50462 | Loss: 2.5466 | LR: 1.42e-04 | 3195.9ms/step | 18.8 samples/s +Step 32690/50462 | Loss: 2.5145 | LR: 1.42e-04 | 3200.1ms/step | 18.7 samples/s +Step 32700/50462 | Loss: 2.5137 | LR: 1.42e-04 | 3199.6ms/step | 18.8 samples/s +Step 32710/50462 | Loss: 2.5620 | LR: 1.41e-04 | 3200.3ms/step | 18.7 samples/s +Step 32720/50462 | Loss: 2.5509 | LR: 1.41e-04 | 3198.1ms/step | 18.8 samples/s +Step 32730/50462 | Loss: 2.5255 | LR: 1.41e-04 | 3196.0ms/step | 18.8 samples/s +Step 32740/50462 | Loss: 2.5603 | LR: 1.41e-04 | 3200.0ms/step | 18.8 samples/s +Step 32750/50462 | Loss: 2.5498 | LR: 1.41e-04 | 3199.9ms/step | 18.8 samples/s +Validation at step 32750: Loss = 2.5516, Perplexity = 12.83 +Step 32760/50462 | Loss: 2.5569 | LR: 1.41e-04 | 4928.1ms/step | 12.2 samples/s +Step 32770/50462 | Loss: 2.5388 | LR: 1.41e-04 | 3252.2ms/step | 18.4 samples/s +Step 32780/50462 | Loss: 2.5238 | LR: 1.41e-04 | 3199.8ms/step | 18.8 samples/s +Step 32790/50462 | Loss: 2.5499 | LR: 1.40e-04 | 3247.9ms/step | 18.5 samples/s +Step 32800/50462 | Loss: 2.5079 | LR: 1.40e-04 | 3198.1ms/step | 18.8 samples/s +Step 32810/50462 | Loss: 2.5373 | LR: 1.40e-04 | 3191.3ms/step | 18.8 samples/s +Step 32820/50462 | Loss: 2.5539 | LR: 1.40e-04 | 3196.7ms/step | 18.8 samples/s +Step 32830/50462 | Loss: 2.5536 | LR: 1.40e-04 | 3200.0ms/step | 18.8 samples/s +Step 32840/50462 | Loss: 2.5670 | LR: 1.40e-04 | 3196.3ms/step | 18.8 samples/s +Step 32850/50462 | Loss: 2.5828 | LR: 1.40e-04 | 3197.8ms/step | 18.8 samples/s +Step 32860/50462 | Loss: 2.5478 | LR: 1.39e-04 | 3199.8ms/step | 18.8 samples/s +Step 32870/50462 | Loss: 2.5268 | LR: 1.39e-04 | 3195.1ms/step | 18.8 samples/s +Step 32880/50462 | Loss: 2.5321 | LR: 1.39e-04 | 3192.5ms/step | 18.8 samples/s +Step 32890/50462 | Loss: 2.5594 | LR: 1.39e-04 | 3196.5ms/step | 18.8 samples/s +Step 32900/50462 | Loss: 2.5418 | LR: 1.39e-04 | 3189.8ms/step | 18.8 samples/s +Step 32910/50462 | Loss: 2.5302 | LR: 1.39e-04 | 3200.4ms/step | 18.7 samples/s +Step 32920/50462 | Loss: 2.5460 | LR: 1.39e-04 | 3199.3ms/step | 18.8 samples/s +Step 32930/50462 | Loss: 2.5390 | LR: 1.38e-04 | 3193.8ms/step | 18.8 samples/s +Step 32940/50462 | Loss: 2.5348 | LR: 1.38e-04 | 3196.3ms/step | 18.8 samples/s +Step 32950/50462 | Loss: 2.5242 | LR: 1.38e-04 | 3195.8ms/step | 18.8 samples/s +Step 32960/50462 | Loss: 2.5103 | LR: 1.38e-04 | 3196.7ms/step | 18.8 samples/s +Step 32970/50462 | Loss: 2.5591 | LR: 1.38e-04 | 3197.8ms/step | 18.8 samples/s +Step 32980/50462 | Loss: 2.5263 | LR: 1.38e-04 | 3199.6ms/step | 18.8 samples/s +Step 32990/50462 | Loss: 2.5211 | LR: 1.38e-04 | 3197.4ms/step | 18.8 samples/s +Step 33000/50462 | Loss: 2.5356 | LR: 1.37e-04 | 3199.0ms/step | 18.8 samples/s +Validation at step 33000: Loss = 2.5506, Perplexity = 12.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-33000 +Step 33010/50462 | Loss: 2.5500 | LR: 1.37e-04 | 5786.1ms/step | 10.4 samples/s +Step 33020/50462 | Loss: 2.5252 | LR: 1.37e-04 | 3182.2ms/step | 18.9 samples/s +Step 33030/50462 | Loss: 2.5555 | LR: 1.37e-04 | 3190.0ms/step | 18.8 samples/s +Step 33040/50462 | Loss: 2.5442 | LR: 1.37e-04 | 3197.8ms/step | 18.8 samples/s +Step 33050/50462 | Loss: 2.5086 | LR: 1.37e-04 | 3195.0ms/step | 18.8 samples/s +Step 33060/50462 | Loss: 2.5404 | LR: 1.37e-04 | 3198.8ms/step | 18.8 samples/s +Step 33070/50462 | Loss: 2.5334 | LR: 1.36e-04 | 3198.6ms/step | 18.8 samples/s +Step 33080/50462 | Loss: 2.5391 | LR: 1.36e-04 | 3199.9ms/step | 18.8 samples/s +Step 33090/50462 | Loss: 2.5368 | LR: 1.36e-04 | 3199.9ms/step | 18.8 samples/s +Step 33100/50462 | Loss: 2.5248 | LR: 1.36e-04 | 3253.9ms/step | 18.4 samples/s +Step 33110/50462 | Loss: 2.5215 | LR: 1.36e-04 | 3252.3ms/step | 18.4 samples/s +Step 33120/50462 | Loss: 2.5314 | LR: 1.36e-04 | 3198.0ms/step | 18.8 samples/s +Step 33130/50462 | Loss: 2.5268 | LR: 1.36e-04 | 3249.3ms/step | 18.5 samples/s +Step 33140/50462 | Loss: 2.5380 | LR: 1.35e-04 | 3198.6ms/step | 18.8 samples/s +Step 33150/50462 | Loss: 2.5453 | LR: 1.35e-04 | 3193.7ms/step | 18.8 samples/s +Step 33160/50462 | Loss: 2.5678 | LR: 1.35e-04 | 3199.9ms/step | 18.8 samples/s +Step 33170/50462 | Loss: 2.5450 | LR: 1.35e-04 | 3200.3ms/step | 18.7 samples/s +Step 33180/50462 | Loss: 2.5303 | LR: 1.35e-04 | 3199.9ms/step | 18.8 samples/s +Step 33190/50462 | Loss: 2.5198 | LR: 1.35e-04 | 3200.5ms/step | 18.7 samples/s +Step 33200/50462 | Loss: 2.5440 | LR: 1.35e-04 | 3199.2ms/step | 18.8 samples/s +Step 33210/50462 | Loss: 2.5712 | LR: 1.34e-04 | 3195.8ms/step | 18.8 samples/s +Step 33220/50462 | Loss: 2.5452 | LR: 1.34e-04 | 3198.5ms/step | 18.8 samples/s +Step 33230/50462 | Loss: 2.5405 | LR: 1.34e-04 | 3200.1ms/step | 18.7 samples/s +Step 33240/50462 | Loss: 2.4913 | LR: 1.34e-04 | 3198.2ms/step | 18.8 samples/s +Step 33250/50462 | Loss: 2.5548 | LR: 1.34e-04 | 3199.6ms/step | 18.8 samples/s +Validation at step 33250: Loss = 2.5497, Perplexity = 12.80 +Step 33260/50462 | Loss: 2.5056 | LR: 1.34e-04 | 4874.0ms/step | 12.3 samples/s +Step 33270/50462 | Loss: 2.5509 | LR: 1.34e-04 | 3198.1ms/step | 18.8 samples/s +Step 33280/50462 | Loss: 2.5517 | LR: 1.33e-04 | 3198.1ms/step | 18.8 samples/s +Step 33290/50462 | Loss: 2.5338 | LR: 1.33e-04 | 3200.0ms/step | 18.7 samples/s +Step 33300/50462 | Loss: 2.5459 | LR: 1.33e-04 | 3199.5ms/step | 18.8 samples/s +Step 33310/50462 | Loss: 2.5610 | LR: 1.33e-04 | 3199.4ms/step | 18.8 samples/s +Step 33320/50462 | Loss: 2.5459 | LR: 1.33e-04 | 3195.0ms/step | 18.8 samples/s +Step 33330/50462 | Loss: 2.5685 | LR: 1.33e-04 | 3199.7ms/step | 18.8 samples/s +Step 33340/50462 | Loss: 2.5907 | LR: 1.33e-04 | 3198.3ms/step | 18.8 samples/s +Step 33350/50462 | Loss: 2.5422 | LR: 1.32e-04 | 3200.3ms/step | 18.7 samples/s +Step 33360/50462 | Loss: 2.5181 | LR: 1.32e-04 | 3199.8ms/step | 18.8 samples/s +Step 33370/50462 | Loss: 2.5125 | LR: 1.32e-04 | 3199.9ms/step | 18.8 samples/s +Step 33380/50462 | Loss: 2.5474 | LR: 1.32e-04 | 3196.0ms/step | 18.8 samples/s +Step 33390/50462 | Loss: 2.5206 | LR: 1.32e-04 | 3200.1ms/step | 18.7 samples/s +Step 33400/50462 | Loss: 2.5260 | LR: 1.32e-04 | 3199.6ms/step | 18.8 samples/s +Step 33410/50462 | Loss: 2.5242 | LR: 1.32e-04 | 3198.3ms/step | 18.8 samples/s +Step 33420/50462 | Loss: 2.5299 | LR: 1.32e-04 | 3198.1ms/step | 18.8 samples/s +Step 33430/50462 | Loss: 2.5436 | LR: 1.31e-04 | 3200.1ms/step | 18.7 samples/s +Step 33440/50462 | Loss: 2.5840 | LR: 1.31e-04 | 3253.6ms/step | 18.4 samples/s +Step 33450/50462 | Loss: 2.5506 | LR: 1.31e-04 | 3245.6ms/step | 18.5 samples/s +Step 33460/50462 | Loss: 2.5269 | LR: 1.31e-04 | 3192.8ms/step | 18.8 samples/s +Step 33470/50462 | Loss: 2.5289 | LR: 1.31e-04 | 3246.8ms/step | 18.5 samples/s +Step 33480/50462 | Loss: 2.5127 | LR: 1.31e-04 | 3196.0ms/step | 18.8 samples/s +Step 33490/50462 | Loss: 2.5527 | LR: 1.31e-04 | 3195.6ms/step | 18.8 samples/s +Step 33500/50462 | Loss: 2.5295 | LR: 1.30e-04 | 3194.9ms/step | 18.8 samples/s +Validation at step 33500: Loss = 2.5489, Perplexity = 12.79 +Step 33510/50462 | Loss: 2.5520 | LR: 1.30e-04 | 4872.6ms/step | 12.3 samples/s +Step 33520/50462 | Loss: 2.5341 | LR: 1.30e-04 | 3199.5ms/step | 18.8 samples/s +Step 33530/50462 | Loss: 2.5040 | LR: 1.30e-04 | 3198.5ms/step | 18.8 samples/s +Step 33540/50462 | Loss: 2.5387 | LR: 1.30e-04 | 3198.1ms/step | 18.8 samples/s +Step 33550/50462 | Loss: 2.5409 | LR: 1.30e-04 | 3199.7ms/step | 18.8 samples/s +Step 33560/50462 | Loss: 2.5268 | LR: 1.30e-04 | 3200.4ms/step | 18.7 samples/s +Step 33570/50462 | Loss: 2.4982 | LR: 1.29e-04 | 3199.5ms/step | 18.8 samples/s +Step 33580/50462 | Loss: 2.5364 | LR: 1.29e-04 | 3198.3ms/step | 18.8 samples/s +Step 33590/50462 | Loss: 2.5362 | LR: 1.29e-04 | 3197.9ms/step | 18.8 samples/s +Step 33600/50462 | Loss: 2.5301 | LR: 1.29e-04 | 3197.5ms/step | 18.8 samples/s +Step 33610/50462 | Loss: 2.5534 | LR: 1.29e-04 | 3199.8ms/step | 18.8 samples/s +Step 33620/50462 | Loss: 2.5374 | LR: 1.29e-04 | 3200.0ms/step | 18.8 samples/s +Step 33630/50462 | Loss: 2.4917 | LR: 1.29e-04 | 3198.8ms/step | 18.8 samples/s +Step 33640/50462 | Loss: 2.5402 | LR: 1.28e-04 | 3198.1ms/step | 18.8 samples/s +Step 33650/50462 | Loss: 2.5328 | LR: 1.28e-04 | 3198.1ms/step | 18.8 samples/s +Step 33660/50462 | Loss: 2.5317 | LR: 1.28e-04 | 3197.9ms/step | 18.8 samples/s +Step 33670/50462 | Loss: 2.5235 | LR: 1.28e-04 | 3197.4ms/step | 18.8 samples/s +Step 33680/50462 | Loss: 2.5492 | LR: 1.28e-04 | 3196.1ms/step | 18.8 samples/s +Step 33690/50462 | Loss: 2.5339 | LR: 1.28e-04 | 3196.3ms/step | 18.8 samples/s +Step 33700/50462 | Loss: 2.5179 | LR: 1.28e-04 | 3197.5ms/step | 18.8 samples/s +Step 33710/50462 | Loss: 2.5605 | LR: 1.27e-04 | 3200.7ms/step | 18.7 samples/s +Step 33720/50462 | Loss: 2.5437 | LR: 1.27e-04 | 3196.0ms/step | 18.8 samples/s +Step 33730/50462 | Loss: 2.5109 | LR: 1.27e-04 | 3199.2ms/step | 18.8 samples/s +Step 33740/50462 | Loss: 2.5314 | LR: 1.27e-04 | 3196.4ms/step | 18.8 samples/s +Step 33750/50462 | Loss: 2.5469 | LR: 1.27e-04 | 3197.8ms/step | 18.8 samples/s +Validation at step 33750: Loss = 2.5480, Perplexity = 12.78 +Step 33760/50462 | Loss: 2.5074 | LR: 1.27e-04 | 4872.7ms/step | 12.3 samples/s +Step 33770/50462 | Loss: 2.5298 | LR: 1.27e-04 | 3251.8ms/step | 18.5 samples/s +Step 33780/50462 | Loss: 2.5470 | LR: 1.27e-04 | 3243.5ms/step | 18.5 samples/s +Step 33790/50462 | Loss: 2.5484 | LR: 1.26e-04 | 3184.4ms/step | 18.8 samples/s +Step 33800/50462 | Loss: 2.5372 | LR: 1.26e-04 | 3188.2ms/step | 18.8 samples/s +Step 33810/50462 | Loss: 2.5436 | LR: 1.26e-04 | 3249.3ms/step | 18.5 samples/s +Step 33820/50462 | Loss: 2.5075 | LR: 1.26e-04 | 3188.0ms/step | 18.8 samples/s +Step 33830/50462 | Loss: 2.5037 | LR: 1.26e-04 | 3194.6ms/step | 18.8 samples/s +Step 33840/50462 | Loss: 2.5456 | LR: 1.26e-04 | 3197.6ms/step | 18.8 samples/s +Step 33850/50462 | Loss: 2.5306 | LR: 1.26e-04 | 3195.8ms/step | 18.8 samples/s +Step 33860/50462 | Loss: 2.5316 | LR: 1.25e-04 | 3193.2ms/step | 18.8 samples/s +Step 33870/50462 | Loss: 2.5175 | LR: 1.25e-04 | 3197.5ms/step | 18.8 samples/s +Step 33880/50462 | Loss: 2.5758 | LR: 1.25e-04 | 3198.0ms/step | 18.8 samples/s +Step 33890/50462 | Loss: 2.5401 | LR: 1.25e-04 | 3195.3ms/step | 18.8 samples/s +Step 33900/50462 | Loss: 2.5598 | LR: 1.25e-04 | 3198.3ms/step | 18.8 samples/s +Step 33910/50462 | Loss: 2.5277 | LR: 1.25e-04 | 3194.1ms/step | 18.8 samples/s +Step 33920/50462 | Loss: 2.5646 | LR: 1.25e-04 | 3196.1ms/step | 18.8 samples/s +Step 33930/50462 | Loss: 2.5134 | LR: 1.24e-04 | 3200.0ms/step | 18.7 samples/s +Step 33940/50462 | Loss: 2.5261 | LR: 1.24e-04 | 3198.0ms/step | 18.8 samples/s +Step 33950/50462 | Loss: 2.5372 | LR: 1.24e-04 | 3199.4ms/step | 18.8 samples/s +Step 33960/50462 | Loss: 2.5664 | LR: 1.24e-04 | 3196.2ms/step | 18.8 samples/s +Step 33970/50462 | Loss: 2.5016 | LR: 1.24e-04 | 3195.9ms/step | 18.8 samples/s +Step 33980/50462 | Loss: 2.5475 | LR: 1.24e-04 | 3194.0ms/step | 18.8 samples/s +Step 33990/50462 | Loss: 2.5399 | LR: 1.24e-04 | 3200.1ms/step | 18.7 samples/s +Step 34000/50462 | Loss: 2.5009 | LR: 1.24e-04 | 3194.5ms/step | 18.8 samples/s +Validation at step 34000: Loss = 2.5469, Perplexity = 12.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-34000 +Step 34010/50462 | Loss: 2.5290 | LR: 1.23e-04 | 5792.0ms/step | 10.4 samples/s +Step 34020/50462 | Loss: 2.5394 | LR: 1.23e-04 | 3186.7ms/step | 18.8 samples/s +Step 34030/50462 | Loss: 2.5117 | LR: 1.23e-04 | 3195.3ms/step | 18.8 samples/s +Step 34040/50462 | Loss: 2.5434 | LR: 1.23e-04 | 3198.1ms/step | 18.8 samples/s +Step 34050/50462 | Loss: 2.5546 | LR: 1.23e-04 | 3196.0ms/step | 18.8 samples/s +Step 34060/50462 | Loss: 2.5417 | LR: 1.23e-04 | 3194.5ms/step | 18.8 samples/s +Step 34070/50462 | Loss: 2.5305 | LR: 1.23e-04 | 3199.5ms/step | 18.8 samples/s +Step 34080/50462 | Loss: 2.5555 | LR: 1.22e-04 | 3199.9ms/step | 18.8 samples/s +Step 34090/50462 | Loss: 2.5526 | LR: 1.22e-04 | 3195.7ms/step | 18.8 samples/s +Step 34100/50462 | Loss: 2.5549 | LR: 1.22e-04 | 3198.3ms/step | 18.8 samples/s +Step 34110/50462 | Loss: 2.5391 | LR: 1.22e-04 | 3253.7ms/step | 18.4 samples/s +Step 34120/50462 | Loss: 2.5325 | LR: 1.22e-04 | 3196.3ms/step | 18.8 samples/s +Step 34130/50462 | Loss: 2.5163 | LR: 1.22e-04 | 3247.4ms/step | 18.5 samples/s +Step 34140/50462 | Loss: 2.5610 | LR: 1.22e-04 | 3198.0ms/step | 18.8 samples/s +Step 34150/50462 | Loss: 2.5136 | LR: 1.21e-04 | 3252.6ms/step | 18.4 samples/s +Step 34160/50462 | Loss: 2.5456 | LR: 1.21e-04 | 3200.0ms/step | 18.7 samples/s +Step 34170/50462 | Loss: 2.5216 | LR: 1.21e-04 | 3200.0ms/step | 18.7 samples/s +Step 34180/50462 | Loss: 2.5351 | LR: 1.21e-04 | 3200.1ms/step | 18.7 samples/s +Step 34190/50462 | Loss: 2.5263 | LR: 1.21e-04 | 3197.9ms/step | 18.8 samples/s +Step 34200/50462 | Loss: 2.5223 | LR: 1.21e-04 | 3195.8ms/step | 18.8 samples/s +Step 34210/50462 | Loss: 2.5346 | LR: 1.21e-04 | 3198.0ms/step | 18.8 samples/s +Step 34220/50462 | Loss: 2.5370 | LR: 1.21e-04 | 3200.0ms/step | 18.7 samples/s +Step 34230/50462 | Loss: 2.5460 | LR: 1.20e-04 | 3200.3ms/step | 18.7 samples/s +Step 34240/50462 | Loss: 2.5168 | LR: 1.20e-04 | 3195.6ms/step | 18.8 samples/s +Step 34250/50462 | Loss: 2.5269 | LR: 1.20e-04 | 3199.8ms/step | 18.8 samples/s +Validation at step 34250: Loss = 2.5463, Perplexity = 12.76 +Step 34260/50462 | Loss: 2.4858 | LR: 1.20e-04 | 4871.1ms/step | 12.3 samples/s +Step 34270/50462 | Loss: 2.5496 | LR: 1.20e-04 | 3196.8ms/step | 18.8 samples/s +Step 34280/50462 | Loss: 2.5326 | LR: 1.20e-04 | 3198.3ms/step | 18.8 samples/s +Step 34290/50462 | Loss: 2.5554 | LR: 1.20e-04 | 3193.9ms/step | 18.8 samples/s +Step 34300/50462 | Loss: 2.5528 | LR: 1.19e-04 | 3194.2ms/step | 18.8 samples/s +Step 34310/50462 | Loss: 2.5662 | LR: 1.19e-04 | 3195.6ms/step | 18.8 samples/s +Step 34320/50462 | Loss: 2.5145 | LR: 1.19e-04 | 3196.7ms/step | 18.8 samples/s +Step 34330/50462 | Loss: 2.5229 | LR: 1.19e-04 | 3196.1ms/step | 18.8 samples/s +Step 34340/50462 | Loss: 2.5137 | LR: 1.19e-04 | 3200.3ms/step | 18.7 samples/s +Step 34350/50462 | Loss: 2.5282 | LR: 1.19e-04 | 3196.1ms/step | 18.8 samples/s +Step 34360/50462 | Loss: 2.5396 | LR: 1.19e-04 | 3198.4ms/step | 18.8 samples/s +Step 34370/50462 | Loss: 2.5256 | LR: 1.19e-04 | 3195.3ms/step | 18.8 samples/s +Step 34380/50462 | Loss: 2.5073 | LR: 1.18e-04 | 3192.5ms/step | 18.8 samples/s +Step 34390/50462 | Loss: 2.5337 | LR: 1.18e-04 | 3195.1ms/step | 18.8 samples/s +Step 34400/50462 | Loss: 2.5421 | LR: 1.18e-04 | 3195.8ms/step | 18.8 samples/s +Step 34410/50462 | Loss: 2.5281 | LR: 1.18e-04 | 3196.1ms/step | 18.8 samples/s +Step 34420/50462 | Loss: 2.5192 | LR: 1.18e-04 | 3197.7ms/step | 18.8 samples/s +Step 34430/50462 | Loss: 2.5394 | LR: 1.18e-04 | 3196.6ms/step | 18.8 samples/s +Step 34440/50462 | Loss: 2.5236 | LR: 1.18e-04 | 3200.2ms/step | 18.7 samples/s +Step 34450/50462 | Loss: 2.5567 | LR: 1.17e-04 | 3250.1ms/step | 18.5 samples/s +Step 34460/50462 | Loss: 2.5172 | LR: 1.17e-04 | 3252.8ms/step | 18.4 samples/s +Step 34470/50462 | Loss: 2.5553 | LR: 1.17e-04 | 3183.2ms/step | 18.8 samples/s +Step 34480/50462 | Loss: 2.5587 | LR: 1.17e-04 | 3184.2ms/step | 18.8 samples/s +Step 34490/50462 | Loss: 2.5284 | LR: 1.17e-04 | 3248.9ms/step | 18.5 samples/s +Step 34500/50462 | Loss: 2.5395 | LR: 1.17e-04 | 3198.1ms/step | 18.8 samples/s +Validation at step 34500: Loss = 2.5458, Perplexity = 12.75 +Step 34510/50462 | Loss: 2.5365 | LR: 1.17e-04 | 4874.4ms/step | 12.3 samples/s +Step 34520/50462 | Loss: 2.5195 | LR: 1.16e-04 | 3200.2ms/step | 18.7 samples/s +Step 34530/50462 | Loss: 2.5256 | LR: 1.16e-04 | 3195.7ms/step | 18.8 samples/s +Step 34540/50462 | Loss: 2.5284 | LR: 1.16e-04 | 3197.6ms/step | 18.8 samples/s +Step 34550/50462 | Loss: 2.5106 | LR: 1.16e-04 | 3198.8ms/step | 18.8 samples/s +Step 34560/50462 | Loss: 2.5482 | LR: 1.16e-04 | 3194.6ms/step | 18.8 samples/s +Step 34570/50462 | Loss: 2.5172 | LR: 1.16e-04 | 3199.5ms/step | 18.8 samples/s +Step 34580/50462 | Loss: 2.5319 | LR: 1.16e-04 | 3197.0ms/step | 18.8 samples/s +Step 34590/50462 | Loss: 2.5365 | LR: 1.16e-04 | 3196.7ms/step | 18.8 samples/s +Step 34600/50462 | Loss: 2.5254 | LR: 1.15e-04 | 3195.9ms/step | 18.8 samples/s +Step 34610/50462 | Loss: 2.5343 | LR: 1.15e-04 | 3197.7ms/step | 18.8 samples/s +Step 34620/50462 | Loss: 2.5408 | LR: 1.15e-04 | 3199.8ms/step | 18.8 samples/s +Step 34630/50462 | Loss: 2.5288 | LR: 1.15e-04 | 3198.7ms/step | 18.8 samples/s +Step 34640/50462 | Loss: 2.5585 | LR: 1.15e-04 | 3198.2ms/step | 18.8 samples/s +Step 34650/50462 | Loss: 2.5439 | LR: 1.15e-04 | 3197.3ms/step | 18.8 samples/s +Step 34660/50462 | Loss: 2.5431 | LR: 1.15e-04 | 3194.6ms/step | 18.8 samples/s +Step 34670/50462 | Loss: 2.5582 | LR: 1.15e-04 | 3194.8ms/step | 18.8 samples/s +Step 34680/50462 | Loss: 2.5483 | LR: 1.14e-04 | 3193.3ms/step | 18.8 samples/s +Step 34690/50462 | Loss: 2.5452 | LR: 1.14e-04 | 3197.6ms/step | 18.8 samples/s +Step 34700/50462 | Loss: 2.5424 | LR: 1.14e-04 | 3200.4ms/step | 18.7 samples/s +Step 34710/50462 | Loss: 2.5344 | LR: 1.14e-04 | 3197.1ms/step | 18.8 samples/s +Step 34720/50462 | Loss: 2.5294 | LR: 1.14e-04 | 3198.1ms/step | 18.8 samples/s +Step 34730/50462 | Loss: 2.5303 | LR: 1.14e-04 | 3197.0ms/step | 18.8 samples/s +Step 34740/50462 | Loss: 2.5440 | LR: 1.14e-04 | 3199.0ms/step | 18.8 samples/s +Step 34750/50462 | Loss: 2.5447 | LR: 1.13e-04 | 3198.1ms/step | 18.8 samples/s +Validation at step 34750: Loss = 2.5448, Perplexity = 12.74 +Step 34760/50462 | Loss: 2.5319 | LR: 1.13e-04 | 4872.2ms/step | 12.3 samples/s +Step 34770/50462 | Loss: 2.5143 | LR: 1.13e-04 | 3197.4ms/step | 18.8 samples/s +Step 34780/50462 | Loss: 2.5299 | LR: 1.13e-04 | 3197.2ms/step | 18.8 samples/s +Step 34790/50462 | Loss: 2.5213 | LR: 1.13e-04 | 3243.4ms/step | 18.5 samples/s +Step 34800/50462 | Loss: 2.5351 | LR: 1.13e-04 | 3244.5ms/step | 18.5 samples/s +Step 34810/50462 | Loss: 2.5492 | LR: 1.13e-04 | 3199.9ms/step | 18.8 samples/s +Step 34820/50462 | Loss: 2.5431 | LR: 1.13e-04 | 3198.0ms/step | 18.8 samples/s +Step 34830/50462 | Loss: 2.5350 | LR: 1.12e-04 | 3251.9ms/step | 18.5 samples/s +Step 34840/50462 | Loss: 2.5482 | LR: 1.12e-04 | 3198.0ms/step | 18.8 samples/s +Step 34850/50462 | Loss: 2.5680 | LR: 1.12e-04 | 3197.0ms/step | 18.8 samples/s +Step 34860/50462 | Loss: 2.5506 | LR: 1.12e-04 | 3199.0ms/step | 18.8 samples/s +Step 34870/50462 | Loss: 2.5133 | LR: 1.12e-04 | 3197.7ms/step | 18.8 samples/s +Step 34880/50462 | Loss: 2.5373 | LR: 1.12e-04 | 3200.4ms/step | 18.7 samples/s +Step 34890/50462 | Loss: 2.5272 | LR: 1.12e-04 | 3198.0ms/step | 18.8 samples/s +Step 34900/50462 | Loss: 2.5339 | LR: 1.11e-04 | 3199.8ms/step | 18.8 samples/s +Step 34910/50462 | Loss: 2.5426 | LR: 1.11e-04 | 3198.0ms/step | 18.8 samples/s +Step 34920/50462 | Loss: 2.4804 | LR: 1.11e-04 | 3197.8ms/step | 18.8 samples/s +Step 34930/50462 | Loss: 2.5191 | LR: 1.11e-04 | 3199.5ms/step | 18.8 samples/s +Step 34940/50462 | Loss: 2.5338 | LR: 1.11e-04 | 3198.8ms/step | 18.8 samples/s +Step 34950/50462 | Loss: 2.5361 | LR: 1.11e-04 | 3199.5ms/step | 18.8 samples/s +Step 34960/50462 | Loss: 2.5380 | LR: 1.11e-04 | 3197.1ms/step | 18.8 samples/s +Step 34970/50462 | Loss: 2.5177 | LR: 1.11e-04 | 3198.7ms/step | 18.8 samples/s +Step 34980/50462 | Loss: 2.5210 | LR: 1.10e-04 | 3198.5ms/step | 18.8 samples/s +Step 34990/50462 | Loss: 2.5519 | LR: 1.10e-04 | 3197.9ms/step | 18.8 samples/s +Step 35000/50462 | Loss: 2.5621 | LR: 1.10e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 35000: Loss = 2.5442, Perplexity = 12.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-35000 +Step 35010/50462 | Loss: 2.5419 | LR: 1.10e-04 | 5782.6ms/step | 10.4 samples/s +Step 35020/50462 | Loss: 2.5258 | LR: 1.10e-04 | 3188.7ms/step | 18.8 samples/s +Step 35030/50462 | Loss: 2.5355 | LR: 1.10e-04 | 3196.6ms/step | 18.8 samples/s +Step 35040/50462 | Loss: 2.5217 | LR: 1.10e-04 | 3197.8ms/step | 18.8 samples/s +Step 35050/50462 | Loss: 2.5248 | LR: 1.09e-04 | 3196.2ms/step | 18.8 samples/s +Step 35060/50462 | Loss: 2.5153 | LR: 1.09e-04 | 3198.4ms/step | 18.8 samples/s +Step 35070/50462 | Loss: 2.5136 | LR: 1.09e-04 | 3195.9ms/step | 18.8 samples/s +Step 35080/50462 | Loss: 2.5513 | LR: 1.09e-04 | 3199.3ms/step | 18.8 samples/s +Step 35090/50462 | Loss: 2.5071 | LR: 1.09e-04 | 3198.7ms/step | 18.8 samples/s +Step 35100/50462 | Loss: 2.5377 | LR: 1.09e-04 | 3200.0ms/step | 18.8 samples/s +Step 35110/50462 | Loss: 2.5402 | LR: 1.09e-04 | 3198.1ms/step | 18.8 samples/s +Step 35120/50462 | Loss: 2.5229 | LR: 1.09e-04 | 3255.4ms/step | 18.4 samples/s +Step 35130/50462 | Loss: 2.5035 | LR: 1.08e-04 | 3198.6ms/step | 18.8 samples/s +Step 35140/50462 | Loss: 2.5130 | LR: 1.08e-04 | 3253.6ms/step | 18.4 samples/s +Step 35150/50462 | Loss: 2.5318 | LR: 1.08e-04 | 3199.3ms/step | 18.8 samples/s +Step 35160/50462 | Loss: 2.5432 | LR: 1.08e-04 | 3196.2ms/step | 18.8 samples/s +Step 35170/50462 | Loss: 2.5356 | LR: 1.08e-04 | 3241.9ms/step | 18.5 samples/s +Step 35180/50462 | Loss: 2.5287 | LR: 1.08e-04 | 3186.5ms/step | 18.8 samples/s +Step 35190/50462 | Loss: 2.5404 | LR: 1.08e-04 | 3188.7ms/step | 18.8 samples/s +Step 35200/50462 | Loss: 2.5320 | LR: 1.08e-04 | 3191.4ms/step | 18.8 samples/s +Step 35210/50462 | Loss: 2.5297 | LR: 1.07e-04 | 3195.0ms/step | 18.8 samples/s +Step 35220/50462 | Loss: 2.5351 | LR: 1.07e-04 | 3197.1ms/step | 18.8 samples/s +Step 35230/50462 | Loss: 2.5446 | LR: 1.07e-04 | 3200.1ms/step | 18.7 samples/s +Step 35240/50462 | Loss: 2.5375 | LR: 1.07e-04 | 3199.9ms/step | 18.8 samples/s +Step 35250/50462 | Loss: 2.5117 | LR: 1.07e-04 | 3198.2ms/step | 18.8 samples/s +Validation at step 35250: Loss = 2.5436, Perplexity = 12.73 +Step 35260/50462 | Loss: 2.4930 | LR: 1.07e-04 | 4869.3ms/step | 12.3 samples/s +Step 35270/50462 | Loss: 2.5183 | LR: 1.07e-04 | 3192.4ms/step | 18.8 samples/s +Step 35280/50462 | Loss: 2.5224 | LR: 1.07e-04 | 3200.2ms/step | 18.7 samples/s +Step 35290/50462 | Loss: 2.5036 | LR: 1.06e-04 | 3196.7ms/step | 18.8 samples/s +Step 35300/50462 | Loss: 2.5243 | LR: 1.06e-04 | 3192.5ms/step | 18.8 samples/s +Step 35310/50462 | Loss: 2.5473 | LR: 1.06e-04 | 3200.7ms/step | 18.7 samples/s +Step 35320/50462 | Loss: 2.5391 | LR: 1.06e-04 | 3200.2ms/step | 18.7 samples/s +Step 35330/50462 | Loss: 2.5507 | LR: 1.06e-04 | 3196.0ms/step | 18.8 samples/s +Step 35340/50462 | Loss: 2.5311 | LR: 1.06e-04 | 3197.6ms/step | 18.8 samples/s +Step 35350/50462 | Loss: 2.5414 | LR: 1.06e-04 | 3198.4ms/step | 18.8 samples/s +Step 35360/50462 | Loss: 2.5311 | LR: 1.05e-04 | 3194.3ms/step | 18.8 samples/s +Step 35370/50462 | Loss: 2.5447 | LR: 1.05e-04 | 3189.7ms/step | 18.8 samples/s +Step 35380/50462 | Loss: 2.5345 | LR: 1.05e-04 | 3193.2ms/step | 18.8 samples/s +Step 35390/50462 | Loss: 2.5358 | LR: 1.05e-04 | 3194.7ms/step | 18.8 samples/s +Step 35400/50462 | Loss: 2.5575 | LR: 1.05e-04 | 3197.1ms/step | 18.8 samples/s +Step 35410/50462 | Loss: 2.5372 | LR: 1.05e-04 | 3199.0ms/step | 18.8 samples/s +Step 35420/50462 | Loss: 2.5327 | LR: 1.05e-04 | 3200.1ms/step | 18.7 samples/s +Step 35430/50462 | Loss: 2.5320 | LR: 1.05e-04 | 3198.0ms/step | 18.8 samples/s +Step 35440/50462 | Loss: 2.5330 | LR: 1.04e-04 | 3200.2ms/step | 18.7 samples/s +Step 35450/50462 | Loss: 2.5283 | LR: 1.04e-04 | 3199.6ms/step | 18.8 samples/s +Step 35460/50462 | Loss: 2.5323 | LR: 1.04e-04 | 3250.2ms/step | 18.5 samples/s +Step 35470/50462 | Loss: 2.5277 | LR: 1.04e-04 | 3196.3ms/step | 18.8 samples/s +Step 35480/50462 | Loss: 2.5192 | LR: 1.04e-04 | 3247.8ms/step | 18.5 samples/s +Step 35490/50462 | Loss: 2.5314 | LR: 1.04e-04 | 3195.3ms/step | 18.8 samples/s +Step 35500/50462 | Loss: 2.5502 | LR: 1.04e-04 | 3186.5ms/step | 18.8 samples/s +Validation at step 35500: Loss = 2.5431, Perplexity = 12.72 +Step 35510/50462 | Loss: 2.5361 | LR: 1.04e-04 | 4916.1ms/step | 12.2 samples/s +Step 35520/50462 | Loss: 2.5170 | LR: 1.03e-04 | 3186.7ms/step | 18.8 samples/s +Step 35530/50462 | Loss: 2.5342 | LR: 1.03e-04 | 3194.8ms/step | 18.8 samples/s +Step 35540/50462 | Loss: 2.4930 | LR: 1.03e-04 | 3197.1ms/step | 18.8 samples/s +Step 35550/50462 | Loss: 2.5369 | LR: 1.03e-04 | 3192.7ms/step | 18.8 samples/s +Step 35560/50462 | Loss: 2.5386 | LR: 1.03e-04 | 3191.8ms/step | 18.8 samples/s +Step 35570/50462 | Loss: 2.5466 | LR: 1.03e-04 | 3200.4ms/step | 18.7 samples/s +Step 35580/50462 | Loss: 2.5242 | LR: 1.03e-04 | 3198.3ms/step | 18.8 samples/s +Step 35590/50462 | Loss: 2.5466 | LR: 1.03e-04 | 3198.0ms/step | 18.8 samples/s +Step 35600/50462 | Loss: 2.4985 | LR: 1.02e-04 | 3199.9ms/step | 18.8 samples/s +Step 35610/50462 | Loss: 2.5168 | LR: 1.02e-04 | 3188.8ms/step | 18.8 samples/s +Step 35620/50462 | Loss: 2.5226 | LR: 1.02e-04 | 3188.8ms/step | 18.8 samples/s +Step 35630/50462 | Loss: 2.5366 | LR: 1.02e-04 | 3186.8ms/step | 18.8 samples/s +Step 35640/50462 | Loss: 2.5073 | LR: 1.02e-04 | 3189.4ms/step | 18.8 samples/s +Step 35650/50462 | Loss: 2.5322 | LR: 1.02e-04 | 3190.1ms/step | 18.8 samples/s +Step 35660/50462 | Loss: 2.5275 | LR: 1.02e-04 | 3197.9ms/step | 18.8 samples/s +Step 35670/50462 | Loss: 2.5115 | LR: 1.02e-04 | 3189.4ms/step | 18.8 samples/s +Step 35680/50462 | Loss: 2.5475 | LR: 1.01e-04 | 3196.9ms/step | 18.8 samples/s +Step 35690/50462 | Loss: 2.5142 | LR: 1.01e-04 | 3195.3ms/step | 18.8 samples/s +Step 35700/50462 | Loss: 2.5252 | LR: 1.01e-04 | 3200.4ms/step | 18.7 samples/s +Step 35710/50462 | Loss: 2.5593 | LR: 1.01e-04 | 3197.7ms/step | 18.8 samples/s +Step 35720/50462 | Loss: 2.4970 | LR: 1.01e-04 | 3192.3ms/step | 18.8 samples/s +Step 35730/50462 | Loss: 2.5572 | LR: 1.01e-04 | 3198.4ms/step | 18.8 samples/s +Step 35740/50462 | Loss: 2.5106 | LR: 1.01e-04 | 3199.2ms/step | 18.8 samples/s +Step 35750/50462 | Loss: 2.5324 | LR: 1.00e-04 | 3200.1ms/step | 18.7 samples/s +Validation at step 35750: Loss = 2.5424, Perplexity = 12.71 +Step 35760/50462 | Loss: 2.5258 | LR: 1.00e-04 | 4872.4ms/step | 12.3 samples/s +Step 35770/50462 | Loss: 2.5320 | LR: 1.00e-04 | 3200.1ms/step | 18.7 samples/s +Step 35780/50462 | Loss: 2.5107 | LR: 1.00e-04 | 3196.5ms/step | 18.8 samples/s +Step 35790/50462 | Loss: 2.5316 | LR: 1.00e-04 | 3195.5ms/step | 18.8 samples/s +Step 35800/50462 | Loss: 2.5192 | LR: 9.99e-05 | 3250.2ms/step | 18.5 samples/s +Step 35810/50462 | Loss: 2.5411 | LR: 9.97e-05 | 3195.1ms/step | 18.8 samples/s +Step 35820/50462 | Loss: 2.5421 | LR: 9.96e-05 | 3246.5ms/step | 18.5 samples/s +Step 35830/50462 | Loss: 2.5235 | LR: 9.95e-05 | 3195.9ms/step | 18.8 samples/s +Step 35840/50462 | Loss: 2.5294 | LR: 9.94e-05 | 3198.1ms/step | 18.8 samples/s +Step 35850/50462 | Loss: 2.5214 | LR: 9.92e-05 | 3260.1ms/step | 18.4 samples/s +Step 35860/50462 | Loss: 2.5219 | LR: 9.91e-05 | 3198.2ms/step | 18.8 samples/s +Step 35870/50462 | Loss: 2.5349 | LR: 9.90e-05 | 3197.7ms/step | 18.8 samples/s +Step 35880/50462 | Loss: 2.5439 | LR: 9.88e-05 | 3200.1ms/step | 18.7 samples/s +Step 35890/50462 | Loss: 2.5361 | LR: 9.87e-05 | 3196.5ms/step | 18.8 samples/s +Step 35900/50462 | Loss: 2.5144 | LR: 9.86e-05 | 3199.9ms/step | 18.8 samples/s +Step 35910/50462 | Loss: 2.5508 | LR: 9.85e-05 | 3197.4ms/step | 18.8 samples/s +Step 35920/50462 | Loss: 2.5446 | LR: 9.83e-05 | 3198.6ms/step | 18.8 samples/s +Step 35930/50462 | Loss: 2.5214 | LR: 9.82e-05 | 3200.1ms/step | 18.7 samples/s +Step 35940/50462 | Loss: 2.5215 | LR: 9.81e-05 | 3197.2ms/step | 18.8 samples/s +Step 35950/50462 | Loss: 2.5043 | LR: 9.80e-05 | 3200.7ms/step | 18.7 samples/s +Step 35960/50462 | Loss: 2.5284 | LR: 9.78e-05 | 3194.8ms/step | 18.8 samples/s +Step 35970/50462 | Loss: 2.5209 | LR: 9.77e-05 | 3194.9ms/step | 18.8 samples/s +Step 35980/50462 | Loss: 2.5171 | LR: 9.76e-05 | 3198.2ms/step | 18.8 samples/s +Step 35990/50462 | Loss: 2.5098 | LR: 9.75e-05 | 3197.6ms/step | 18.8 samples/s +Step 36000/50462 | Loss: 2.5097 | LR: 9.73e-05 | 3200.1ms/step | 18.7 samples/s +Validation at step 36000: Loss = 2.5417, Perplexity = 12.70 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-36000 +Step 36010/50462 | Loss: 2.5051 | LR: 9.72e-05 | 5745.2ms/step | 10.4 samples/s +Step 36020/50462 | Loss: 2.5326 | LR: 9.71e-05 | 3190.7ms/step | 18.8 samples/s +Step 36030/50462 | Loss: 2.5086 | LR: 9.70e-05 | 3194.0ms/step | 18.8 samples/s +Step 36040/50462 | Loss: 2.5638 | LR: 9.68e-05 | 3198.0ms/step | 18.8 samples/s +Step 36050/50462 | Loss: 2.5313 | LR: 9.67e-05 | 3200.2ms/step | 18.7 samples/s +Step 36060/50462 | Loss: 2.5414 | LR: 9.66e-05 | 3200.2ms/step | 18.7 samples/s +Step 36070/50462 | Loss: 2.5196 | LR: 9.65e-05 | 3198.0ms/step | 18.8 samples/s +Step 36080/50462 | Loss: 2.5364 | LR: 9.63e-05 | 3195.2ms/step | 18.8 samples/s +Step 36090/50462 | Loss: 2.5143 | LR: 9.62e-05 | 3196.8ms/step | 18.8 samples/s +Step 36100/50462 | Loss: 2.5480 | LR: 9.61e-05 | 3199.6ms/step | 18.8 samples/s +Step 36110/50462 | Loss: 2.5070 | LR: 9.60e-05 | 3198.2ms/step | 18.8 samples/s +Step 36120/50462 | Loss: 2.5371 | LR: 9.58e-05 | 3199.6ms/step | 18.8 samples/s +Step 36130/50462 | Loss: 2.5401 | LR: 9.57e-05 | 3250.0ms/step | 18.5 samples/s +Step 36140/50462 | Loss: 2.5385 | LR: 9.56e-05 | 3192.0ms/step | 18.8 samples/s +Step 36150/50462 | Loss: 2.5128 | LR: 9.55e-05 | 3192.3ms/step | 18.8 samples/s +Step 36160/50462 | Loss: 2.5130 | LR: 9.53e-05 | 3249.6ms/step | 18.5 samples/s +Step 36170/50462 | Loss: 2.5306 | LR: 9.52e-05 | 3200.3ms/step | 18.7 samples/s +Step 36180/50462 | Loss: 2.5271 | LR: 9.51e-05 | 3200.3ms/step | 18.7 samples/s +Step 36190/50462 | Loss: 2.5377 | LR: 9.50e-05 | 3254.1ms/step | 18.4 samples/s +Step 36200/50462 | Loss: 2.5539 | LR: 9.49e-05 | 3199.7ms/step | 18.8 samples/s +Step 36210/50462 | Loss: 2.5450 | LR: 9.47e-05 | 3197.6ms/step | 18.8 samples/s +Step 36220/50462 | Loss: 2.5192 | LR: 9.46e-05 | 3196.8ms/step | 18.8 samples/s +Step 36230/50462 | Loss: 2.5322 | LR: 9.45e-05 | 3191.2ms/step | 18.8 samples/s +Step 36240/50462 | Loss: 2.5504 | LR: 9.44e-05 | 3188.2ms/step | 18.8 samples/s +Step 36250/50462 | Loss: 2.5172 | LR: 9.42e-05 | 3191.2ms/step | 18.8 samples/s +Validation at step 36250: Loss = 2.5413, Perplexity = 12.70 +Step 36260/50462 | Loss: 2.5379 | LR: 9.41e-05 | 4865.3ms/step | 12.3 samples/s +Step 36270/50462 | Loss: 2.5127 | LR: 9.40e-05 | 3195.6ms/step | 18.8 samples/s +Step 36280/50462 | Loss: 2.5316 | LR: 9.39e-05 | 3196.6ms/step | 18.8 samples/s +Step 36290/50462 | Loss: 2.5001 | LR: 9.37e-05 | 3197.8ms/step | 18.8 samples/s +Step 36300/50462 | Loss: 2.4893 | LR: 9.36e-05 | 3199.9ms/step | 18.8 samples/s +Step 36310/50462 | Loss: 2.5151 | LR: 9.35e-05 | 3199.8ms/step | 18.8 samples/s +Step 36320/50462 | Loss: 2.5241 | LR: 9.34e-05 | 3200.4ms/step | 18.7 samples/s +Step 36330/50462 | Loss: 2.5367 | LR: 9.32e-05 | 3197.1ms/step | 18.8 samples/s +Step 36340/50462 | Loss: 2.5360 | LR: 9.31e-05 | 3196.8ms/step | 18.8 samples/s +Step 36350/50462 | Loss: 2.5322 | LR: 9.30e-05 | 3194.1ms/step | 18.8 samples/s +Step 36360/50462 | Loss: 2.5392 | LR: 9.29e-05 | 3197.9ms/step | 18.8 samples/s +Step 36370/50462 | Loss: 2.5714 | LR: 9.28e-05 | 3196.0ms/step | 18.8 samples/s +Step 36380/50462 | Loss: 2.5087 | LR: 9.26e-05 | 3195.3ms/step | 18.8 samples/s +Step 36390/50462 | Loss: 2.5106 | LR: 9.25e-05 | 3198.4ms/step | 18.8 samples/s +Step 36400/50462 | Loss: 2.5433 | LR: 9.24e-05 | 3199.1ms/step | 18.8 samples/s +Step 36410/50462 | Loss: 2.5153 | LR: 9.23e-05 | 3195.0ms/step | 18.8 samples/s +Step 36420/50462 | Loss: 2.5462 | LR: 9.21e-05 | 3198.2ms/step | 18.8 samples/s +Step 36430/50462 | Loss: 2.5567 | LR: 9.20e-05 | 3199.7ms/step | 18.8 samples/s +Step 36440/50462 | Loss: 2.5101 | LR: 9.19e-05 | 3197.2ms/step | 18.8 samples/s +Step 36450/50462 | Loss: 2.5101 | LR: 9.18e-05 | 3198.7ms/step | 18.8 samples/s +Step 36460/50462 | Loss: 2.5353 | LR: 9.17e-05 | 3198.0ms/step | 18.8 samples/s +Step 36470/50462 | Loss: 2.5321 | LR: 9.15e-05 | 3245.6ms/step | 18.5 samples/s +Step 36480/50462 | Loss: 2.5295 | LR: 9.14e-05 | 3198.6ms/step | 18.8 samples/s +Step 36490/50462 | Loss: 2.5536 | LR: 9.13e-05 | 3199.5ms/step | 18.8 samples/s +Step 36500/50462 | Loss: 2.5520 | LR: 9.12e-05 | 3250.8ms/step | 18.5 samples/s +Validation at step 36500: Loss = 2.5409, Perplexity = 12.69 +Step 36510/50462 | Loss: 2.5173 | LR: 9.10e-05 | 4871.1ms/step | 12.3 samples/s +Step 36520/50462 | Loss: 2.5141 | LR: 9.09e-05 | 3196.5ms/step | 18.8 samples/s +Step 36530/50462 | Loss: 2.5648 | LR: 9.08e-05 | 3251.6ms/step | 18.5 samples/s +Step 36540/50462 | Loss: 2.5272 | LR: 9.07e-05 | 3200.1ms/step | 18.7 samples/s +Step 36550/50462 | Loss: 2.5287 | LR: 9.06e-05 | 3199.8ms/step | 18.8 samples/s +Step 36560/50462 | Loss: 2.5215 | LR: 9.04e-05 | 3196.7ms/step | 18.8 samples/s +Step 36570/50462 | Loss: 2.5153 | LR: 9.03e-05 | 3199.8ms/step | 18.8 samples/s +Step 36580/50462 | Loss: 2.5254 | LR: 9.02e-05 | 3200.3ms/step | 18.7 samples/s +Step 36590/50462 | Loss: 2.5430 | LR: 9.01e-05 | 3199.1ms/step | 18.8 samples/s +Step 36600/50462 | Loss: 2.5049 | LR: 8.99e-05 | 3200.8ms/step | 18.7 samples/s +Step 36610/50462 | Loss: 2.5154 | LR: 8.98e-05 | 3197.7ms/step | 18.8 samples/s +Step 36620/50462 | Loss: 2.5188 | LR: 8.97e-05 | 3199.5ms/step | 18.8 samples/s +Step 36630/50462 | Loss: 2.5088 | LR: 8.96e-05 | 3194.8ms/step | 18.8 samples/s +Step 36640/50462 | Loss: 2.5483 | LR: 8.95e-05 | 3195.8ms/step | 18.8 samples/s +Step 36650/50462 | Loss: 2.5071 | LR: 8.93e-05 | 3196.8ms/step | 18.8 samples/s +Step 36660/50462 | Loss: 2.5436 | LR: 8.92e-05 | 3195.3ms/step | 18.8 samples/s +Step 36670/50462 | Loss: 2.5114 | LR: 8.91e-05 | 3193.6ms/step | 18.8 samples/s +Step 36680/50462 | Loss: 2.5168 | LR: 8.90e-05 | 3194.3ms/step | 18.8 samples/s +Step 36690/50462 | Loss: 2.5609 | LR: 8.89e-05 | 3200.2ms/step | 18.7 samples/s +Step 36700/50462 | Loss: 2.5257 | LR: 8.87e-05 | 3195.8ms/step | 18.8 samples/s +Step 36710/50462 | Loss: 2.5092 | LR: 8.86e-05 | 3194.0ms/step | 18.8 samples/s +Step 36720/50462 | Loss: 2.5168 | LR: 8.85e-05 | 3195.9ms/step | 18.8 samples/s +Step 36730/50462 | Loss: 2.5569 | LR: 8.84e-05 | 3195.7ms/step | 18.8 samples/s +Step 36740/50462 | Loss: 2.5325 | LR: 8.83e-05 | 3198.0ms/step | 18.8 samples/s +Step 36750/50462 | Loss: 2.5312 | LR: 8.81e-05 | 3199.7ms/step | 18.8 samples/s +Validation at step 36750: Loss = 2.5402, Perplexity = 12.68 +Step 36760/50462 | Loss: 2.5086 | LR: 8.80e-05 | 4870.7ms/step | 12.3 samples/s +Step 36770/50462 | Loss: 2.5281 | LR: 8.79e-05 | 3198.3ms/step | 18.8 samples/s +Step 36780/50462 | Loss: 2.5565 | LR: 8.78e-05 | 3195.7ms/step | 18.8 samples/s +Step 36790/50462 | Loss: 2.5120 | LR: 8.77e-05 | 3199.4ms/step | 18.8 samples/s +Step 36800/50462 | Loss: 2.5182 | LR: 8.75e-05 | 3198.7ms/step | 18.8 samples/s +Step 36810/50462 | Loss: 2.5412 | LR: 8.74e-05 | 3249.8ms/step | 18.5 samples/s +Step 36820/50462 | Loss: 2.5442 | LR: 8.73e-05 | 3200.1ms/step | 18.7 samples/s +Step 36830/50462 | Loss: 2.5442 | LR: 8.72e-05 | 3196.2ms/step | 18.8 samples/s +Step 36840/50462 | Loss: 2.5424 | LR: 8.71e-05 | 3248.8ms/step | 18.5 samples/s +Step 36850/50462 | Loss: 2.5400 | LR: 8.69e-05 | 3196.6ms/step | 18.8 samples/s +Step 36860/50462 | Loss: 2.5480 | LR: 8.68e-05 | 3199.4ms/step | 18.8 samples/s +Step 36870/50462 | Loss: 2.5156 | LR: 8.67e-05 | 3261.0ms/step | 18.4 samples/s +Step 36880/50462 | Loss: 2.5119 | LR: 8.66e-05 | 3198.8ms/step | 18.8 samples/s +Step 36890/50462 | Loss: 2.4987 | LR: 8.65e-05 | 3199.1ms/step | 18.8 samples/s +Step 36900/50462 | Loss: 2.5068 | LR: 8.63e-05 | 3197.3ms/step | 18.8 samples/s +Step 36910/50462 | Loss: 2.5501 | LR: 8.62e-05 | 3198.5ms/step | 18.8 samples/s +Step 36920/50462 | Loss: 2.5288 | LR: 8.61e-05 | 3197.7ms/step | 18.8 samples/s +Step 36930/50462 | Loss: 2.5267 | LR: 8.60e-05 | 3196.3ms/step | 18.8 samples/s +Step 36940/50462 | Loss: 2.5294 | LR: 8.59e-05 | 3197.8ms/step | 18.8 samples/s +Step 36950/50462 | Loss: 2.5315 | LR: 8.57e-05 | 3192.9ms/step | 18.8 samples/s +Step 36960/50462 | Loss: 2.5114 | LR: 8.56e-05 | 3193.7ms/step | 18.8 samples/s +Step 36970/50462 | Loss: 2.5144 | LR: 8.55e-05 | 3195.8ms/step | 18.8 samples/s +Step 36980/50462 | Loss: 2.4934 | LR: 8.54e-05 | 3192.7ms/step | 18.8 samples/s +Step 36990/50462 | Loss: 2.4937 | LR: 8.53e-05 | 3197.9ms/step | 18.8 samples/s +Step 37000/50462 | Loss: 2.5169 | LR: 8.51e-05 | 3199.5ms/step | 18.8 samples/s +Validation at step 37000: Loss = 2.5397, Perplexity = 12.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-37000 +Step 37010/50462 | Loss: 2.5558 | LR: 8.50e-05 | 5744.0ms/step | 10.4 samples/s +Step 37020/50462 | Loss: 2.5263 | LR: 8.49e-05 | 3187.5ms/step | 18.8 samples/s +Step 37030/50462 | Loss: 2.5146 | LR: 8.48e-05 | 3194.9ms/step | 18.8 samples/s +Step 37040/50462 | Loss: 2.4854 | LR: 8.47e-05 | 3195.6ms/step | 18.8 samples/s +Step 37050/50462 | Loss: 2.5505 | LR: 8.46e-05 | 3196.0ms/step | 18.8 samples/s +Step 37060/50462 | Loss: 2.5152 | LR: 8.44e-05 | 3198.0ms/step | 18.8 samples/s +Step 37070/50462 | Loss: 2.5137 | LR: 8.43e-05 | 3199.9ms/step | 18.8 samples/s +Step 37080/50462 | Loss: 2.5573 | LR: 8.42e-05 | 3195.9ms/step | 18.8 samples/s +Step 37090/50462 | Loss: 2.5252 | LR: 8.41e-05 | 3196.0ms/step | 18.8 samples/s +Step 37100/50462 | Loss: 2.5279 | LR: 8.40e-05 | 3193.6ms/step | 18.8 samples/s +Step 37110/50462 | Loss: 2.5241 | LR: 8.38e-05 | 3200.6ms/step | 18.7 samples/s +Step 37120/50462 | Loss: 2.5107 | LR: 8.37e-05 | 3195.9ms/step | 18.8 samples/s +Step 37130/50462 | Loss: 2.5381 | LR: 8.36e-05 | 3199.9ms/step | 18.8 samples/s +Step 37140/50462 | Loss: 2.5222 | LR: 8.35e-05 | 3252.2ms/step | 18.4 samples/s +Step 37150/50462 | Loss: 2.5566 | LR: 8.34e-05 | 3198.2ms/step | 18.8 samples/s +Step 37160/50462 | Loss: 2.5333 | LR: 8.33e-05 | 3195.7ms/step | 18.8 samples/s +Step 37170/50462 | Loss: 2.5412 | LR: 8.31e-05 | 3197.9ms/step | 18.8 samples/s +Step 37180/50462 | Loss: 2.5145 | LR: 8.30e-05 | 3198.0ms/step | 18.8 samples/s +Step 37190/50462 | Loss: 2.5296 | LR: 8.29e-05 | 3248.0ms/step | 18.5 samples/s +Step 37200/50462 | Loss: 2.5295 | LR: 8.28e-05 | 3197.4ms/step | 18.8 samples/s +Step 37210/50462 | Loss: 2.4812 | LR: 8.27e-05 | 3246.5ms/step | 18.5 samples/s +Step 37220/50462 | Loss: 2.5269 | LR: 8.25e-05 | 3190.2ms/step | 18.8 samples/s +Step 37230/50462 | Loss: 2.5152 | LR: 8.24e-05 | 3185.3ms/step | 18.8 samples/s +Step 37240/50462 | Loss: 2.5160 | LR: 8.23e-05 | 3186.7ms/step | 18.8 samples/s +Step 37250/50462 | Loss: 2.5215 | LR: 8.22e-05 | 3187.3ms/step | 18.8 samples/s +Validation at step 37250: Loss = 2.5391, Perplexity = 12.67 +Step 37260/50462 | Loss: 2.5286 | LR: 8.21e-05 | 4862.5ms/step | 12.3 samples/s +Step 37270/50462 | Loss: 2.5417 | LR: 8.20e-05 | 3193.2ms/step | 18.8 samples/s +Step 37280/50462 | Loss: 2.5403 | LR: 8.18e-05 | 3186.7ms/step | 18.8 samples/s +Step 37290/50462 | Loss: 2.5160 | LR: 8.17e-05 | 3190.2ms/step | 18.8 samples/s +Step 37300/50462 | Loss: 2.5297 | LR: 8.16e-05 | 3185.3ms/step | 18.8 samples/s +Step 37310/50462 | Loss: 2.5098 | LR: 8.15e-05 | 3188.5ms/step | 18.8 samples/s +Step 37320/50462 | Loss: 2.5316 | LR: 8.14e-05 | 3190.4ms/step | 18.8 samples/s +Step 37330/50462 | Loss: 2.5371 | LR: 8.13e-05 | 3187.2ms/step | 18.8 samples/s +Step 37340/50462 | Loss: 2.4957 | LR: 8.11e-05 | 3188.5ms/step | 18.8 samples/s +Step 37350/50462 | Loss: 2.5392 | LR: 8.10e-05 | 3188.4ms/step | 18.8 samples/s +Step 37360/50462 | Loss: 2.5442 | LR: 8.09e-05 | 3191.6ms/step | 18.8 samples/s +Step 37370/50462 | Loss: 2.5145 | LR: 8.08e-05 | 3186.7ms/step | 18.8 samples/s +Step 37380/50462 | Loss: 2.5297 | LR: 8.07e-05 | 3189.6ms/step | 18.8 samples/s +Step 37390/50462 | Loss: 2.5402 | LR: 8.06e-05 | 3189.5ms/step | 18.8 samples/s +Step 37400/50462 | Loss: 2.5315 | LR: 8.04e-05 | 3193.8ms/step | 18.8 samples/s +Step 37410/50462 | Loss: 2.5267 | LR: 8.03e-05 | 3196.7ms/step | 18.8 samples/s +Step 37420/50462 | Loss: 2.5326 | LR: 8.02e-05 | 3189.4ms/step | 18.8 samples/s +Step 37430/50462 | Loss: 2.5347 | LR: 8.01e-05 | 3189.5ms/step | 18.8 samples/s +Step 37440/50462 | Loss: 2.5208 | LR: 8.00e-05 | 3189.4ms/step | 18.8 samples/s +Step 37450/50462 | Loss: 2.5557 | LR: 7.99e-05 | 3196.6ms/step | 18.8 samples/s +Step 37460/50462 | Loss: 2.5217 | LR: 7.98e-05 | 3197.2ms/step | 18.8 samples/s +Step 37470/50462 | Loss: 2.5438 | LR: 7.96e-05 | 3192.6ms/step | 18.8 samples/s +Step 37480/50462 | Loss: 2.5198 | LR: 7.95e-05 | 3249.2ms/step | 18.5 samples/s +Step 37490/50462 | Loss: 2.5351 | LR: 7.94e-05 | 3200.3ms/step | 18.7 samples/s +Step 37500/50462 | Loss: 2.5181 | LR: 7.93e-05 | 3199.3ms/step | 18.8 samples/s +Validation at step 37500: Loss = 2.5388, Perplexity = 12.66 +Step 37510/50462 | Loss: 2.5302 | LR: 7.92e-05 | 4874.7ms/step | 12.3 samples/s +Step 37520/50462 | Loss: 2.5081 | LR: 7.91e-05 | 3199.8ms/step | 18.8 samples/s +Step 37530/50462 | Loss: 2.5284 | LR: 7.89e-05 | 3254.2ms/step | 18.4 samples/s +Step 37540/50462 | Loss: 2.5250 | LR: 7.88e-05 | 3199.3ms/step | 18.8 samples/s +Step 37550/50462 | Loss: 2.5443 | LR: 7.87e-05 | 3250.8ms/step | 18.5 samples/s +Step 37560/50462 | Loss: 2.5191 | LR: 7.86e-05 | 3198.3ms/step | 18.8 samples/s +Step 37570/50462 | Loss: 2.5324 | LR: 7.85e-05 | 3197.5ms/step | 18.8 samples/s +Step 37580/50462 | Loss: 2.5210 | LR: 7.84e-05 | 3195.7ms/step | 18.8 samples/s +Step 37590/50462 | Loss: 2.5236 | LR: 7.83e-05 | 3188.2ms/step | 18.8 samples/s +Step 37600/50462 | Loss: 2.5278 | LR: 7.81e-05 | 3192.1ms/step | 18.8 samples/s +Step 37610/50462 | Loss: 2.5347 | LR: 7.80e-05 | 3188.1ms/step | 18.8 samples/s +Step 37620/50462 | Loss: 2.5389 | LR: 7.79e-05 | 3191.9ms/step | 18.8 samples/s +Step 37630/50462 | Loss: 2.5148 | LR: 7.78e-05 | 3191.6ms/step | 18.8 samples/s +Step 37640/50462 | Loss: 2.5242 | LR: 7.77e-05 | 3198.1ms/step | 18.8 samples/s +Step 37650/50462 | Loss: 2.5247 | LR: 7.76e-05 | 3198.4ms/step | 18.8 samples/s +Step 37660/50462 | Loss: 2.5309 | LR: 7.75e-05 | 3198.0ms/step | 18.8 samples/s +Step 37670/50462 | Loss: 2.4982 | LR: 7.73e-05 | 3198.0ms/step | 18.8 samples/s +Step 37680/50462 | Loss: 2.5273 | LR: 7.72e-05 | 3197.6ms/step | 18.8 samples/s +Step 37690/50462 | Loss: 2.5447 | LR: 7.71e-05 | 3198.5ms/step | 18.8 samples/s +Step 37700/50462 | Loss: 2.5157 | LR: 7.70e-05 | 3200.0ms/step | 18.8 samples/s +Step 37710/50462 | Loss: 2.5523 | LR: 7.69e-05 | 3195.8ms/step | 18.8 samples/s +Step 37720/50462 | Loss: 2.4881 | LR: 7.68e-05 | 3196.0ms/step | 18.8 samples/s +Step 37730/50462 | Loss: 2.5371 | LR: 7.67e-05 | 3194.1ms/step | 18.8 samples/s +Step 37740/50462 | Loss: 2.5477 | LR: 7.65e-05 | 3198.1ms/step | 18.8 samples/s +Step 37750/50462 | Loss: 2.5030 | LR: 7.64e-05 | 3197.4ms/step | 18.8 samples/s +Validation at step 37750: Loss = 2.5385, Perplexity = 12.66 +Step 37760/50462 | Loss: 2.5308 | LR: 7.63e-05 | 4869.8ms/step | 12.3 samples/s +Step 37770/50462 | Loss: 2.5318 | LR: 7.62e-05 | 3184.6ms/step | 18.8 samples/s +Step 37780/50462 | Loss: 2.5061 | LR: 7.61e-05 | 3184.2ms/step | 18.8 samples/s +Step 37790/50462 | Loss: 2.4887 | LR: 7.60e-05 | 3191.9ms/step | 18.8 samples/s +Step 37800/50462 | Loss: 2.5070 | LR: 7.59e-05 | 3199.6ms/step | 18.8 samples/s +Step 37810/50462 | Loss: 2.5199 | LR: 7.57e-05 | 3196.0ms/step | 18.8 samples/s +Step 37820/50462 | Loss: 2.5364 | LR: 7.56e-05 | 3251.8ms/step | 18.5 samples/s +Step 37830/50462 | Loss: 2.5373 | LR: 7.55e-05 | 3198.7ms/step | 18.8 samples/s +Step 37840/50462 | Loss: 2.5444 | LR: 7.54e-05 | 3200.0ms/step | 18.7 samples/s +Step 37850/50462 | Loss: 2.5434 | LR: 7.53e-05 | 3199.9ms/step | 18.8 samples/s +Step 37860/50462 | Loss: 2.5460 | LR: 7.52e-05 | 3198.0ms/step | 18.8 samples/s +Step 37870/50462 | Loss: 2.5217 | LR: 7.51e-05 | 3247.9ms/step | 18.5 samples/s +Step 37880/50462 | Loss: 2.5530 | LR: 7.50e-05 | 3197.3ms/step | 18.8 samples/s +Step 37890/50462 | Loss: 2.5322 | LR: 7.48e-05 | 3261.0ms/step | 18.4 samples/s +Step 37900/50462 | Loss: 2.5026 | LR: 7.47e-05 | 3199.2ms/step | 18.8 samples/s +Step 37910/50462 | Loss: 2.5296 | LR: 7.46e-05 | 3196.5ms/step | 18.8 samples/s +Step 37920/50462 | Loss: 2.5018 | LR: 7.45e-05 | 3199.8ms/step | 18.8 samples/s +Step 37930/50462 | Loss: 2.5116 | LR: 7.44e-05 | 3196.3ms/step | 18.8 samples/s +Step 37940/50462 | Loss: 2.5242 | LR: 7.43e-05 | 3195.5ms/step | 18.8 samples/s +Step 37950/50462 | Loss: 2.5371 | LR: 7.42e-05 | 3197.4ms/step | 18.8 samples/s +Step 37960/50462 | Loss: 2.5390 | LR: 7.41e-05 | 3198.9ms/step | 18.8 samples/s +Step 37970/50462 | Loss: 2.5416 | LR: 7.39e-05 | 3198.1ms/step | 18.8 samples/s +Step 37980/50462 | Loss: 2.5114 | LR: 7.38e-05 | 3199.2ms/step | 18.8 samples/s +Step 37990/50462 | Loss: 2.4998 | LR: 7.37e-05 | 3198.2ms/step | 18.8 samples/s +Step 38000/50462 | Loss: 2.5238 | LR: 7.36e-05 | 3197.8ms/step | 18.8 samples/s +Validation at step 38000: Loss = 2.5380, Perplexity = 12.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-38000 +Step 38010/50462 | Loss: 2.5274 | LR: 7.35e-05 | 5776.3ms/step | 10.4 samples/s +Step 38020/50462 | Loss: 2.5164 | LR: 7.34e-05 | 3182.2ms/step | 18.9 samples/s +Step 38030/50462 | Loss: 2.5448 | LR: 7.33e-05 | 3192.1ms/step | 18.8 samples/s +Step 38040/50462 | Loss: 2.5118 | LR: 7.32e-05 | 3198.0ms/step | 18.8 samples/s +Step 38050/50462 | Loss: 2.5288 | LR: 7.30e-05 | 3191.0ms/step | 18.8 samples/s +Step 38060/50462 | Loss: 2.5263 | LR: 7.29e-05 | 3194.7ms/step | 18.8 samples/s +Step 38070/50462 | Loss: 2.5669 | LR: 7.28e-05 | 3193.9ms/step | 18.8 samples/s +Step 38080/50462 | Loss: 2.5250 | LR: 7.27e-05 | 3198.7ms/step | 18.8 samples/s +Step 38090/50462 | Loss: 2.5247 | LR: 7.26e-05 | 3193.8ms/step | 18.8 samples/s +Step 38100/50462 | Loss: 2.5469 | LR: 7.25e-05 | 3199.0ms/step | 18.8 samples/s +Step 38110/50462 | Loss: 2.4832 | LR: 7.24e-05 | 3199.2ms/step | 18.8 samples/s +Step 38120/50462 | Loss: 2.5329 | LR: 7.23e-05 | 3195.9ms/step | 18.8 samples/s +Step 38130/50462 | Loss: 2.5309 | LR: 7.22e-05 | 3200.0ms/step | 18.8 samples/s +Step 38140/50462 | Loss: 2.5341 | LR: 7.20e-05 | 3199.2ms/step | 18.8 samples/s +Step 38150/50462 | Loss: 2.5540 | LR: 7.19e-05 | 3248.7ms/step | 18.5 samples/s +Step 38160/50462 | Loss: 2.5107 | LR: 7.18e-05 | 3195.7ms/step | 18.8 samples/s +Step 38170/50462 | Loss: 2.5399 | LR: 7.17e-05 | 3199.8ms/step | 18.8 samples/s +Step 38180/50462 | Loss: 2.5356 | LR: 7.16e-05 | 3197.2ms/step | 18.8 samples/s +Step 38190/50462 | Loss: 2.5223 | LR: 7.15e-05 | 3194.7ms/step | 18.8 samples/s +Step 38200/50462 | Loss: 2.5584 | LR: 7.14e-05 | 3196.8ms/step | 18.8 samples/s +Step 38210/50462 | Loss: 2.5476 | LR: 7.13e-05 | 3251.7ms/step | 18.5 samples/s +Step 38220/50462 | Loss: 2.5168 | LR: 7.12e-05 | 3200.5ms/step | 18.7 samples/s +Step 38230/50462 | Loss: 2.5325 | LR: 7.10e-05 | 3264.4ms/step | 18.4 samples/s +Step 38240/50462 | Loss: 2.5304 | LR: 7.09e-05 | 3199.5ms/step | 18.8 samples/s +Step 38250/50462 | Loss: 2.5352 | LR: 7.08e-05 | 3200.1ms/step | 18.7 samples/s +Validation at step 38250: Loss = 2.5376, Perplexity = 12.65 +Step 38260/50462 | Loss: 2.5168 | LR: 7.07e-05 | 4871.6ms/step | 12.3 samples/s +Step 38270/50462 | Loss: 2.5549 | LR: 7.06e-05 | 3198.3ms/step | 18.8 samples/s +Step 38280/50462 | Loss: 2.4977 | LR: 7.05e-05 | 3197.2ms/step | 18.8 samples/s +Step 38290/50462 | Loss: 2.5187 | LR: 7.04e-05 | 3196.2ms/step | 18.8 samples/s +Step 38300/50462 | Loss: 2.5270 | LR: 7.03e-05 | 3194.8ms/step | 18.8 samples/s +Step 38310/50462 | Loss: 2.5537 | LR: 7.02e-05 | 3197.7ms/step | 18.8 samples/s +Step 38320/50462 | Loss: 2.5599 | LR: 7.01e-05 | 3200.4ms/step | 18.7 samples/s +Step 38330/50462 | Loss: 2.5183 | LR: 6.99e-05 | 3195.9ms/step | 18.8 samples/s +Step 38340/50462 | Loss: 2.5204 | LR: 6.98e-05 | 3199.8ms/step | 18.8 samples/s +Step 38350/50462 | Loss: 2.5397 | LR: 6.97e-05 | 3200.0ms/step | 18.7 samples/s +Step 38360/50462 | Loss: 2.5373 | LR: 6.96e-05 | 3200.0ms/step | 18.7 samples/s +Step 38370/50462 | Loss: 2.5347 | LR: 6.95e-05 | 3195.9ms/step | 18.8 samples/s +Step 38380/50462 | Loss: 2.5149 | LR: 6.94e-05 | 3197.8ms/step | 18.8 samples/s +Step 38390/50462 | Loss: 2.5318 | LR: 6.93e-05 | 3195.5ms/step | 18.8 samples/s +Step 38400/50462 | Loss: 2.5234 | LR: 6.92e-05 | 3193.2ms/step | 18.8 samples/s +Step 38410/50462 | Loss: 2.5266 | LR: 6.91e-05 | 3194.8ms/step | 18.8 samples/s +Step 38420/50462 | Loss: 2.5419 | LR: 6.90e-05 | 3196.2ms/step | 18.8 samples/s +Step 38430/50462 | Loss: 2.5269 | LR: 6.89e-05 | 3197.8ms/step | 18.8 samples/s +Step 38440/50462 | Loss: 2.5151 | LR: 6.87e-05 | 3200.6ms/step | 18.7 samples/s +Step 38450/50462 | Loss: 2.5368 | LR: 6.86e-05 | 3199.8ms/step | 18.8 samples/s +Step 38460/50462 | Loss: 2.5246 | LR: 6.85e-05 | 3198.1ms/step | 18.8 samples/s +Step 38470/50462 | Loss: 2.5243 | LR: 6.84e-05 | 3196.4ms/step | 18.8 samples/s +Step 38480/50462 | Loss: 2.5136 | LR: 6.83e-05 | 3195.2ms/step | 18.8 samples/s +Step 38490/50462 | Loss: 2.5506 | LR: 6.82e-05 | 3248.1ms/step | 18.5 samples/s +Step 38500/50462 | Loss: 2.5288 | LR: 6.81e-05 | 3184.5ms/step | 18.8 samples/s +Validation at step 38500: Loss = 2.5373, Perplexity = 12.65 +Step 38510/50462 | Loss: 2.5261 | LR: 6.80e-05 | 4869.4ms/step | 12.3 samples/s +Step 38520/50462 | Loss: 2.5141 | LR: 6.79e-05 | 3191.7ms/step | 18.8 samples/s +Step 38530/50462 | Loss: 2.5188 | LR: 6.78e-05 | 3192.2ms/step | 18.8 samples/s +Step 38540/50462 | Loss: 2.4983 | LR: 6.77e-05 | 3192.9ms/step | 18.8 samples/s +Step 38550/50462 | Loss: 2.5351 | LR: 6.76e-05 | 3250.4ms/step | 18.5 samples/s +Step 38560/50462 | Loss: 2.5087 | LR: 6.74e-05 | 3199.0ms/step | 18.8 samples/s +Step 38570/50462 | Loss: 2.5119 | LR: 6.73e-05 | 3256.9ms/step | 18.4 samples/s +Step 38580/50462 | Loss: 2.5220 | LR: 6.72e-05 | 3187.1ms/step | 18.8 samples/s +Step 38590/50462 | Loss: 2.5241 | LR: 6.71e-05 | 3185.9ms/step | 18.8 samples/s +Step 38600/50462 | Loss: 2.5290 | LR: 6.70e-05 | 3185.5ms/step | 18.8 samples/s +Step 38610/50462 | Loss: 2.5062 | LR: 6.69e-05 | 3188.3ms/step | 18.8 samples/s +Step 38620/50462 | Loss: 2.5127 | LR: 6.68e-05 | 3188.5ms/step | 18.8 samples/s +Step 38630/50462 | Loss: 2.5370 | LR: 6.67e-05 | 3187.9ms/step | 18.8 samples/s +Step 38640/50462 | Loss: 2.5137 | LR: 6.66e-05 | 3186.1ms/step | 18.8 samples/s +Step 38650/50462 | Loss: 2.5418 | LR: 6.65e-05 | 3186.3ms/step | 18.8 samples/s +Step 38660/50462 | Loss: 2.5101 | LR: 6.64e-05 | 3187.6ms/step | 18.8 samples/s +Step 38670/50462 | Loss: 2.5176 | LR: 6.63e-05 | 3188.5ms/step | 18.8 samples/s +Step 38680/50462 | Loss: 2.5093 | LR: 6.62e-05 | 3189.2ms/step | 18.8 samples/s +Step 38690/50462 | Loss: 2.4867 | LR: 6.60e-05 | 3194.6ms/step | 18.8 samples/s +Step 38700/50462 | Loss: 2.5394 | LR: 6.59e-05 | 3189.7ms/step | 18.8 samples/s +Step 38710/50462 | Loss: 2.5131 | LR: 6.58e-05 | 3190.1ms/step | 18.8 samples/s +Step 38720/50462 | Loss: 2.5138 | LR: 6.57e-05 | 3190.0ms/step | 18.8 samples/s +Step 38730/50462 | Loss: 2.5565 | LR: 6.56e-05 | 3191.4ms/step | 18.8 samples/s +Step 38740/50462 | Loss: 2.5240 | LR: 6.55e-05 | 3186.6ms/step | 18.8 samples/s +Step 38750/50462 | Loss: 2.5272 | LR: 6.54e-05 | 3185.5ms/step | 18.8 samples/s +Validation at step 38750: Loss = 2.5370, Perplexity = 12.64 +Step 38760/50462 | Loss: 2.5291 | LR: 6.53e-05 | 4864.3ms/step | 12.3 samples/s +Step 38770/50462 | Loss: 2.5446 | LR: 6.52e-05 | 3191.8ms/step | 18.8 samples/s +Step 38780/50462 | Loss: 2.5159 | LR: 6.51e-05 | 3188.7ms/step | 18.8 samples/s +Step 38790/50462 | Loss: 2.5674 | LR: 6.50e-05 | 3188.9ms/step | 18.8 samples/s +Step 38800/50462 | Loss: 2.5324 | LR: 6.49e-05 | 3192.1ms/step | 18.8 samples/s +Step 38810/50462 | Loss: 2.5162 | LR: 6.48e-05 | 3188.3ms/step | 18.8 samples/s +Step 38820/50462 | Loss: 2.5212 | LR: 6.47e-05 | 3187.9ms/step | 18.8 samples/s +Step 38830/50462 | Loss: 2.5294 | LR: 6.46e-05 | 3237.2ms/step | 18.5 samples/s +Step 38840/50462 | Loss: 2.5396 | LR: 6.45e-05 | 3188.8ms/step | 18.8 samples/s +Step 38850/50462 | Loss: 2.5496 | LR: 6.43e-05 | 3192.7ms/step | 18.8 samples/s +Step 38860/50462 | Loss: 2.5121 | LR: 6.42e-05 | 3197.8ms/step | 18.8 samples/s +Step 38870/50462 | Loss: 2.5282 | LR: 6.41e-05 | 3199.5ms/step | 18.8 samples/s +Step 38880/50462 | Loss: 2.5233 | LR: 6.40e-05 | 3198.4ms/step | 18.8 samples/s +Step 38890/50462 | Loss: 2.5337 | LR: 6.39e-05 | 3242.9ms/step | 18.5 samples/s +Step 38900/50462 | Loss: 2.5301 | LR: 6.38e-05 | 3186.6ms/step | 18.8 samples/s +Step 38910/50462 | Loss: 2.5330 | LR: 6.37e-05 | 3238.3ms/step | 18.5 samples/s +Step 38920/50462 | Loss: 2.5222 | LR: 6.36e-05 | 3191.7ms/step | 18.8 samples/s +Step 38930/50462 | Loss: 2.5381 | LR: 6.35e-05 | 3196.6ms/step | 18.8 samples/s +Step 38940/50462 | Loss: 2.5304 | LR: 6.34e-05 | 3196.2ms/step | 18.8 samples/s +Step 38950/50462 | Loss: 2.5386 | LR: 6.33e-05 | 3195.9ms/step | 18.8 samples/s +Step 38960/50462 | Loss: 2.5206 | LR: 6.32e-05 | 3197.3ms/step | 18.8 samples/s +Step 38970/50462 | Loss: 2.5154 | LR: 6.31e-05 | 3199.1ms/step | 18.8 samples/s +Step 38980/50462 | Loss: 2.5241 | LR: 6.30e-05 | 3193.7ms/step | 18.8 samples/s +Step 38990/50462 | Loss: 2.4997 | LR: 6.29e-05 | 3198.7ms/step | 18.8 samples/s +Step 39000/50462 | Loss: 2.5312 | LR: 6.28e-05 | 3195.7ms/step | 18.8 samples/s +Validation at step 39000: Loss = 2.5367, Perplexity = 12.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-39000 +Step 39010/50462 | Loss: 2.5480 | LR: 6.27e-05 | 5770.0ms/step | 10.4 samples/s +Step 39020/50462 | Loss: 2.5017 | LR: 6.26e-05 | 3184.0ms/step | 18.8 samples/s +Step 39030/50462 | Loss: 2.5316 | LR: 6.25e-05 | 3188.4ms/step | 18.8 samples/s +Step 39040/50462 | Loss: 2.5207 | LR: 6.23e-05 | 3195.6ms/step | 18.8 samples/s +Step 39050/50462 | Loss: 2.4975 | LR: 6.22e-05 | 3193.9ms/step | 18.8 samples/s +Step 39060/50462 | Loss: 2.5109 | LR: 6.21e-05 | 3193.9ms/step | 18.8 samples/s +Step 39070/50462 | Loss: 2.5328 | LR: 6.20e-05 | 3194.4ms/step | 18.8 samples/s +Step 39080/50462 | Loss: 2.5482 | LR: 6.19e-05 | 3194.9ms/step | 18.8 samples/s +Step 39090/50462 | Loss: 2.5049 | LR: 6.18e-05 | 3193.1ms/step | 18.8 samples/s +Step 39100/50462 | Loss: 2.5295 | LR: 6.17e-05 | 3195.0ms/step | 18.8 samples/s +Step 39110/50462 | Loss: 2.5022 | LR: 6.16e-05 | 3195.4ms/step | 18.8 samples/s +Step 39120/50462 | Loss: 2.5094 | LR: 6.15e-05 | 3198.1ms/step | 18.8 samples/s +Step 39130/50462 | Loss: 2.5454 | LR: 6.14e-05 | 3198.3ms/step | 18.8 samples/s +Step 39140/50462 | Loss: 2.5187 | LR: 6.13e-05 | 3196.6ms/step | 18.8 samples/s +Step 39150/50462 | Loss: 2.5228 | LR: 6.12e-05 | 3195.6ms/step | 18.8 samples/s +Step 39160/50462 | Loss: 2.5116 | LR: 6.11e-05 | 3200.0ms/step | 18.7 samples/s +Step 39170/50462 | Loss: 2.5379 | LR: 6.10e-05 | 3251.6ms/step | 18.5 samples/s +Step 39180/50462 | Loss: 2.5495 | LR: 6.09e-05 | 3198.1ms/step | 18.8 samples/s +Step 39190/50462 | Loss: 2.5318 | LR: 6.08e-05 | 3198.3ms/step | 18.8 samples/s +Step 39200/50462 | Loss: 2.5005 | LR: 6.07e-05 | 3197.6ms/step | 18.8 samples/s +Step 39210/50462 | Loss: 2.5246 | LR: 6.06e-05 | 3197.2ms/step | 18.8 samples/s +Step 39220/50462 | Loss: 2.4979 | LR: 6.05e-05 | 3197.4ms/step | 18.8 samples/s +Step 39230/50462 | Loss: 2.5429 | LR: 6.04e-05 | 3248.2ms/step | 18.5 samples/s +Step 39240/50462 | Loss: 2.5218 | LR: 6.03e-05 | 3195.4ms/step | 18.8 samples/s +Step 39250/50462 | Loss: 2.5408 | LR: 6.02e-05 | 3247.9ms/step | 18.5 samples/s +Validation at step 39250: Loss = 2.5365, Perplexity = 12.63 +Step 39260/50462 | Loss: 2.5257 | LR: 6.01e-05 | 4870.4ms/step | 12.3 samples/s +Step 39270/50462 | Loss: 2.5266 | LR: 6.00e-05 | 3197.9ms/step | 18.8 samples/s +Step 39280/50462 | Loss: 2.5401 | LR: 5.99e-05 | 3200.1ms/step | 18.7 samples/s +Step 39290/50462 | Loss: 2.5204 | LR: 5.98e-05 | 3196.2ms/step | 18.8 samples/s +Step 39300/50462 | Loss: 2.5389 | LR: 5.97e-05 | 3197.9ms/step | 18.8 samples/s +Step 39310/50462 | Loss: 2.5258 | LR: 5.96e-05 | 3198.2ms/step | 18.8 samples/s +Step 39320/50462 | Loss: 2.4808 | LR: 5.95e-05 | 3199.7ms/step | 18.8 samples/s +Step 39330/50462 | Loss: 2.5260 | LR: 5.94e-05 | 3192.9ms/step | 18.8 samples/s +Step 39340/50462 | Loss: 2.5312 | LR: 5.93e-05 | 3194.5ms/step | 18.8 samples/s +Step 39350/50462 | Loss: 2.5248 | LR: 5.91e-05 | 3196.8ms/step | 18.8 samples/s +Step 39360/50462 | Loss: 2.5001 | LR: 5.90e-05 | 3194.6ms/step | 18.8 samples/s +Step 39370/50462 | Loss: 2.5277 | LR: 5.89e-05 | 3194.1ms/step | 18.8 samples/s +Step 39380/50462 | Loss: 2.5160 | LR: 5.88e-05 | 3195.2ms/step | 18.8 samples/s +Step 39390/50462 | Loss: 2.5303 | LR: 5.87e-05 | 3196.2ms/step | 18.8 samples/s +Step 39400/50462 | Loss: 2.5384 | LR: 5.86e-05 | 3197.9ms/step | 18.8 samples/s +Step 39410/50462 | Loss: 2.4880 | LR: 5.85e-05 | 3197.4ms/step | 18.8 samples/s +Step 39420/50462 | Loss: 2.5059 | LR: 5.84e-05 | 3196.3ms/step | 18.8 samples/s +Step 39430/50462 | Loss: 2.5041 | LR: 5.83e-05 | 3197.1ms/step | 18.8 samples/s +Step 39440/50462 | Loss: 2.5246 | LR: 5.82e-05 | 3198.6ms/step | 18.8 samples/s +Step 39450/50462 | Loss: 2.5131 | LR: 5.81e-05 | 3200.5ms/step | 18.7 samples/s +Step 39460/50462 | Loss: 2.5262 | LR: 5.80e-05 | 3194.4ms/step | 18.8 samples/s +Step 39470/50462 | Loss: 2.5612 | LR: 5.79e-05 | 3193.6ms/step | 18.8 samples/s +Step 39480/50462 | Loss: 2.5464 | LR: 5.78e-05 | 3197.9ms/step | 18.8 samples/s +Step 39490/50462 | Loss: 2.5169 | LR: 5.77e-05 | 3197.9ms/step | 18.8 samples/s +Step 39500/50462 | Loss: 2.5312 | LR: 5.76e-05 | 3196.1ms/step | 18.8 samples/s +Validation at step 39500: Loss = 2.5361, Perplexity = 12.63 +Step 39510/50462 | Loss: 2.5268 | LR: 5.75e-05 | 4929.8ms/step | 12.2 samples/s +Step 39520/50462 | Loss: 2.5389 | LR: 5.74e-05 | 3200.2ms/step | 18.7 samples/s +Step 39530/50462 | Loss: 2.5284 | LR: 5.73e-05 | 3198.0ms/step | 18.8 samples/s +Step 39540/50462 | Loss: 2.5278 | LR: 5.72e-05 | 3200.2ms/step | 18.7 samples/s +Step 39550/50462 | Loss: 2.5304 | LR: 5.71e-05 | 3200.1ms/step | 18.7 samples/s +Step 39560/50462 | Loss: 2.5200 | LR: 5.70e-05 | 3196.0ms/step | 18.8 samples/s +Step 39570/50462 | Loss: 2.5299 | LR: 5.69e-05 | 3245.3ms/step | 18.5 samples/s +Step 39580/50462 | Loss: 2.5119 | LR: 5.68e-05 | 3197.6ms/step | 18.8 samples/s +Step 39590/50462 | Loss: 2.5360 | LR: 5.67e-05 | 3259.2ms/step | 18.4 samples/s +Step 39600/50462 | Loss: 2.5197 | LR: 5.66e-05 | 3188.4ms/step | 18.8 samples/s +Step 39610/50462 | Loss: 2.5120 | LR: 5.65e-05 | 3190.4ms/step | 18.8 samples/s +Step 39620/50462 | Loss: 2.5074 | LR: 5.64e-05 | 3187.9ms/step | 18.8 samples/s +Step 39630/50462 | Loss: 2.4981 | LR: 5.63e-05 | 3189.2ms/step | 18.8 samples/s +Step 39640/50462 | Loss: 2.5530 | LR: 5.62e-05 | 3197.6ms/step | 18.8 samples/s +Step 39650/50462 | Loss: 2.5269 | LR: 5.61e-05 | 3198.0ms/step | 18.8 samples/s +Step 39660/50462 | Loss: 2.5271 | LR: 5.60e-05 | 3190.4ms/step | 18.8 samples/s +Step 39670/50462 | Loss: 2.5261 | LR: 5.59e-05 | 3199.4ms/step | 18.8 samples/s +Step 39680/50462 | Loss: 2.5354 | LR: 5.58e-05 | 3192.2ms/step | 18.8 samples/s +Step 39690/50462 | Loss: 2.5207 | LR: 5.57e-05 | 3198.5ms/step | 18.8 samples/s +Step 39700/50462 | Loss: 2.5181 | LR: 5.56e-05 | 3189.3ms/step | 18.8 samples/s +Step 39710/50462 | Loss: 2.5078 | LR: 5.55e-05 | 3198.6ms/step | 18.8 samples/s +Step 39720/50462 | Loss: 2.4979 | LR: 5.54e-05 | 3199.7ms/step | 18.8 samples/s +Step 39730/50462 | Loss: 2.5012 | LR: 5.53e-05 | 3194.3ms/step | 18.8 samples/s +Step 39740/50462 | Loss: 2.5221 | LR: 5.52e-05 | 3195.5ms/step | 18.8 samples/s +Step 39750/50462 | Loss: 2.5318 | LR: 5.51e-05 | 3191.9ms/step | 18.8 samples/s +Validation at step 39750: Loss = 2.5359, Perplexity = 12.63 +Step 39760/50462 | Loss: 2.5195 | LR: 5.50e-05 | 4870.4ms/step | 12.3 samples/s +Step 39770/50462 | Loss: 2.5428 | LR: 5.49e-05 | 3196.3ms/step | 18.8 samples/s +Step 39780/50462 | Loss: 2.5429 | LR: 5.48e-05 | 3197.3ms/step | 18.8 samples/s +Step 39790/50462 | Loss: 2.5196 | LR: 5.47e-05 | 3196.0ms/step | 18.8 samples/s +Step 39800/50462 | Loss: 2.5138 | LR: 5.46e-05 | 3196.4ms/step | 18.8 samples/s +Step 39810/50462 | Loss: 2.5273 | LR: 5.45e-05 | 3198.3ms/step | 18.8 samples/s +Step 39820/50462 | Loss: 2.5463 | LR: 5.44e-05 | 3197.4ms/step | 18.8 samples/s +Step 39830/50462 | Loss: 2.5194 | LR: 5.43e-05 | 3196.1ms/step | 18.8 samples/s +Step 39840/50462 | Loss: 2.5300 | LR: 5.42e-05 | 3198.0ms/step | 18.8 samples/s +Step 39850/50462 | Loss: 2.5359 | LR: 5.41e-05 | 3246.0ms/step | 18.5 samples/s +Step 39860/50462 | Loss: 2.4993 | LR: 5.40e-05 | 3187.7ms/step | 18.8 samples/s +Step 39870/50462 | Loss: 2.5252 | LR: 5.39e-05 | 3192.3ms/step | 18.8 samples/s +Step 39880/50462 | Loss: 2.5248 | LR: 5.38e-05 | 3190.6ms/step | 18.8 samples/s +Step 39890/50462 | Loss: 2.5353 | LR: 5.38e-05 | 3195.5ms/step | 18.8 samples/s +Step 39900/50462 | Loss: 2.5155 | LR: 5.37e-05 | 3196.0ms/step | 18.8 samples/s +Step 39910/50462 | Loss: 2.5228 | LR: 5.36e-05 | 3244.7ms/step | 18.5 samples/s +Step 39920/50462 | Loss: 2.5227 | LR: 5.35e-05 | 3188.9ms/step | 18.8 samples/s +Step 39930/50462 | Loss: 2.5264 | LR: 5.34e-05 | 3246.2ms/step | 18.5 samples/s +Step 39940/50462 | Loss: 2.5169 | LR: 5.33e-05 | 3195.8ms/step | 18.8 samples/s +Step 39950/50462 | Loss: 2.5093 | LR: 5.32e-05 | 3198.5ms/step | 18.8 samples/s +Step 39960/50462 | Loss: 2.5248 | LR: 5.31e-05 | 3199.9ms/step | 18.8 samples/s +Step 39970/50462 | Loss: 2.5491 | LR: 5.30e-05 | 3196.3ms/step | 18.8 samples/s +Step 39980/50462 | Loss: 2.5252 | LR: 5.29e-05 | 3193.1ms/step | 18.8 samples/s +Step 39990/50462 | Loss: 2.5590 | LR: 5.28e-05 | 3198.6ms/step | 18.8 samples/s +Step 40000/50462 | Loss: 2.4985 | LR: 5.27e-05 | 3197.9ms/step | 18.8 samples/s +Validation at step 40000: Loss = 2.5356, Perplexity = 12.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-40000 +Step 40010/50462 | Loss: 2.5418 | LR: 5.26e-05 | 5742.6ms/step | 10.4 samples/s +Step 40020/50462 | Loss: 2.5620 | LR: 5.25e-05 | 3177.1ms/step | 18.9 samples/s +Step 40030/50462 | Loss: 2.5078 | LR: 5.24e-05 | 3184.3ms/step | 18.8 samples/s +Step 40040/50462 | Loss: 2.5395 | LR: 5.23e-05 | 3188.4ms/step | 18.8 samples/s +Step 40050/50462 | Loss: 2.5270 | LR: 5.22e-05 | 3185.5ms/step | 18.8 samples/s +Step 40060/50462 | Loss: 2.5208 | LR: 5.21e-05 | 3186.0ms/step | 18.8 samples/s +Step 40070/50462 | Loss: 2.5318 | LR: 5.20e-05 | 3189.6ms/step | 18.8 samples/s +Step 40080/50462 | Loss: 2.5059 | LR: 5.19e-05 | 3188.1ms/step | 18.8 samples/s +Step 40090/50462 | Loss: 2.5401 | LR: 5.18e-05 | 3189.7ms/step | 18.8 samples/s +Step 40100/50462 | Loss: 2.5354 | LR: 5.17e-05 | 3187.8ms/step | 18.8 samples/s +Step 40110/50462 | Loss: 2.5243 | LR: 5.16e-05 | 3189.3ms/step | 18.8 samples/s +Step 40120/50462 | Loss: 2.5374 | LR: 5.15e-05 | 3188.5ms/step | 18.8 samples/s +Step 40130/50462 | Loss: 2.5187 | LR: 5.14e-05 | 3187.8ms/step | 18.8 samples/s +Step 40140/50462 | Loss: 2.5170 | LR: 5.13e-05 | 3187.4ms/step | 18.8 samples/s +Step 40150/50462 | Loss: 2.4945 | LR: 5.12e-05 | 3189.1ms/step | 18.8 samples/s +Step 40160/50462 | Loss: 2.4911 | LR: 5.11e-05 | 3190.8ms/step | 18.8 samples/s +Step 40170/50462 | Loss: 2.5310 | LR: 5.10e-05 | 3187.9ms/step | 18.8 samples/s +Step 40180/50462 | Loss: 2.5319 | LR: 5.09e-05 | 3189.0ms/step | 18.8 samples/s +Step 40190/50462 | Loss: 2.5282 | LR: 5.09e-05 | 3241.7ms/step | 18.5 samples/s +Step 40200/50462 | Loss: 2.5376 | LR: 5.08e-05 | 3188.2ms/step | 18.8 samples/s +Step 40210/50462 | Loss: 2.5423 | LR: 5.07e-05 | 3192.9ms/step | 18.8 samples/s +Step 40220/50462 | Loss: 2.5430 | LR: 5.06e-05 | 3191.6ms/step | 18.8 samples/s +Step 40230/50462 | Loss: 2.5339 | LR: 5.05e-05 | 3184.7ms/step | 18.8 samples/s +Step 40240/50462 | Loss: 2.5638 | LR: 5.04e-05 | 3188.8ms/step | 18.8 samples/s +Step 40250/50462 | Loss: 2.5242 | LR: 5.03e-05 | 3236.7ms/step | 18.5 samples/s +Validation at step 40250: Loss = 2.5355, Perplexity = 12.62 +Step 40260/50462 | Loss: 2.5290 | LR: 5.02e-05 | 4860.8ms/step | 12.3 samples/s +Step 40270/50462 | Loss: 2.5053 | LR: 5.01e-05 | 3257.2ms/step | 18.4 samples/s +Step 40280/50462 | Loss: 2.5296 | LR: 5.00e-05 | 3188.9ms/step | 18.8 samples/s +Step 40290/50462 | Loss: 2.5277 | LR: 4.99e-05 | 3189.7ms/step | 18.8 samples/s +Step 40300/50462 | Loss: 2.5229 | LR: 4.98e-05 | 3189.4ms/step | 18.8 samples/s +Step 40310/50462 | Loss: 2.5087 | LR: 4.97e-05 | 3190.2ms/step | 18.8 samples/s +Step 40320/50462 | Loss: 2.5274 | LR: 4.96e-05 | 3187.9ms/step | 18.8 samples/s +Step 40330/50462 | Loss: 2.5508 | LR: 4.95e-05 | 3184.6ms/step | 18.8 samples/s +Step 40340/50462 | Loss: 2.5096 | LR: 4.94e-05 | 3189.7ms/step | 18.8 samples/s +Step 40350/50462 | Loss: 2.5012 | LR: 4.93e-05 | 3184.3ms/step | 18.8 samples/s +Step 40360/50462 | Loss: 2.5299 | LR: 4.92e-05 | 3188.3ms/step | 18.8 samples/s +Step 40370/50462 | Loss: 2.5421 | LR: 4.91e-05 | 3189.6ms/step | 18.8 samples/s +Step 40380/50462 | Loss: 2.5289 | LR: 4.90e-05 | 3189.5ms/step | 18.8 samples/s +Step 40390/50462 | Loss: 2.5238 | LR: 4.90e-05 | 3185.8ms/step | 18.8 samples/s +Step 40400/50462 | Loss: 2.5360 | LR: 4.89e-05 | 3189.5ms/step | 18.8 samples/s +Step 40410/50462 | Loss: 2.5505 | LR: 4.88e-05 | 3189.3ms/step | 18.8 samples/s +Step 40420/50462 | Loss: 2.5488 | LR: 4.87e-05 | 3190.1ms/step | 18.8 samples/s +Step 40430/50462 | Loss: 2.5160 | LR: 4.86e-05 | 3187.9ms/step | 18.8 samples/s +Step 40440/50462 | Loss: 2.5381 | LR: 4.85e-05 | 3189.2ms/step | 18.8 samples/s +Step 40450/50462 | Loss: 2.5242 | LR: 4.84e-05 | 3188.7ms/step | 18.8 samples/s +Step 40460/50462 | Loss: 2.5569 | LR: 4.83e-05 | 3188.8ms/step | 18.8 samples/s +Step 40470/50462 | Loss: 2.5114 | LR: 4.82e-05 | 3190.3ms/step | 18.8 samples/s +Step 40480/50462 | Loss: 2.5218 | LR: 4.81e-05 | 3186.9ms/step | 18.8 samples/s +Step 40490/50462 | Loss: 2.5377 | LR: 4.80e-05 | 3183.7ms/step | 18.8 samples/s +Step 40500/50462 | Loss: 2.5277 | LR: 4.79e-05 | 3186.4ms/step | 18.8 samples/s +Validation at step 40500: Loss = 2.5353, Perplexity = 12.62 +Step 40510/50462 | Loss: 2.5266 | LR: 4.78e-05 | 4864.0ms/step | 12.3 samples/s +Step 40520/50462 | Loss: 2.5129 | LR: 4.77e-05 | 3250.5ms/step | 18.5 samples/s +Step 40530/50462 | Loss: 2.5160 | LR: 4.76e-05 | 3195.2ms/step | 18.8 samples/s +Step 40540/50462 | Loss: 2.4901 | LR: 4.76e-05 | 3196.0ms/step | 18.8 samples/s +Step 40550/50462 | Loss: 2.5445 | LR: 4.75e-05 | 3198.0ms/step | 18.8 samples/s +Step 40560/50462 | Loss: 2.5365 | LR: 4.74e-05 | 3198.1ms/step | 18.8 samples/s +Step 40570/50462 | Loss: 2.5103 | LR: 4.73e-05 | 3197.8ms/step | 18.8 samples/s +Step 40580/50462 | Loss: 2.5391 | LR: 4.72e-05 | 3196.4ms/step | 18.8 samples/s +Step 40590/50462 | Loss: 2.5220 | LR: 4.71e-05 | 3247.5ms/step | 18.5 samples/s +Step 40600/50462 | Loss: 2.5243 | LR: 4.70e-05 | 3185.2ms/step | 18.8 samples/s +Step 40610/50462 | Loss: 2.5421 | LR: 4.69e-05 | 3242.5ms/step | 18.5 samples/s +Step 40620/50462 | Loss: 2.4993 | LR: 4.68e-05 | 3177.8ms/step | 18.9 samples/s +Step 40630/50462 | Loss: 2.5205 | LR: 4.67e-05 | 3184.5ms/step | 18.8 samples/s +Step 40640/50462 | Loss: 2.5316 | LR: 4.66e-05 | 3182.5ms/step | 18.9 samples/s +Step 40650/50462 | Loss: 2.5314 | LR: 4.65e-05 | 3183.9ms/step | 18.8 samples/s +Step 40660/50462 | Loss: 2.5421 | LR: 4.64e-05 | 3183.6ms/step | 18.8 samples/s +Step 40670/50462 | Loss: 2.5199 | LR: 4.64e-05 | 3186.0ms/step | 18.8 samples/s +Step 40680/50462 | Loss: 2.5248 | LR: 4.63e-05 | 3185.8ms/step | 18.8 samples/s +Step 40690/50462 | Loss: 2.5207 | LR: 4.62e-05 | 3192.7ms/step | 18.8 samples/s +Step 40700/50462 | Loss: 2.4929 | LR: 4.61e-05 | 3192.4ms/step | 18.8 samples/s +Step 40710/50462 | Loss: 2.5329 | LR: 4.60e-05 | 3196.9ms/step | 18.8 samples/s +Step 40720/50462 | Loss: 2.4999 | LR: 4.59e-05 | 3198.5ms/step | 18.8 samples/s +Step 40730/50462 | Loss: 2.5105 | LR: 4.58e-05 | 3196.1ms/step | 18.8 samples/s +Step 40740/50462 | Loss: 2.5066 | LR: 4.57e-05 | 3195.9ms/step | 18.8 samples/s +Step 40750/50462 | Loss: 2.5169 | LR: 4.56e-05 | 3196.0ms/step | 18.8 samples/s +Validation at step 40750: Loss = 2.5351, Perplexity = 12.62 +Step 40760/50462 | Loss: 2.5420 | LR: 4.55e-05 | 4865.5ms/step | 12.3 samples/s +Step 40770/50462 | Loss: 2.5227 | LR: 4.54e-05 | 3194.5ms/step | 18.8 samples/s +Step 40780/50462 | Loss: 2.5378 | LR: 4.54e-05 | 3198.0ms/step | 18.8 samples/s +Step 40790/50462 | Loss: 2.5229 | LR: 4.53e-05 | 3193.4ms/step | 18.8 samples/s +Step 40800/50462 | Loss: 2.5314 | LR: 4.52e-05 | 3196.2ms/step | 18.8 samples/s +Step 40810/50462 | Loss: 2.4893 | LR: 4.51e-05 | 3193.8ms/step | 18.8 samples/s +Step 40820/50462 | Loss: 2.5629 | LR: 4.50e-05 | 3192.7ms/step | 18.8 samples/s +Step 40830/50462 | Loss: 2.4775 | LR: 4.49e-05 | 3200.0ms/step | 18.8 samples/s +Step 40840/50462 | Loss: 2.5017 | LR: 4.48e-05 | 3197.4ms/step | 18.8 samples/s +Step 40850/50462 | Loss: 2.5454 | LR: 4.47e-05 | 3198.6ms/step | 18.8 samples/s +Step 40860/50462 | Loss: 2.4907 | LR: 4.46e-05 | 3255.0ms/step | 18.4 samples/s +Step 40870/50462 | Loss: 2.5458 | LR: 4.45e-05 | 3192.3ms/step | 18.8 samples/s +Step 40880/50462 | Loss: 2.5530 | LR: 4.45e-05 | 3194.6ms/step | 18.8 samples/s +Step 40890/50462 | Loss: 2.4857 | LR: 4.44e-05 | 3193.9ms/step | 18.8 samples/s +Step 40900/50462 | Loss: 2.5107 | LR: 4.43e-05 | 3190.1ms/step | 18.8 samples/s +Step 40910/50462 | Loss: 2.5312 | LR: 4.42e-05 | 3191.5ms/step | 18.8 samples/s +Step 40920/50462 | Loss: 2.5399 | LR: 4.41e-05 | 3192.6ms/step | 18.8 samples/s +Step 40930/50462 | Loss: 2.5308 | LR: 4.40e-05 | 3247.8ms/step | 18.5 samples/s +Step 40940/50462 | Loss: 2.5481 | LR: 4.39e-05 | 3193.6ms/step | 18.8 samples/s +Step 40950/50462 | Loss: 2.5015 | LR: 4.38e-05 | 3247.6ms/step | 18.5 samples/s +Step 40960/50462 | Loss: 2.5323 | LR: 4.37e-05 | 3193.2ms/step | 18.8 samples/s +Step 40970/50462 | Loss: 2.5220 | LR: 4.36e-05 | 3195.7ms/step | 18.8 samples/s +Step 40980/50462 | Loss: 2.5216 | LR: 4.36e-05 | 3197.6ms/step | 18.8 samples/s +Step 40990/50462 | Loss: 2.4908 | LR: 4.35e-05 | 3198.4ms/step | 18.8 samples/s +Step 41000/50462 | Loss: 2.5362 | LR: 4.34e-05 | 3199.9ms/step | 18.8 samples/s +Validation at step 41000: Loss = 2.5349, Perplexity = 12.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-41000 +Step 41010/50462 | Loss: 2.5421 | LR: 4.33e-05 | 5758.5ms/step | 10.4 samples/s +Step 41020/50462 | Loss: 2.5372 | LR: 4.32e-05 | 3176.9ms/step | 18.9 samples/s +Step 41030/50462 | Loss: 2.5402 | LR: 4.31e-05 | 3190.5ms/step | 18.8 samples/s +Step 41040/50462 | Loss: 2.5029 | LR: 4.30e-05 | 3196.3ms/step | 18.8 samples/s +Step 41050/50462 | Loss: 2.5340 | LR: 4.29e-05 | 3197.4ms/step | 18.8 samples/s +Step 41060/50462 | Loss: 2.5400 | LR: 4.28e-05 | 3196.5ms/step | 18.8 samples/s +Step 41070/50462 | Loss: 2.5177 | LR: 4.28e-05 | 3194.0ms/step | 18.8 samples/s +Step 41080/50462 | Loss: 2.5117 | LR: 4.27e-05 | 3199.3ms/step | 18.8 samples/s +Step 41090/50462 | Loss: 2.4895 | LR: 4.26e-05 | 3194.6ms/step | 18.8 samples/s +Step 41100/50462 | Loss: 2.5239 | LR: 4.25e-05 | 3191.4ms/step | 18.8 samples/s +Step 41110/50462 | Loss: 2.5279 | LR: 4.24e-05 | 3196.7ms/step | 18.8 samples/s +Step 41120/50462 | Loss: 2.5187 | LR: 4.23e-05 | 3195.6ms/step | 18.8 samples/s +Step 41130/50462 | Loss: 2.5409 | LR: 4.22e-05 | 3196.5ms/step | 18.8 samples/s +Step 41140/50462 | Loss: 2.5144 | LR: 4.21e-05 | 3195.7ms/step | 18.8 samples/s +Step 41150/50462 | Loss: 2.5289 | LR: 4.21e-05 | 3192.9ms/step | 18.8 samples/s +Step 41160/50462 | Loss: 2.5259 | LR: 4.20e-05 | 3194.9ms/step | 18.8 samples/s +Step 41170/50462 | Loss: 2.5318 | LR: 4.19e-05 | 3196.4ms/step | 18.8 samples/s +Step 41180/50462 | Loss: 2.5164 | LR: 4.18e-05 | 3193.2ms/step | 18.8 samples/s +Step 41190/50462 | Loss: 2.4757 | LR: 4.17e-05 | 3194.6ms/step | 18.8 samples/s +Step 41200/50462 | Loss: 2.5121 | LR: 4.16e-05 | 3249.6ms/step | 18.5 samples/s +Step 41210/50462 | Loss: 2.4898 | LR: 4.15e-05 | 3184.2ms/step | 18.8 samples/s +Step 41220/50462 | Loss: 2.5052 | LR: 4.14e-05 | 3185.9ms/step | 18.8 samples/s +Step 41230/50462 | Loss: 2.5274 | LR: 4.14e-05 | 3194.3ms/step | 18.8 samples/s +Step 41240/50462 | Loss: 2.5175 | LR: 4.13e-05 | 3197.5ms/step | 18.8 samples/s +Step 41250/50462 | Loss: 2.5316 | LR: 4.12e-05 | 3198.1ms/step | 18.8 samples/s +Validation at step 41250: Loss = 2.5347, Perplexity = 12.61 +Step 41260/50462 | Loss: 2.5271 | LR: 4.11e-05 | 4870.2ms/step | 12.3 samples/s +Step 41270/50462 | Loss: 2.5016 | LR: 4.10e-05 | 3250.2ms/step | 18.5 samples/s +Step 41280/50462 | Loss: 2.4804 | LR: 4.09e-05 | 3196.2ms/step | 18.8 samples/s +Step 41290/50462 | Loss: 2.5422 | LR: 4.08e-05 | 3245.9ms/step | 18.5 samples/s +Step 41300/50462 | Loss: 2.5315 | LR: 4.07e-05 | 3187.3ms/step | 18.8 samples/s +Step 41310/50462 | Loss: 2.4936 | LR: 4.07e-05 | 3192.0ms/step | 18.8 samples/s +Step 41320/50462 | Loss: 2.5757 | LR: 4.06e-05 | 3188.0ms/step | 18.8 samples/s +Step 41330/50462 | Loss: 2.5112 | LR: 4.05e-05 | 3190.0ms/step | 18.8 samples/s +Step 41340/50462 | Loss: 2.5047 | LR: 4.04e-05 | 3192.8ms/step | 18.8 samples/s +Step 41350/50462 | Loss: 2.5257 | LR: 4.03e-05 | 3190.3ms/step | 18.8 samples/s +Step 41360/50462 | Loss: 2.5531 | LR: 4.02e-05 | 3185.8ms/step | 18.8 samples/s +Step 41370/50462 | Loss: 2.5242 | LR: 4.01e-05 | 3195.4ms/step | 18.8 samples/s +Step 41380/50462 | Loss: 2.5169 | LR: 4.01e-05 | 3198.0ms/step | 18.8 samples/s +Step 41390/50462 | Loss: 2.4927 | LR: 4.00e-05 | 3196.1ms/step | 18.8 samples/s +Step 41400/50462 | Loss: 2.5323 | LR: 3.99e-05 | 3195.6ms/step | 18.8 samples/s +Step 41410/50462 | Loss: 2.5394 | LR: 3.98e-05 | 3195.4ms/step | 18.8 samples/s +Step 41420/50462 | Loss: 2.5021 | LR: 3.97e-05 | 3196.9ms/step | 18.8 samples/s +Step 41430/50462 | Loss: 2.5347 | LR: 3.96e-05 | 3192.9ms/step | 18.8 samples/s +Step 41440/50462 | Loss: 2.5022 | LR: 3.95e-05 | 3195.4ms/step | 18.8 samples/s +Step 41450/50462 | Loss: 2.5224 | LR: 3.95e-05 | 3196.3ms/step | 18.8 samples/s +Step 41460/50462 | Loss: 2.5047 | LR: 3.94e-05 | 3195.5ms/step | 18.8 samples/s +Step 41470/50462 | Loss: 2.4838 | LR: 3.93e-05 | 3198.0ms/step | 18.8 samples/s +Step 41480/50462 | Loss: 2.5419 | LR: 3.92e-05 | 3200.0ms/step | 18.8 samples/s +Step 41490/50462 | Loss: 2.4965 | LR: 3.91e-05 | 3195.8ms/step | 18.8 samples/s +Step 41500/50462 | Loss: 2.5375 | LR: 3.90e-05 | 3183.9ms/step | 18.8 samples/s +Validation at step 41500: Loss = 2.5346, Perplexity = 12.61 +Step 41510/50462 | Loss: 2.5087 | LR: 3.90e-05 | 4870.5ms/step | 12.3 samples/s +Step 41520/50462 | Loss: 2.5226 | LR: 3.89e-05 | 3200.1ms/step | 18.7 samples/s +Step 41530/50462 | Loss: 2.5338 | LR: 3.88e-05 | 3199.9ms/step | 18.8 samples/s +Step 41540/50462 | Loss: 2.5033 | LR: 3.87e-05 | 3254.0ms/step | 18.4 samples/s +Step 41550/50462 | Loss: 2.5293 | LR: 3.86e-05 | 3198.2ms/step | 18.8 samples/s +Step 41560/50462 | Loss: 2.5405 | LR: 3.85e-05 | 3200.0ms/step | 18.7 samples/s +Step 41570/50462 | Loss: 2.5280 | LR: 3.84e-05 | 3199.3ms/step | 18.8 samples/s +Step 41580/50462 | Loss: 2.5331 | LR: 3.84e-05 | 3200.7ms/step | 18.7 samples/s +Step 41590/50462 | Loss: 2.5151 | LR: 3.83e-05 | 3199.4ms/step | 18.8 samples/s +Step 41600/50462 | Loss: 2.5300 | LR: 3.82e-05 | 3198.7ms/step | 18.8 samples/s +Step 41610/50462 | Loss: 2.5129 | LR: 3.81e-05 | 3254.0ms/step | 18.4 samples/s +Step 41620/50462 | Loss: 2.5178 | LR: 3.80e-05 | 3197.8ms/step | 18.8 samples/s +Step 41630/50462 | Loss: 2.4994 | LR: 3.79e-05 | 3243.8ms/step | 18.5 samples/s +Step 41640/50462 | Loss: 2.5160 | LR: 3.79e-05 | 3185.0ms/step | 18.8 samples/s +Step 41650/50462 | Loss: 2.5013 | LR: 3.78e-05 | 3181.8ms/step | 18.9 samples/s +Step 41660/50462 | Loss: 2.5306 | LR: 3.77e-05 | 3189.5ms/step | 18.8 samples/s +Step 41670/50462 | Loss: 2.5223 | LR: 3.76e-05 | 3199.9ms/step | 18.8 samples/s +Step 41680/50462 | Loss: 2.5076 | LR: 3.75e-05 | 3200.0ms/step | 18.7 samples/s +Step 41690/50462 | Loss: 2.5048 | LR: 3.74e-05 | 3198.1ms/step | 18.8 samples/s +Step 41700/50462 | Loss: 2.5165 | LR: 3.74e-05 | 3199.6ms/step | 18.8 samples/s +Step 41710/50462 | Loss: 2.5091 | LR: 3.73e-05 | 3197.7ms/step | 18.8 samples/s +Step 41720/50462 | Loss: 2.5053 | LR: 3.72e-05 | 3199.9ms/step | 18.8 samples/s +Step 41730/50462 | Loss: 2.5107 | LR: 3.71e-05 | 3198.5ms/step | 18.8 samples/s +Step 41740/50462 | Loss: 2.5035 | LR: 3.70e-05 | 3196.2ms/step | 18.8 samples/s +Step 41750/50462 | Loss: 2.5102 | LR: 3.69e-05 | 3199.9ms/step | 18.8 samples/s +Validation at step 41750: Loss = 2.5344, Perplexity = 12.61 +Step 41760/50462 | Loss: 2.5226 | LR: 3.69e-05 | 4874.6ms/step | 12.3 samples/s +Step 41770/50462 | Loss: 2.5293 | LR: 3.68e-05 | 3195.4ms/step | 18.8 samples/s +Step 41780/50462 | Loss: 2.5155 | LR: 3.67e-05 | 3198.1ms/step | 18.8 samples/s +Step 41790/50462 | Loss: 2.5017 | LR: 3.66e-05 | 3199.9ms/step | 18.8 samples/s +Step 41800/50462 | Loss: 2.5071 | LR: 3.65e-05 | 3200.0ms/step | 18.7 samples/s +Step 41810/50462 | Loss: 2.5403 | LR: 3.64e-05 | 3198.0ms/step | 18.8 samples/s +Step 41820/50462 | Loss: 2.4662 | LR: 3.64e-05 | 3200.0ms/step | 18.7 samples/s +Step 41830/50462 | Loss: 2.5149 | LR: 3.63e-05 | 3199.8ms/step | 18.8 samples/s +Step 41840/50462 | Loss: 2.4888 | LR: 3.62e-05 | 3200.0ms/step | 18.7 samples/s +Step 41850/50462 | Loss: 2.5309 | LR: 3.61e-05 | 3199.8ms/step | 18.8 samples/s +Step 41860/50462 | Loss: 2.5164 | LR: 3.60e-05 | 3197.0ms/step | 18.8 samples/s +Step 41870/50462 | Loss: 2.5144 | LR: 3.60e-05 | 3198.7ms/step | 18.8 samples/s +Step 41880/50462 | Loss: 2.5267 | LR: 3.59e-05 | 3254.4ms/step | 18.4 samples/s +Step 41890/50462 | Loss: 2.5188 | LR: 3.58e-05 | 3196.6ms/step | 18.8 samples/s +Step 41900/50462 | Loss: 2.5433 | LR: 3.57e-05 | 3194.7ms/step | 18.8 samples/s +Step 41910/50462 | Loss: 2.5332 | LR: 3.56e-05 | 3192.8ms/step | 18.8 samples/s +Step 41920/50462 | Loss: 2.5250 | LR: 3.55e-05 | 3200.0ms/step | 18.8 samples/s +Step 41930/50462 | Loss: 2.5055 | LR: 3.55e-05 | 3199.9ms/step | 18.8 samples/s +Step 41940/50462 | Loss: 2.5336 | LR: 3.54e-05 | 3200.0ms/step | 18.8 samples/s +Step 41950/50462 | Loss: 2.5043 | LR: 3.53e-05 | 3247.2ms/step | 18.5 samples/s +Step 41960/50462 | Loss: 2.5302 | LR: 3.52e-05 | 3197.0ms/step | 18.8 samples/s +Step 41970/50462 | Loss: 2.5197 | LR: 3.51e-05 | 3254.0ms/step | 18.4 samples/s +Step 41980/50462 | Loss: 2.5171 | LR: 3.51e-05 | 3200.1ms/step | 18.7 samples/s +Step 41990/50462 | Loss: 2.5488 | LR: 3.50e-05 | 3199.5ms/step | 18.8 samples/s +Step 42000/50462 | Loss: 2.5274 | LR: 3.49e-05 | 3200.1ms/step | 18.7 samples/s +Validation at step 42000: Loss = 2.5344, Perplexity = 12.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-42000 +Step 42010/50462 | Loss: 2.4960 | LR: 3.48e-05 | 5774.6ms/step | 10.4 samples/s +Step 42020/50462 | Loss: 2.5378 | LR: 3.47e-05 | 3183.1ms/step | 18.8 samples/s +Step 42030/50462 | Loss: 2.5563 | LR: 3.47e-05 | 3195.7ms/step | 18.8 samples/s +Step 42040/50462 | Loss: 2.5216 | LR: 3.46e-05 | 3198.4ms/step | 18.8 samples/s +Step 42050/50462 | Loss: 2.5236 | LR: 3.45e-05 | 3200.3ms/step | 18.7 samples/s +Step 42060/50462 | Loss: 2.5564 | LR: 3.44e-05 | 3198.1ms/step | 18.8 samples/s +Step 42070/50462 | Loss: 2.5386 | LR: 3.43e-05 | 3200.1ms/step | 18.7 samples/s +Step 42080/50462 | Loss: 2.5108 | LR: 3.43e-05 | 3199.9ms/step | 18.8 samples/s +Step 42090/50462 | Loss: 2.5411 | LR: 3.42e-05 | 3199.5ms/step | 18.8 samples/s +Step 42100/50462 | Loss: 2.5217 | LR: 3.41e-05 | 3200.5ms/step | 18.7 samples/s +Step 42110/50462 | Loss: 2.4908 | LR: 3.40e-05 | 3200.0ms/step | 18.8 samples/s +Step 42120/50462 | Loss: 2.5249 | LR: 3.39e-05 | 3199.7ms/step | 18.8 samples/s +Step 42130/50462 | Loss: 2.5369 | LR: 3.39e-05 | 3200.2ms/step | 18.7 samples/s +Step 42140/50462 | Loss: 2.5332 | LR: 3.38e-05 | 3200.1ms/step | 18.7 samples/s +Step 42150/50462 | Loss: 2.5289 | LR: 3.37e-05 | 3200.0ms/step | 18.8 samples/s +Step 42160/50462 | Loss: 2.4885 | LR: 3.36e-05 | 3200.1ms/step | 18.7 samples/s +Step 42170/50462 | Loss: 2.5145 | LR: 3.35e-05 | 3199.8ms/step | 18.8 samples/s +Step 42180/50462 | Loss: 2.5216 | LR: 3.35e-05 | 3200.3ms/step | 18.7 samples/s +Step 42190/50462 | Loss: 2.5509 | LR: 3.34e-05 | 3199.9ms/step | 18.8 samples/s +Step 42200/50462 | Loss: 2.5075 | LR: 3.33e-05 | 3200.0ms/step | 18.8 samples/s +Step 42210/50462 | Loss: 2.5120 | LR: 3.32e-05 | 3199.7ms/step | 18.8 samples/s +Step 42220/50462 | Loss: 2.5398 | LR: 3.32e-05 | 3256.2ms/step | 18.4 samples/s +Step 42230/50462 | Loss: 2.5257 | LR: 3.31e-05 | 3199.8ms/step | 18.8 samples/s +Step 42240/50462 | Loss: 2.4811 | LR: 3.30e-05 | 3200.4ms/step | 18.7 samples/s +Step 42250/50462 | Loss: 2.5324 | LR: 3.29e-05 | 3199.7ms/step | 18.8 samples/s +Validation at step 42250: Loss = 2.5342, Perplexity = 12.61 +Step 42260/50462 | Loss: 2.5388 | LR: 3.28e-05 | 4874.0ms/step | 12.3 samples/s +Step 42270/50462 | Loss: 2.5500 | LR: 3.28e-05 | 3200.1ms/step | 18.7 samples/s +Step 42280/50462 | Loss: 2.5197 | LR: 3.27e-05 | 3199.8ms/step | 18.8 samples/s +Step 42290/50462 | Loss: 2.5337 | LR: 3.26e-05 | 3252.2ms/step | 18.4 samples/s +Step 42300/50462 | Loss: 2.5193 | LR: 3.25e-05 | 3200.1ms/step | 18.7 samples/s +Step 42310/50462 | Loss: 2.5122 | LR: 3.24e-05 | 3251.9ms/step | 18.5 samples/s +Step 42320/50462 | Loss: 2.4945 | LR: 3.24e-05 | 3197.8ms/step | 18.8 samples/s +Step 42330/50462 | Loss: 2.5419 | LR: 3.23e-05 | 3200.3ms/step | 18.7 samples/s +Step 42340/50462 | Loss: 2.5483 | LR: 3.22e-05 | 3200.0ms/step | 18.8 samples/s +Step 42350/50462 | Loss: 2.4963 | LR: 3.21e-05 | 3199.9ms/step | 18.8 samples/s +Step 42360/50462 | Loss: 2.5078 | LR: 3.21e-05 | 3200.1ms/step | 18.7 samples/s +Step 42370/50462 | Loss: 2.5205 | LR: 3.20e-05 | 3200.0ms/step | 18.7 samples/s +Step 42380/50462 | Loss: 2.5063 | LR: 3.19e-05 | 3199.9ms/step | 18.8 samples/s +Step 42390/50462 | Loss: 2.5342 | LR: 3.18e-05 | 3198.0ms/step | 18.8 samples/s +Step 42400/50462 | Loss: 2.5417 | LR: 3.18e-05 | 3192.3ms/step | 18.8 samples/s +Step 42410/50462 | Loss: 2.5353 | LR: 3.17e-05 | 3193.7ms/step | 18.8 samples/s +Step 42420/50462 | Loss: 2.5279 | LR: 3.16e-05 | 3200.1ms/step | 18.7 samples/s +Step 42430/50462 | Loss: 2.5159 | LR: 3.15e-05 | 3199.9ms/step | 18.8 samples/s +Step 42440/50462 | Loss: 2.5149 | LR: 3.14e-05 | 3200.0ms/step | 18.8 samples/s +Step 42450/50462 | Loss: 2.5342 | LR: 3.14e-05 | 3200.0ms/step | 18.7 samples/s +Step 42460/50462 | Loss: 2.5137 | LR: 3.13e-05 | 3200.0ms/step | 18.8 samples/s +Step 42470/50462 | Loss: 2.5248 | LR: 3.12e-05 | 3200.2ms/step | 18.7 samples/s +Step 42480/50462 | Loss: 2.4824 | LR: 3.11e-05 | 3199.8ms/step | 18.8 samples/s +Step 42490/50462 | Loss: 2.5283 | LR: 3.11e-05 | 3200.0ms/step | 18.7 samples/s +Step 42500/50462 | Loss: 2.5383 | LR: 3.10e-05 | 3200.1ms/step | 18.7 samples/s +Validation at step 42500: Loss = 2.5341, Perplexity = 12.61 +Step 42510/50462 | Loss: 2.5339 | LR: 3.09e-05 | 4873.3ms/step | 12.3 samples/s +Step 42520/50462 | Loss: 2.5179 | LR: 3.08e-05 | 3196.6ms/step | 18.8 samples/s +Step 42530/50462 | Loss: 2.5111 | LR: 3.08e-05 | 3197.8ms/step | 18.8 samples/s +Step 42540/50462 | Loss: 2.5177 | LR: 3.07e-05 | 3200.0ms/step | 18.7 samples/s +Step 42550/50462 | Loss: 2.5426 | LR: 3.06e-05 | 3200.2ms/step | 18.7 samples/s +Step 42560/50462 | Loss: 2.6001 | LR: 3.05e-05 | 3252.1ms/step | 18.4 samples/s +Step 42570/50462 | Loss: 2.5303 | LR: 3.05e-05 | 3199.9ms/step | 18.8 samples/s +Step 42580/50462 | Loss: 2.5244 | LR: 3.04e-05 | 3200.7ms/step | 18.7 samples/s +Step 42590/50462 | Loss: 2.5191 | LR: 3.03e-05 | 3201.3ms/step | 18.7 samples/s +Step 42600/50462 | Loss: 2.5663 | LR: 3.02e-05 | 3199.7ms/step | 18.8 samples/s +Step 42610/50462 | Loss: 2.5155 | LR: 3.02e-05 | 3200.2ms/step | 18.7 samples/s +Step 42620/50462 | Loss: 2.5077 | LR: 3.01e-05 | 3199.9ms/step | 18.8 samples/s +Step 42630/50462 | Loss: 2.5405 | LR: 3.00e-05 | 3252.4ms/step | 18.4 samples/s +Step 42640/50462 | Loss: 2.5658 | LR: 2.99e-05 | 3199.9ms/step | 18.8 samples/s +Step 42650/50462 | Loss: 2.5346 | LR: 2.99e-05 | 3255.8ms/step | 18.4 samples/s +Step 42660/50462 | Loss: 2.5385 | LR: 2.98e-05 | 3185.7ms/step | 18.8 samples/s +Step 42670/50462 | Loss: 2.5071 | LR: 2.97e-05 | 3194.0ms/step | 18.8 samples/s +Step 42680/50462 | Loss: 2.5074 | LR: 2.96e-05 | 3195.5ms/step | 18.8 samples/s +Step 42690/50462 | Loss: 2.5275 | LR: 2.96e-05 | 3200.9ms/step | 18.7 samples/s +Step 42700/50462 | Loss: 2.5349 | LR: 2.95e-05 | 3200.0ms/step | 18.8 samples/s +Step 42710/50462 | Loss: 2.5166 | LR: 2.94e-05 | 3199.5ms/step | 18.8 samples/s +Step 42720/50462 | Loss: 2.5179 | LR: 2.93e-05 | 3200.6ms/step | 18.7 samples/s +Step 42730/50462 | Loss: 2.5196 | LR: 2.93e-05 | 3199.8ms/step | 18.8 samples/s +Step 42740/50462 | Loss: 2.5232 | LR: 2.92e-05 | 3199.9ms/step | 18.8 samples/s +Step 42750/50462 | Loss: 2.5193 | LR: 2.91e-05 | 3198.3ms/step | 18.8 samples/s +Validation at step 42750: Loss = 2.5340, Perplexity = 12.60 +Step 42760/50462 | Loss: 2.5334 | LR: 2.90e-05 | 4875.9ms/step | 12.3 samples/s +Step 42770/50462 | Loss: 2.5242 | LR: 2.90e-05 | 3200.0ms/step | 18.7 samples/s +Step 42780/50462 | Loss: 2.5216 | LR: 2.89e-05 | 3198.0ms/step | 18.8 samples/s +Step 42790/50462 | Loss: 2.5387 | LR: 2.88e-05 | 3200.1ms/step | 18.7 samples/s +Step 42800/50462 | Loss: 2.5391 | LR: 2.87e-05 | 3200.0ms/step | 18.7 samples/s +Step 42810/50462 | Loss: 2.5439 | LR: 2.87e-05 | 3199.8ms/step | 18.8 samples/s +Step 42820/50462 | Loss: 2.5148 | LR: 2.86e-05 | 3197.3ms/step | 18.8 samples/s +Step 42830/50462 | Loss: 2.5076 | LR: 2.85e-05 | 3198.6ms/step | 18.8 samples/s +Step 42840/50462 | Loss: 2.5360 | LR: 2.84e-05 | 3200.4ms/step | 18.7 samples/s +Step 42850/50462 | Loss: 2.5246 | LR: 2.84e-05 | 3199.8ms/step | 18.8 samples/s +Step 42860/50462 | Loss: 2.5540 | LR: 2.83e-05 | 3199.9ms/step | 18.8 samples/s +Step 42870/50462 | Loss: 2.5357 | LR: 2.82e-05 | 3200.3ms/step | 18.7 samples/s +Step 42880/50462 | Loss: 2.5322 | LR: 2.82e-05 | 3199.5ms/step | 18.8 samples/s +Step 42890/50462 | Loss: 2.5153 | LR: 2.81e-05 | 3199.1ms/step | 18.8 samples/s +Step 42900/50462 | Loss: 2.5239 | LR: 2.80e-05 | 3252.7ms/step | 18.4 samples/s +Step 42910/50462 | Loss: 2.5541 | LR: 2.79e-05 | 3198.7ms/step | 18.8 samples/s +Step 42920/50462 | Loss: 2.5135 | LR: 2.79e-05 | 3199.5ms/step | 18.8 samples/s +Step 42930/50462 | Loss: 2.5119 | LR: 2.78e-05 | 3200.4ms/step | 18.7 samples/s +Step 42940/50462 | Loss: 2.5025 | LR: 2.77e-05 | 3199.9ms/step | 18.8 samples/s +Step 42950/50462 | Loss: 2.5517 | LR: 2.76e-05 | 3200.1ms/step | 18.7 samples/s +Step 42960/50462 | Loss: 2.5350 | LR: 2.76e-05 | 3199.8ms/step | 18.8 samples/s +Step 42970/50462 | Loss: 2.5058 | LR: 2.75e-05 | 3249.5ms/step | 18.5 samples/s +Step 42980/50462 | Loss: 2.5207 | LR: 2.74e-05 | 3196.6ms/step | 18.8 samples/s +Step 42990/50462 | Loss: 2.5283 | LR: 2.74e-05 | 3255.8ms/step | 18.4 samples/s +Step 43000/50462 | Loss: 2.5058 | LR: 2.73e-05 | 3200.2ms/step | 18.7 samples/s +Validation at step 43000: Loss = 2.5339, Perplexity = 12.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-43000 +Step 43010/50462 | Loss: 2.5463 | LR: 2.72e-05 | 5767.1ms/step | 10.4 samples/s +Step 43020/50462 | Loss: 2.5186 | LR: 2.71e-05 | 3186.8ms/step | 18.8 samples/s +Step 43030/50462 | Loss: 2.5255 | LR: 2.71e-05 | 3193.3ms/step | 18.8 samples/s +Step 43040/50462 | Loss: 2.5119 | LR: 2.70e-05 | 3200.8ms/step | 18.7 samples/s +Step 43050/50462 | Loss: 2.5329 | LR: 2.69e-05 | 3197.9ms/step | 18.8 samples/s +Step 43060/50462 | Loss: 2.5082 | LR: 2.69e-05 | 3197.9ms/step | 18.8 samples/s +Step 43070/50462 | Loss: 2.5261 | LR: 2.68e-05 | 3196.3ms/step | 18.8 samples/s +Step 43080/50462 | Loss: 2.5054 | LR: 2.67e-05 | 3200.0ms/step | 18.7 samples/s +Step 43090/50462 | Loss: 2.5006 | LR: 2.66e-05 | 3199.5ms/step | 18.8 samples/s +Step 43100/50462 | Loss: 2.4998 | LR: 2.66e-05 | 3198.4ms/step | 18.8 samples/s +Step 43110/50462 | Loss: 2.5087 | LR: 2.65e-05 | 3200.0ms/step | 18.8 samples/s +Step 43120/50462 | Loss: 2.5232 | LR: 2.64e-05 | 3200.1ms/step | 18.7 samples/s +Step 43130/50462 | Loss: 2.5340 | LR: 2.64e-05 | 3199.9ms/step | 18.8 samples/s +Step 43140/50462 | Loss: 2.5035 | LR: 2.63e-05 | 3199.4ms/step | 18.8 samples/s +Step 43150/50462 | Loss: 2.5050 | LR: 2.62e-05 | 3200.5ms/step | 18.7 samples/s +Step 43160/50462 | Loss: 2.5445 | LR: 2.61e-05 | 3200.1ms/step | 18.7 samples/s +Step 43170/50462 | Loss: 2.5361 | LR: 2.61e-05 | 3200.1ms/step | 18.7 samples/s +Step 43180/50462 | Loss: 2.4900 | LR: 2.60e-05 | 3197.8ms/step | 18.8 samples/s +Step 43190/50462 | Loss: 2.5444 | LR: 2.59e-05 | 3200.2ms/step | 18.7 samples/s +Step 43200/50462 | Loss: 2.5223 | LR: 2.59e-05 | 3199.8ms/step | 18.8 samples/s +Step 43210/50462 | Loss: 2.5285 | LR: 2.58e-05 | 3200.0ms/step | 18.8 samples/s +Step 43220/50462 | Loss: 2.5356 | LR: 2.57e-05 | 3200.1ms/step | 18.7 samples/s +Step 43230/50462 | Loss: 2.5205 | LR: 2.57e-05 | 3200.1ms/step | 18.7 samples/s +Step 43240/50462 | Loss: 2.5434 | LR: 2.56e-05 | 3247.9ms/step | 18.5 samples/s +Step 43250/50462 | Loss: 2.4930 | LR: 2.55e-05 | 3200.0ms/step | 18.8 samples/s +Validation at step 43250: Loss = 2.5338, Perplexity = 12.60 +Step 43260/50462 | Loss: 2.4955 | LR: 2.54e-05 | 4875.2ms/step | 12.3 samples/s +Step 43270/50462 | Loss: 2.5057 | LR: 2.54e-05 | 3197.5ms/step | 18.8 samples/s +Step 43280/50462 | Loss: 2.5285 | LR: 2.53e-05 | 3197.2ms/step | 18.8 samples/s +Step 43290/50462 | Loss: 2.5071 | LR: 2.52e-05 | 3200.3ms/step | 18.7 samples/s +Step 43300/50462 | Loss: 2.5509 | LR: 2.52e-05 | 3199.5ms/step | 18.8 samples/s +Step 43310/50462 | Loss: 2.5158 | LR: 2.51e-05 | 3252.5ms/step | 18.4 samples/s +Step 43320/50462 | Loss: 2.5192 | LR: 2.50e-05 | 3199.4ms/step | 18.8 samples/s +Step 43330/50462 | Loss: 2.5049 | LR: 2.50e-05 | 3254.1ms/step | 18.4 samples/s +Step 43340/50462 | Loss: 2.5158 | LR: 2.49e-05 | 3193.8ms/step | 18.8 samples/s +Step 43350/50462 | Loss: 2.5023 | LR: 2.48e-05 | 3198.6ms/step | 18.8 samples/s +Step 43360/50462 | Loss: 2.5351 | LR: 2.48e-05 | 3199.9ms/step | 18.8 samples/s +Step 43370/50462 | Loss: 2.5272 | LR: 2.47e-05 | 3198.7ms/step | 18.8 samples/s +Step 43380/50462 | Loss: 2.5061 | LR: 2.46e-05 | 3198.9ms/step | 18.8 samples/s +Step 43390/50462 | Loss: 2.5354 | LR: 2.46e-05 | 3199.7ms/step | 18.8 samples/s +Step 43400/50462 | Loss: 2.5169 | LR: 2.45e-05 | 3200.6ms/step | 18.7 samples/s +Step 43410/50462 | Loss: 2.5371 | LR: 2.44e-05 | 3199.5ms/step | 18.8 samples/s +Step 43420/50462 | Loss: 2.5156 | LR: 2.44e-05 | 3200.5ms/step | 18.7 samples/s +Step 43430/50462 | Loss: 2.5219 | LR: 2.43e-05 | 3199.4ms/step | 18.8 samples/s +Step 43440/50462 | Loss: 2.5184 | LR: 2.42e-05 | 3200.6ms/step | 18.7 samples/s +Step 43450/50462 | Loss: 2.5431 | LR: 2.41e-05 | 3199.5ms/step | 18.8 samples/s +Step 43460/50462 | Loss: 2.5309 | LR: 2.41e-05 | 3198.5ms/step | 18.8 samples/s +Step 43470/50462 | Loss: 2.5451 | LR: 2.40e-05 | 3197.9ms/step | 18.8 samples/s +Step 43480/50462 | Loss: 2.5453 | LR: 2.39e-05 | 3199.6ms/step | 18.8 samples/s +Step 43490/50462 | Loss: 2.5132 | LR: 2.39e-05 | 3198.4ms/step | 18.8 samples/s +Step 43500/50462 | Loss: 2.5315 | LR: 2.38e-05 | 3200.0ms/step | 18.8 samples/s +Validation at step 43500: Loss = 2.5338, Perplexity = 12.60 +Step 43510/50462 | Loss: 2.5336 | LR: 2.37e-05 | 4874.2ms/step | 12.3 samples/s +Step 43520/50462 | Loss: 2.4959 | LR: 2.37e-05 | 3197.9ms/step | 18.8 samples/s +Step 43530/50462 | Loss: 2.5180 | LR: 2.36e-05 | 3200.1ms/step | 18.7 samples/s +Step 43540/50462 | Loss: 2.5148 | LR: 2.35e-05 | 3198.6ms/step | 18.8 samples/s +Step 43550/50462 | Loss: 2.5232 | LR: 2.35e-05 | 3198.9ms/step | 18.8 samples/s +Step 43560/50462 | Loss: 2.5033 | LR: 2.34e-05 | 3200.4ms/step | 18.7 samples/s +Step 43570/50462 | Loss: 2.5237 | LR: 2.33e-05 | 3198.0ms/step | 18.8 samples/s +Step 43580/50462 | Loss: 2.5515 | LR: 2.33e-05 | 3255.9ms/step | 18.4 samples/s +Step 43590/50462 | Loss: 2.5083 | LR: 2.32e-05 | 3200.1ms/step | 18.7 samples/s +Step 43600/50462 | Loss: 2.5421 | LR: 2.31e-05 | 3198.2ms/step | 18.8 samples/s +Step 43610/50462 | Loss: 2.5292 | LR: 2.31e-05 | 3199.7ms/step | 18.8 samples/s +Step 43620/50462 | Loss: 2.5109 | LR: 2.30e-05 | 3200.2ms/step | 18.7 samples/s +Step 43630/50462 | Loss: 2.5494 | LR: 2.29e-05 | 3197.4ms/step | 18.8 samples/s +Step 43640/50462 | Loss: 2.5109 | LR: 2.29e-05 | 3200.0ms/step | 18.8 samples/s +Step 43650/50462 | Loss: 2.5319 | LR: 2.28e-05 | 3252.1ms/step | 18.4 samples/s +Step 43660/50462 | Loss: 2.5445 | LR: 2.27e-05 | 3196.2ms/step | 18.8 samples/s +Step 43670/50462 | Loss: 2.5318 | LR: 2.27e-05 | 3258.9ms/step | 18.4 samples/s +Step 43680/50462 | Loss: 2.5353 | LR: 2.26e-05 | 3196.6ms/step | 18.8 samples/s +Step 43690/50462 | Loss: 2.5288 | LR: 2.25e-05 | 3194.8ms/step | 18.8 samples/s +Step 43700/50462 | Loss: 2.5452 | LR: 2.25e-05 | 3197.5ms/step | 18.8 samples/s +Step 43710/50462 | Loss: 2.5512 | LR: 2.24e-05 | 3200.2ms/step | 18.7 samples/s +Step 43720/50462 | Loss: 2.5063 | LR: 2.24e-05 | 3200.2ms/step | 18.7 samples/s +Step 43730/50462 | Loss: 2.4963 | LR: 2.23e-05 | 3197.7ms/step | 18.8 samples/s +Step 43740/50462 | Loss: 2.5197 | LR: 2.22e-05 | 3199.7ms/step | 18.8 samples/s +Step 43750/50462 | Loss: 2.5207 | LR: 2.22e-05 | 3200.3ms/step | 18.7 samples/s +Validation at step 43750: Loss = 2.5338, Perplexity = 12.60 +Step 43760/50462 | Loss: 2.5265 | LR: 2.21e-05 | 4874.1ms/step | 12.3 samples/s +Step 43770/50462 | Loss: 2.5113 | LR: 2.20e-05 | 3198.2ms/step | 18.8 samples/s +Step 43780/50462 | Loss: 2.5194 | LR: 2.20e-05 | 3199.2ms/step | 18.8 samples/s +Step 43790/50462 | Loss: 2.5271 | LR: 2.19e-05 | 3200.6ms/step | 18.7 samples/s +Step 43800/50462 | Loss: 2.5402 | LR: 2.18e-05 | 3200.1ms/step | 18.7 samples/s +Step 43810/50462 | Loss: 2.4849 | LR: 2.18e-05 | 3192.0ms/step | 18.8 samples/s +Step 43820/50462 | Loss: 2.4883 | LR: 2.17e-05 | 3199.3ms/step | 18.8 samples/s +Step 43830/50462 | Loss: 2.5330 | LR: 2.16e-05 | 3196.7ms/step | 18.8 samples/s +Step 43840/50462 | Loss: 2.5482 | LR: 2.16e-05 | 3199.8ms/step | 18.8 samples/s +Step 43850/50462 | Loss: 2.5420 | LR: 2.15e-05 | 3200.1ms/step | 18.7 samples/s +Step 43860/50462 | Loss: 2.5254 | LR: 2.14e-05 | 3200.2ms/step | 18.7 samples/s +Step 43870/50462 | Loss: 2.5149 | LR: 2.14e-05 | 3199.8ms/step | 18.8 samples/s +Step 43880/50462 | Loss: 2.5348 | LR: 2.13e-05 | 3198.1ms/step | 18.8 samples/s +Step 43890/50462 | Loss: 2.5266 | LR: 2.13e-05 | 3198.3ms/step | 18.8 samples/s +Step 43900/50462 | Loss: 2.5792 | LR: 2.12e-05 | 3199.4ms/step | 18.8 samples/s +Step 43910/50462 | Loss: 2.5448 | LR: 2.11e-05 | 3198.0ms/step | 18.8 samples/s +Step 43920/50462 | Loss: 2.5209 | LR: 2.11e-05 | 3250.6ms/step | 18.5 samples/s +Step 43930/50462 | Loss: 2.5472 | LR: 2.10e-05 | 3196.9ms/step | 18.8 samples/s +Step 43940/50462 | Loss: 2.5225 | LR: 2.09e-05 | 3198.8ms/step | 18.8 samples/s +Step 43950/50462 | Loss: 2.5344 | LR: 2.09e-05 | 3199.4ms/step | 18.8 samples/s +Step 43960/50462 | Loss: 2.5167 | LR: 2.08e-05 | 3200.5ms/step | 18.7 samples/s +Step 43970/50462 | Loss: 2.5118 | LR: 2.07e-05 | 3199.8ms/step | 18.8 samples/s +Step 43980/50462 | Loss: 2.5046 | LR: 2.07e-05 | 3199.8ms/step | 18.8 samples/s +Step 43990/50462 | Loss: 2.5399 | LR: 2.06e-05 | 3254.5ms/step | 18.4 samples/s +Step 44000/50462 | Loss: 2.5458 | LR: 2.06e-05 | 3197.9ms/step | 18.8 samples/s +Validation at step 44000: Loss = 2.5337, Perplexity = 12.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-44000 +Step 44010/50462 | Loss: 2.5307 | LR: 2.05e-05 | 5799.8ms/step | 10.3 samples/s +Step 44020/50462 | Loss: 2.5227 | LR: 2.04e-05 | 3179.0ms/step | 18.9 samples/s +Step 44030/50462 | Loss: 2.5193 | LR: 2.04e-05 | 3187.6ms/step | 18.8 samples/s +Step 44040/50462 | Loss: 2.5131 | LR: 2.03e-05 | 3187.6ms/step | 18.8 samples/s +Step 44050/50462 | Loss: 2.5468 | LR: 2.02e-05 | 3191.8ms/step | 18.8 samples/s +Step 44060/50462 | Loss: 2.5459 | LR: 2.02e-05 | 3190.3ms/step | 18.8 samples/s +Step 44070/50462 | Loss: 2.5449 | LR: 2.01e-05 | 3199.9ms/step | 18.8 samples/s +Step 44080/50462 | Loss: 2.5270 | LR: 2.01e-05 | 3199.0ms/step | 18.8 samples/s +Step 44090/50462 | Loss: 2.4956 | LR: 2.00e-05 | 3200.9ms/step | 18.7 samples/s +Step 44100/50462 | Loss: 2.5145 | LR: 1.99e-05 | 3197.8ms/step | 18.8 samples/s +Step 44110/50462 | Loss: 2.5185 | LR: 1.99e-05 | 3200.2ms/step | 18.7 samples/s +Step 44120/50462 | Loss: 2.5340 | LR: 1.98e-05 | 3198.8ms/step | 18.8 samples/s +Step 44130/50462 | Loss: 2.5398 | LR: 1.97e-05 | 3199.2ms/step | 18.8 samples/s +Step 44140/50462 | Loss: 2.5080 | LR: 1.97e-05 | 3200.0ms/step | 18.7 samples/s +Step 44150/50462 | Loss: 2.5040 | LR: 1.96e-05 | 3200.1ms/step | 18.7 samples/s +Step 44160/50462 | Loss: 2.5329 | LR: 1.96e-05 | 3197.8ms/step | 18.8 samples/s +Step 44170/50462 | Loss: 2.5385 | LR: 1.95e-05 | 3200.1ms/step | 18.7 samples/s +Step 44180/50462 | Loss: 2.5050 | LR: 1.94e-05 | 3199.4ms/step | 18.8 samples/s +Step 44190/50462 | Loss: 2.5099 | LR: 1.94e-05 | 3200.6ms/step | 18.7 samples/s +Step 44200/50462 | Loss: 2.5321 | LR: 1.93e-05 | 3200.0ms/step | 18.8 samples/s +Step 44210/50462 | Loss: 2.5208 | LR: 1.93e-05 | 3197.8ms/step | 18.8 samples/s +Step 44220/50462 | Loss: 2.5251 | LR: 1.92e-05 | 3200.2ms/step | 18.7 samples/s +Step 44230/50462 | Loss: 2.5377 | LR: 1.91e-05 | 3199.4ms/step | 18.8 samples/s +Step 44240/50462 | Loss: 2.5250 | LR: 1.91e-05 | 3200.6ms/step | 18.7 samples/s +Step 44250/50462 | Loss: 2.5391 | LR: 1.90e-05 | 3253.3ms/step | 18.4 samples/s +Validation at step 44250: Loss = 2.5337, Perplexity = 12.60 +Step 44260/50462 | Loss: 2.5329 | LR: 1.90e-05 | 4870.8ms/step | 12.3 samples/s +Step 44270/50462 | Loss: 2.5045 | LR: 1.89e-05 | 3196.6ms/step | 18.8 samples/s +Step 44280/50462 | Loss: 2.5262 | LR: 1.88e-05 | 3199.3ms/step | 18.8 samples/s +Step 44290/50462 | Loss: 2.5229 | LR: 1.88e-05 | 3197.8ms/step | 18.8 samples/s +Step 44300/50462 | Loss: 2.4745 | LR: 1.87e-05 | 3200.3ms/step | 18.7 samples/s +Step 44310/50462 | Loss: 2.5036 | LR: 1.87e-05 | 3199.8ms/step | 18.8 samples/s +Step 44320/50462 | Loss: 2.5373 | LR: 1.86e-05 | 3200.0ms/step | 18.7 samples/s +Step 44330/50462 | Loss: 2.5275 | LR: 1.85e-05 | 3250.0ms/step | 18.5 samples/s +Step 44340/50462 | Loss: 2.5226 | LR: 1.85e-05 | 3184.3ms/step | 18.8 samples/s +Step 44350/50462 | Loss: 2.5297 | LR: 1.84e-05 | 3261.0ms/step | 18.4 samples/s +Step 44360/50462 | Loss: 2.5018 | LR: 1.84e-05 | 3194.8ms/step | 18.8 samples/s +Step 44370/50462 | Loss: 2.5272 | LR: 1.83e-05 | 3197.8ms/step | 18.8 samples/s +Step 44380/50462 | Loss: 2.5176 | LR: 1.82e-05 | 3200.0ms/step | 18.7 samples/s +Step 44390/50462 | Loss: 2.5379 | LR: 1.82e-05 | 3200.1ms/step | 18.7 samples/s +Step 44400/50462 | Loss: 2.5054 | LR: 1.81e-05 | 3198.0ms/step | 18.8 samples/s +Step 44410/50462 | Loss: 2.5137 | LR: 1.81e-05 | 3195.8ms/step | 18.8 samples/s +Step 44420/50462 | Loss: 2.5236 | LR: 1.80e-05 | 3196.0ms/step | 18.8 samples/s +Step 44430/50462 | Loss: 2.5315 | LR: 1.79e-05 | 3196.1ms/step | 18.8 samples/s +Step 44440/50462 | Loss: 2.5204 | LR: 1.79e-05 | 3200.0ms/step | 18.7 samples/s +Step 44450/50462 | Loss: 2.5081 | LR: 1.78e-05 | 3200.1ms/step | 18.7 samples/s +Step 44460/50462 | Loss: 2.5247 | LR: 1.78e-05 | 3197.7ms/step | 18.8 samples/s +Step 44470/50462 | Loss: 2.5370 | LR: 1.77e-05 | 3200.4ms/step | 18.7 samples/s +Step 44480/50462 | Loss: 2.5517 | LR: 1.77e-05 | 3197.8ms/step | 18.8 samples/s +Step 44490/50462 | Loss: 2.5469 | LR: 1.76e-05 | 3198.2ms/step | 18.8 samples/s +Step 44500/50462 | Loss: 2.5099 | LR: 1.75e-05 | 3199.3ms/step | 18.8 samples/s +Validation at step 44500: Loss = 2.5337, Perplexity = 12.60 +Step 44510/50462 | Loss: 2.4983 | LR: 1.75e-05 | 4876.5ms/step | 12.3 samples/s +Step 44520/50462 | Loss: 2.4978 | LR: 1.74e-05 | 3200.1ms/step | 18.7 samples/s +Step 44530/50462 | Loss: 2.5238 | LR: 1.74e-05 | 3199.4ms/step | 18.8 samples/s +Step 44540/50462 | Loss: 2.5127 | LR: 1.73e-05 | 3200.5ms/step | 18.7 samples/s +Step 44550/50462 | Loss: 2.5259 | LR: 1.72e-05 | 3200.1ms/step | 18.7 samples/s +Step 44560/50462 | Loss: 2.5299 | LR: 1.72e-05 | 3199.9ms/step | 18.8 samples/s +Step 44570/50462 | Loss: 2.5178 | LR: 1.71e-05 | 3199.5ms/step | 18.8 samples/s +Step 44580/50462 | Loss: 2.4958 | LR: 1.71e-05 | 3200.7ms/step | 18.7 samples/s +Step 44590/50462 | Loss: 2.5142 | LR: 1.70e-05 | 3251.8ms/step | 18.5 samples/s +Step 44600/50462 | Loss: 2.5377 | LR: 1.70e-05 | 3199.7ms/step | 18.8 samples/s +Step 44610/50462 | Loss: 2.5200 | LR: 1.69e-05 | 3196.3ms/step | 18.8 samples/s +Step 44620/50462 | Loss: 2.5383 | LR: 1.68e-05 | 3196.1ms/step | 18.8 samples/s +Step 44630/50462 | Loss: 2.5214 | LR: 1.68e-05 | 3196.1ms/step | 18.8 samples/s +Step 44640/50462 | Loss: 2.5128 | LR: 1.67e-05 | 3199.9ms/step | 18.8 samples/s +Step 44650/50462 | Loss: 2.5012 | LR: 1.67e-05 | 3199.7ms/step | 18.8 samples/s +Step 44660/50462 | Loss: 2.5396 | LR: 1.66e-05 | 3198.3ms/step | 18.8 samples/s +Step 44670/50462 | Loss: 2.5488 | LR: 1.66e-05 | 3249.3ms/step | 18.5 samples/s +Step 44680/50462 | Loss: 2.5348 | LR: 1.65e-05 | 3187.4ms/step | 18.8 samples/s +Step 44690/50462 | Loss: 2.5318 | LR: 1.64e-05 | 3259.3ms/step | 18.4 samples/s +Step 44700/50462 | Loss: 2.5375 | LR: 1.64e-05 | 3195.1ms/step | 18.8 samples/s +Step 44710/50462 | Loss: 2.5208 | LR: 1.63e-05 | 3192.6ms/step | 18.8 samples/s +Step 44720/50462 | Loss: 2.5221 | LR: 1.63e-05 | 3195.6ms/step | 18.8 samples/s +Step 44730/50462 | Loss: 2.5093 | LR: 1.62e-05 | 3198.7ms/step | 18.8 samples/s +Step 44740/50462 | Loss: 2.5058 | LR: 1.62e-05 | 3197.4ms/step | 18.8 samples/s +Step 44750/50462 | Loss: 2.5304 | LR: 1.61e-05 | 3198.6ms/step | 18.8 samples/s +Validation at step 44750: Loss = 2.5336, Perplexity = 12.60 +Step 44760/50462 | Loss: 2.5454 | LR: 1.61e-05 | 4866.0ms/step | 12.3 samples/s +Step 44770/50462 | Loss: 2.4955 | LR: 1.60e-05 | 3199.7ms/step | 18.8 samples/s +Step 44780/50462 | Loss: 2.5233 | LR: 1.59e-05 | 3198.1ms/step | 18.8 samples/s +Step 44790/50462 | Loss: 2.5121 | LR: 1.59e-05 | 3197.9ms/step | 18.8 samples/s +Step 44800/50462 | Loss: 2.5130 | LR: 1.58e-05 | 3200.2ms/step | 18.7 samples/s +Step 44810/50462 | Loss: 2.5405 | LR: 1.58e-05 | 3197.9ms/step | 18.8 samples/s +Step 44820/50462 | Loss: 2.5294 | LR: 1.57e-05 | 3191.5ms/step | 18.8 samples/s +Step 44830/50462 | Loss: 2.5527 | LR: 1.57e-05 | 3200.7ms/step | 18.7 samples/s +Step 44840/50462 | Loss: 2.5076 | LR: 1.56e-05 | 3196.2ms/step | 18.8 samples/s +Step 44850/50462 | Loss: 2.5123 | LR: 1.56e-05 | 3196.3ms/step | 18.8 samples/s +Step 44860/50462 | Loss: 2.5101 | LR: 1.55e-05 | 3197.3ms/step | 18.8 samples/s +Step 44870/50462 | Loss: 2.5009 | LR: 1.54e-05 | 3198.0ms/step | 18.8 samples/s +Step 44880/50462 | Loss: 2.5252 | LR: 1.54e-05 | 3200.2ms/step | 18.7 samples/s +Step 44890/50462 | Loss: 2.4871 | LR: 1.53e-05 | 3195.8ms/step | 18.8 samples/s +Step 44900/50462 | Loss: 2.4865 | LR: 1.53e-05 | 3196.0ms/step | 18.8 samples/s +Step 44910/50462 | Loss: 2.5080 | LR: 1.52e-05 | 3196.1ms/step | 18.8 samples/s +Step 44920/50462 | Loss: 2.5122 | LR: 1.52e-05 | 3196.1ms/step | 18.8 samples/s +Step 44930/50462 | Loss: 2.5160 | LR: 1.51e-05 | 3249.6ms/step | 18.5 samples/s +Step 44940/50462 | Loss: 2.5179 | LR: 1.51e-05 | 3196.3ms/step | 18.8 samples/s +Step 44950/50462 | Loss: 2.4764 | LR: 1.50e-05 | 3199.4ms/step | 18.8 samples/s +Step 44960/50462 | Loss: 2.5554 | LR: 1.50e-05 | 3196.8ms/step | 18.8 samples/s +Step 44970/50462 | Loss: 2.5045 | LR: 1.49e-05 | 3197.7ms/step | 18.8 samples/s +Step 44980/50462 | Loss: 2.5174 | LR: 1.49e-05 | 3195.5ms/step | 18.8 samples/s +Step 44990/50462 | Loss: 2.5444 | LR: 1.48e-05 | 3193.3ms/step | 18.8 samples/s +Step 45000/50462 | Loss: 2.5366 | LR: 1.47e-05 | 3184.2ms/step | 18.8 samples/s +Validation at step 45000: Loss = 2.5336, Perplexity = 12.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-45000 +Step 45010/50462 | Loss: 2.5282 | LR: 1.47e-05 | 5827.3ms/step | 10.3 samples/s +Step 45020/50462 | Loss: 2.5120 | LR: 1.46e-05 | 3180.6ms/step | 18.9 samples/s +Step 45030/50462 | Loss: 2.5188 | LR: 1.46e-05 | 3238.8ms/step | 18.5 samples/s +Step 45040/50462 | Loss: 2.5318 | LR: 1.45e-05 | 3182.7ms/step | 18.9 samples/s +Step 45050/50462 | Loss: 2.5058 | LR: 1.45e-05 | 3197.7ms/step | 18.8 samples/s +Step 45060/50462 | Loss: 2.5299 | LR: 1.44e-05 | 3195.7ms/step | 18.8 samples/s +Step 45070/50462 | Loss: 2.5234 | LR: 1.44e-05 | 3188.5ms/step | 18.8 samples/s +Step 45080/50462 | Loss: 2.5149 | LR: 1.43e-05 | 3190.2ms/step | 18.8 samples/s +Step 45090/50462 | Loss: 2.5143 | LR: 1.43e-05 | 3193.7ms/step | 18.8 samples/s +Step 45100/50462 | Loss: 2.5488 | LR: 1.42e-05 | 3194.0ms/step | 18.8 samples/s +Step 45110/50462 | Loss: 2.5481 | LR: 1.42e-05 | 3190.0ms/step | 18.8 samples/s +Step 45120/50462 | Loss: 2.5319 | LR: 1.41e-05 | 3194.1ms/step | 18.8 samples/s +Step 45130/50462 | Loss: 2.5380 | LR: 1.41e-05 | 3195.8ms/step | 18.8 samples/s +Step 45140/50462 | Loss: 2.5088 | LR: 1.40e-05 | 3196.0ms/step | 18.8 samples/s +Step 45150/50462 | Loss: 2.5329 | LR: 1.40e-05 | 3195.7ms/step | 18.8 samples/s +Step 45160/50462 | Loss: 2.5394 | LR: 1.39e-05 | 3196.4ms/step | 18.8 samples/s +Step 45170/50462 | Loss: 2.5014 | LR: 1.39e-05 | 3199.9ms/step | 18.8 samples/s +Step 45180/50462 | Loss: 2.5302 | LR: 1.38e-05 | 3199.5ms/step | 18.8 samples/s +Step 45190/50462 | Loss: 2.5165 | LR: 1.37e-05 | 3192.4ms/step | 18.8 samples/s +Step 45200/50462 | Loss: 2.5398 | LR: 1.37e-05 | 3186.2ms/step | 18.8 samples/s +Step 45210/50462 | Loss: 2.5245 | LR: 1.36e-05 | 3189.9ms/step | 18.8 samples/s +Step 45220/50462 | Loss: 2.5099 | LR: 1.36e-05 | 3193.7ms/step | 18.8 samples/s +Step 45230/50462 | Loss: 2.5050 | LR: 1.35e-05 | 3196.3ms/step | 18.8 samples/s +Step 45240/50462 | Loss: 2.5390 | LR: 1.35e-05 | 3177.5ms/step | 18.9 samples/s +Step 45250/50462 | Loss: 2.5177 | LR: 1.34e-05 | 3186.3ms/step | 18.8 samples/s +Validation at step 45250: Loss = 2.5335, Perplexity = 12.60 +Step 45260/50462 | Loss: 2.5293 | LR: 1.34e-05 | 4872.2ms/step | 12.3 samples/s +Step 45270/50462 | Loss: 2.5376 | LR: 1.33e-05 | 3243.1ms/step | 18.5 samples/s +Step 45280/50462 | Loss: 2.4950 | LR: 1.33e-05 | 3182.8ms/step | 18.9 samples/s +Step 45290/50462 | Loss: 2.5220 | LR: 1.32e-05 | 3190.7ms/step | 18.8 samples/s +Step 45300/50462 | Loss: 2.5392 | LR: 1.32e-05 | 3188.0ms/step | 18.8 samples/s +Step 45310/50462 | Loss: 2.5089 | LR: 1.31e-05 | 3188.5ms/step | 18.8 samples/s +Step 45320/50462 | Loss: 2.5171 | LR: 1.31e-05 | 3185.2ms/step | 18.8 samples/s +Step 45330/50462 | Loss: 2.5323 | LR: 1.30e-05 | 3187.4ms/step | 18.8 samples/s +Step 45340/50462 | Loss: 2.5243 | LR: 1.30e-05 | 3184.1ms/step | 18.8 samples/s +Step 45350/50462 | Loss: 2.5220 | LR: 1.29e-05 | 3240.2ms/step | 18.5 samples/s +Step 45360/50462 | Loss: 2.5414 | LR: 1.29e-05 | 3179.3ms/step | 18.9 samples/s +Step 45370/50462 | Loss: 2.5138 | LR: 1.28e-05 | 3237.3ms/step | 18.5 samples/s +Step 45380/50462 | Loss: 2.5114 | LR: 1.28e-05 | 3193.4ms/step | 18.8 samples/s +Step 45390/50462 | Loss: 2.5173 | LR: 1.27e-05 | 3197.2ms/step | 18.8 samples/s +Step 45400/50462 | Loss: 2.5011 | LR: 1.27e-05 | 3198.8ms/step | 18.8 samples/s +Step 45410/50462 | Loss: 2.5453 | LR: 1.26e-05 | 3197.4ms/step | 18.8 samples/s +Step 45420/50462 | Loss: 2.5160 | LR: 1.26e-05 | 3200.3ms/step | 18.7 samples/s +Step 45430/50462 | Loss: 2.4919 | LR: 1.25e-05 | 3199.7ms/step | 18.8 samples/s +Step 45440/50462 | Loss: 2.5323 | LR: 1.25e-05 | 3196.7ms/step | 18.8 samples/s +Step 45450/50462 | Loss: 2.5238 | LR: 1.24e-05 | 3195.4ms/step | 18.8 samples/s +Step 45460/50462 | Loss: 2.5123 | LR: 1.24e-05 | 3197.9ms/step | 18.8 samples/s +Step 45470/50462 | Loss: 2.5216 | LR: 1.23e-05 | 3199.5ms/step | 18.8 samples/s +Step 45480/50462 | Loss: 2.5115 | LR: 1.23e-05 | 3198.6ms/step | 18.8 samples/s +Step 45490/50462 | Loss: 2.5333 | LR: 1.22e-05 | 3195.1ms/step | 18.8 samples/s +Step 45500/50462 | Loss: 2.5336 | LR: 1.22e-05 | 3199.2ms/step | 18.8 samples/s +Validation at step 45500: Loss = 2.5335, Perplexity = 12.60 +Step 45510/50462 | Loss: 2.5075 | LR: 1.21e-05 | 4872.5ms/step | 12.3 samples/s +Step 45520/50462 | Loss: 2.5527 | LR: 1.21e-05 | 3199.7ms/step | 18.8 samples/s +Step 45530/50462 | Loss: 2.4991 | LR: 1.20e-05 | 3198.2ms/step | 18.8 samples/s +Step 45540/50462 | Loss: 2.5480 | LR: 1.20e-05 | 3199.3ms/step | 18.8 samples/s +Step 45550/50462 | Loss: 2.5075 | LR: 1.19e-05 | 3196.3ms/step | 18.8 samples/s +Step 45560/50462 | Loss: 2.5325 | LR: 1.19e-05 | 3199.8ms/step | 18.8 samples/s +Step 45570/50462 | Loss: 2.5364 | LR: 1.19e-05 | 3196.1ms/step | 18.8 samples/s +Step 45580/50462 | Loss: 2.5141 | LR: 1.18e-05 | 3195.8ms/step | 18.8 samples/s +Step 45590/50462 | Loss: 2.5028 | LR: 1.18e-05 | 3197.2ms/step | 18.8 samples/s +Step 45600/50462 | Loss: 2.5115 | LR: 1.17e-05 | 3197.2ms/step | 18.8 samples/s +Step 45610/50462 | Loss: 2.4895 | LR: 1.17e-05 | 3252.1ms/step | 18.4 samples/s +Step 45620/50462 | Loss: 2.5220 | LR: 1.16e-05 | 3200.1ms/step | 18.7 samples/s +Step 45630/50462 | Loss: 2.5385 | LR: 1.16e-05 | 3195.5ms/step | 18.8 samples/s +Step 45640/50462 | Loss: 2.5167 | LR: 1.15e-05 | 3194.7ms/step | 18.8 samples/s +Step 45650/50462 | Loss: 2.5413 | LR: 1.15e-05 | 3191.1ms/step | 18.8 samples/s +Step 45660/50462 | Loss: 2.5258 | LR: 1.14e-05 | 3196.0ms/step | 18.8 samples/s +Step 45670/50462 | Loss: 2.4934 | LR: 1.14e-05 | 3198.8ms/step | 18.8 samples/s +Step 45680/50462 | Loss: 2.5133 | LR: 1.13e-05 | 3195.5ms/step | 18.8 samples/s +Step 45690/50462 | Loss: 2.5331 | LR: 1.13e-05 | 3249.8ms/step | 18.5 samples/s +Step 45700/50462 | Loss: 2.5263 | LR: 1.12e-05 | 3191.3ms/step | 18.8 samples/s +Step 45710/50462 | Loss: 2.5277 | LR: 1.12e-05 | 3249.3ms/step | 18.5 samples/s +Step 45720/50462 | Loss: 2.5194 | LR: 1.11e-05 | 3191.3ms/step | 18.8 samples/s +Step 45730/50462 | Loss: 2.5018 | LR: 1.11e-05 | 3200.3ms/step | 18.7 samples/s +Step 45740/50462 | Loss: 2.5138 | LR: 1.10e-05 | 3196.2ms/step | 18.8 samples/s +Step 45750/50462 | Loss: 2.5330 | LR: 1.10e-05 | 3199.6ms/step | 18.8 samples/s +Validation at step 45750: Loss = 2.5335, Perplexity = 12.60 +Step 45760/50462 | Loss: 2.4866 | LR: 1.10e-05 | 4870.6ms/step | 12.3 samples/s +Step 45770/50462 | Loss: 2.5189 | LR: 1.09e-05 | 3199.5ms/step | 18.8 samples/s +Step 45780/50462 | Loss: 2.5132 | LR: 1.09e-05 | 3197.7ms/step | 18.8 samples/s +Step 45790/50462 | Loss: 2.5486 | LR: 1.08e-05 | 3193.9ms/step | 18.8 samples/s +Step 45800/50462 | Loss: 2.4639 | LR: 1.08e-05 | 3194.6ms/step | 18.8 samples/s +Step 45810/50462 | Loss: 2.5244 | LR: 1.07e-05 | 3196.1ms/step | 18.8 samples/s +Step 45820/50462 | Loss: 2.5201 | LR: 1.07e-05 | 3198.0ms/step | 18.8 samples/s +Step 45830/50462 | Loss: 2.5271 | LR: 1.06e-05 | 3195.9ms/step | 18.8 samples/s +Step 45840/50462 | Loss: 2.5404 | LR: 1.06e-05 | 3195.9ms/step | 18.8 samples/s +Step 45850/50462 | Loss: 2.5221 | LR: 1.05e-05 | 3194.9ms/step | 18.8 samples/s +Step 45860/50462 | Loss: 2.5476 | LR: 1.05e-05 | 3193.4ms/step | 18.8 samples/s +Step 45870/50462 | Loss: 2.5319 | LR: 1.05e-05 | 3195.6ms/step | 18.8 samples/s +Step 45880/50462 | Loss: 2.5326 | LR: 1.04e-05 | 3198.3ms/step | 18.8 samples/s +Step 45890/50462 | Loss: 2.4952 | LR: 1.04e-05 | 3196.8ms/step | 18.8 samples/s +Step 45900/50462 | Loss: 2.5190 | LR: 1.03e-05 | 3199.0ms/step | 18.8 samples/s +Step 45910/50462 | Loss: 2.5482 | LR: 1.03e-05 | 3194.9ms/step | 18.8 samples/s +Step 45920/50462 | Loss: 2.5010 | LR: 1.02e-05 | 3197.5ms/step | 18.8 samples/s +Step 45930/50462 | Loss: 2.5131 | LR: 1.02e-05 | 3197.2ms/step | 18.8 samples/s +Step 45940/50462 | Loss: 2.5320 | LR: 1.01e-05 | 3196.0ms/step | 18.8 samples/s +Step 45950/50462 | Loss: 2.5196 | LR: 1.01e-05 | 3252.6ms/step | 18.4 samples/s +Step 45960/50462 | Loss: 2.5049 | LR: 1.01e-05 | 3199.5ms/step | 18.8 samples/s +Step 45970/50462 | Loss: 2.5108 | LR: 1.00e-05 | 3198.5ms/step | 18.8 samples/s +Step 45980/50462 | Loss: 2.5546 | LR: 9.96e-06 | 3196.0ms/step | 18.8 samples/s +Step 45990/50462 | Loss: 2.5068 | LR: 9.92e-06 | 3195.4ms/step | 18.8 samples/s +Step 46000/50462 | Loss: 2.5229 | LR: 9.87e-06 | 3196.1ms/step | 18.8 samples/s +Validation at step 46000: Loss = 2.5335, Perplexity = 12.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-46000 +Step 46010/50462 | Loss: 2.5259 | LR: 9.83e-06 | 5772.5ms/step | 10.4 samples/s +Step 46020/50462 | Loss: 2.5131 | LR: 9.79e-06 | 3178.0ms/step | 18.9 samples/s +Step 46030/50462 | Loss: 2.5353 | LR: 9.74e-06 | 3239.7ms/step | 18.5 samples/s +Step 46040/50462 | Loss: 2.5352 | LR: 9.70e-06 | 3174.5ms/step | 18.9 samples/s +Step 46050/50462 | Loss: 2.5184 | LR: 9.65e-06 | 3235.5ms/step | 18.5 samples/s +Step 46060/50462 | Loss: 2.5135 | LR: 9.61e-06 | 3192.4ms/step | 18.8 samples/s +Step 46070/50462 | Loss: 2.5327 | LR: 9.57e-06 | 3189.9ms/step | 18.8 samples/s +Step 46080/50462 | Loss: 2.5277 | LR: 9.52e-06 | 3197.6ms/step | 18.8 samples/s +Step 46090/50462 | Loss: 2.4969 | LR: 9.48e-06 | 3194.3ms/step | 18.8 samples/s +Step 46100/50462 | Loss: 2.5241 | LR: 9.44e-06 | 3193.8ms/step | 18.8 samples/s +Step 46110/50462 | Loss: 2.5192 | LR: 9.40e-06 | 3194.3ms/step | 18.8 samples/s +Step 46120/50462 | Loss: 2.5143 | LR: 9.35e-06 | 3194.8ms/step | 18.8 samples/s +Step 46130/50462 | Loss: 2.5160 | LR: 9.31e-06 | 3197.1ms/step | 18.8 samples/s +Step 46140/50462 | Loss: 2.5160 | LR: 9.27e-06 | 3194.9ms/step | 18.8 samples/s +Step 46150/50462 | Loss: 2.5135 | LR: 9.22e-06 | 3197.2ms/step | 18.8 samples/s +Step 46160/50462 | Loss: 2.5326 | LR: 9.18e-06 | 3197.3ms/step | 18.8 samples/s +Step 46170/50462 | Loss: 2.5321 | LR: 9.14e-06 | 3198.4ms/step | 18.8 samples/s +Step 46180/50462 | Loss: 2.5024 | LR: 9.10e-06 | 3198.2ms/step | 18.8 samples/s +Step 46190/50462 | Loss: 2.5027 | LR: 9.06e-06 | 3194.9ms/step | 18.8 samples/s +Step 46200/50462 | Loss: 2.5098 | LR: 9.01e-06 | 3197.1ms/step | 18.8 samples/s +Step 46210/50462 | Loss: 2.5273 | LR: 8.97e-06 | 3197.7ms/step | 18.8 samples/s +Step 46220/50462 | Loss: 2.5300 | LR: 8.93e-06 | 3194.5ms/step | 18.8 samples/s +Step 46230/50462 | Loss: 2.5350 | LR: 8.89e-06 | 3196.0ms/step | 18.8 samples/s +Step 46240/50462 | Loss: 2.5442 | LR: 8.85e-06 | 3193.4ms/step | 18.8 samples/s +Step 46250/50462 | Loss: 2.5420 | LR: 8.80e-06 | 3194.0ms/step | 18.8 samples/s +Validation at step 46250: Loss = 2.5335, Perplexity = 12.60 +Step 46260/50462 | Loss: 2.5413 | LR: 8.76e-06 | 4868.6ms/step | 12.3 samples/s +Step 46270/50462 | Loss: 2.5287 | LR: 8.72e-06 | 3195.9ms/step | 18.8 samples/s +Step 46280/50462 | Loss: 2.5600 | LR: 8.68e-06 | 3247.3ms/step | 18.5 samples/s +Step 46290/50462 | Loss: 2.5470 | LR: 8.64e-06 | 3187.3ms/step | 18.8 samples/s +Step 46300/50462 | Loss: 2.5167 | LR: 8.60e-06 | 3188.8ms/step | 18.8 samples/s +Step 46310/50462 | Loss: 2.5248 | LR: 8.56e-06 | 3189.2ms/step | 18.8 samples/s +Step 46320/50462 | Loss: 2.5495 | LR: 8.52e-06 | 3191.6ms/step | 18.8 samples/s +Step 46330/50462 | Loss: 2.5300 | LR: 8.47e-06 | 3199.9ms/step | 18.8 samples/s +Step 46340/50462 | Loss: 2.5292 | LR: 8.43e-06 | 3193.8ms/step | 18.8 samples/s +Step 46350/50462 | Loss: 2.5503 | LR: 8.39e-06 | 3195.9ms/step | 18.8 samples/s +Step 46360/50462 | Loss: 2.5141 | LR: 8.35e-06 | 3194.0ms/step | 18.8 samples/s +Step 46370/50462 | Loss: 2.4976 | LR: 8.31e-06 | 3249.4ms/step | 18.5 samples/s +Step 46380/50462 | Loss: 2.5236 | LR: 8.27e-06 | 3198.6ms/step | 18.8 samples/s +Step 46390/50462 | Loss: 2.5129 | LR: 8.23e-06 | 3253.9ms/step | 18.4 samples/s +Step 46400/50462 | Loss: 2.5052 | LR: 8.19e-06 | 3195.5ms/step | 18.8 samples/s +Step 46410/50462 | Loss: 2.5172 | LR: 8.15e-06 | 3194.6ms/step | 18.8 samples/s +Step 46420/50462 | Loss: 2.5167 | LR: 8.11e-06 | 3187.4ms/step | 18.8 samples/s +Step 46430/50462 | Loss: 2.5372 | LR: 8.07e-06 | 3193.9ms/step | 18.8 samples/s +Step 46440/50462 | Loss: 2.5222 | LR: 8.03e-06 | 3194.8ms/step | 18.8 samples/s +Step 46450/50462 | Loss: 2.5461 | LR: 7.99e-06 | 3194.2ms/step | 18.8 samples/s +Step 46460/50462 | Loss: 2.5253 | LR: 7.95e-06 | 3190.7ms/step | 18.8 samples/s +Step 46470/50462 | Loss: 2.5159 | LR: 7.91e-06 | 3194.8ms/step | 18.8 samples/s +Step 46480/50462 | Loss: 2.5113 | LR: 7.87e-06 | 3198.3ms/step | 18.8 samples/s +Step 46490/50462 | Loss: 2.5065 | LR: 7.83e-06 | 3194.8ms/step | 18.8 samples/s +Step 46500/50462 | Loss: 2.5247 | LR: 7.80e-06 | 3193.1ms/step | 18.8 samples/s +Validation at step 46500: Loss = 2.5335, Perplexity = 12.60 +Step 46510/50462 | Loss: 2.5175 | LR: 7.76e-06 | 4865.6ms/step | 12.3 samples/s +Step 46520/50462 | Loss: 2.5413 | LR: 7.72e-06 | 3190.3ms/step | 18.8 samples/s +Step 46530/50462 | Loss: 2.5220 | LR: 7.68e-06 | 3194.1ms/step | 18.8 samples/s +Step 46540/50462 | Loss: 2.5351 | LR: 7.64e-06 | 3198.3ms/step | 18.8 samples/s +Step 46550/50462 | Loss: 2.4963 | LR: 7.60e-06 | 3199.9ms/step | 18.8 samples/s +Step 46560/50462 | Loss: 2.5122 | LR: 7.56e-06 | 3195.9ms/step | 18.8 samples/s +Step 46570/50462 | Loss: 2.5238 | LR: 7.52e-06 | 3193.5ms/step | 18.8 samples/s +Step 46580/50462 | Loss: 2.5150 | LR: 7.49e-06 | 3190.1ms/step | 18.8 samples/s +Step 46590/50462 | Loss: 2.5084 | LR: 7.45e-06 | 3192.4ms/step | 18.8 samples/s +Step 46600/50462 | Loss: 2.5219 | LR: 7.41e-06 | 3194.0ms/step | 18.8 samples/s +Step 46610/50462 | Loss: 2.5215 | LR: 7.37e-06 | 3198.0ms/step | 18.8 samples/s +Step 46620/50462 | Loss: 2.5042 | LR: 7.33e-06 | 3254.1ms/step | 18.4 samples/s +Step 46630/50462 | Loss: 2.5185 | LR: 7.29e-06 | 3198.2ms/step | 18.8 samples/s +Step 46640/50462 | Loss: 2.5302 | LR: 7.26e-06 | 3196.6ms/step | 18.8 samples/s +Step 46650/50462 | Loss: 2.5244 | LR: 7.22e-06 | 3196.5ms/step | 18.8 samples/s +Step 46660/50462 | Loss: 2.5137 | LR: 7.18e-06 | 3195.9ms/step | 18.8 samples/s +Step 46670/50462 | Loss: 2.5285 | LR: 7.14e-06 | 3192.9ms/step | 18.8 samples/s +Step 46680/50462 | Loss: 2.5363 | LR: 7.11e-06 | 3199.4ms/step | 18.8 samples/s +Step 46690/50462 | Loss: 2.5524 | LR: 7.07e-06 | 3195.6ms/step | 18.8 samples/s +Step 46700/50462 | Loss: 2.5131 | LR: 7.03e-06 | 3192.6ms/step | 18.8 samples/s +Step 46710/50462 | Loss: 2.5469 | LR: 6.99e-06 | 3243.0ms/step | 18.5 samples/s +Step 46720/50462 | Loss: 2.5297 | LR: 6.96e-06 | 3187.3ms/step | 18.8 samples/s +Step 46730/50462 | Loss: 2.5093 | LR: 6.92e-06 | 3239.3ms/step | 18.5 samples/s +Step 46740/50462 | Loss: 2.5367 | LR: 6.88e-06 | 3182.3ms/step | 18.9 samples/s +Step 46750/50462 | Loss: 2.5183 | LR: 6.85e-06 | 3186.3ms/step | 18.8 samples/s +Validation at step 46750: Loss = 2.5335, Perplexity = 12.60 +Step 46760/50462 | Loss: 2.5337 | LR: 6.81e-06 | 4861.7ms/step | 12.3 samples/s +Step 46770/50462 | Loss: 2.5234 | LR: 6.77e-06 | 3187.1ms/step | 18.8 samples/s +Step 46780/50462 | Loss: 2.5485 | LR: 6.74e-06 | 3192.4ms/step | 18.8 samples/s +Step 46790/50462 | Loss: 2.5056 | LR: 6.70e-06 | 3190.7ms/step | 18.8 samples/s +Step 46800/50462 | Loss: 2.5436 | LR: 6.66e-06 | 3187.4ms/step | 18.8 samples/s +Step 46810/50462 | Loss: 2.5398 | LR: 6.63e-06 | 3189.2ms/step | 18.8 samples/s +Step 46820/50462 | Loss: 2.5029 | LR: 6.59e-06 | 3191.3ms/step | 18.8 samples/s +Step 46830/50462 | Loss: 2.5020 | LR: 6.56e-06 | 3192.4ms/step | 18.8 samples/s +Step 46840/50462 | Loss: 2.5272 | LR: 6.52e-06 | 3199.0ms/step | 18.8 samples/s +Step 46850/50462 | Loss: 2.5227 | LR: 6.48e-06 | 3190.5ms/step | 18.8 samples/s +Step 46860/50462 | Loss: 2.5218 | LR: 6.45e-06 | 3187.9ms/step | 18.8 samples/s +Step 46870/50462 | Loss: 2.4818 | LR: 6.41e-06 | 3187.1ms/step | 18.8 samples/s +Step 46880/50462 | Loss: 2.5161 | LR: 6.38e-06 | 3195.6ms/step | 18.8 samples/s +Step 46890/50462 | Loss: 2.5221 | LR: 6.34e-06 | 3190.4ms/step | 18.8 samples/s +Step 46900/50462 | Loss: 2.5148 | LR: 6.31e-06 | 3190.6ms/step | 18.8 samples/s +Step 46910/50462 | Loss: 2.5059 | LR: 6.27e-06 | 3187.8ms/step | 18.8 samples/s +Step 46920/50462 | Loss: 2.5465 | LR: 6.24e-06 | 3192.7ms/step | 18.8 samples/s +Step 46930/50462 | Loss: 2.5213 | LR: 6.20e-06 | 3190.7ms/step | 18.8 samples/s +Step 46940/50462 | Loss: 2.5238 | LR: 6.17e-06 | 3190.2ms/step | 18.8 samples/s +Step 46950/50462 | Loss: 2.5070 | LR: 6.13e-06 | 3189.2ms/step | 18.8 samples/s +Step 46960/50462 | Loss: 2.5302 | LR: 6.10e-06 | 3244.2ms/step | 18.5 samples/s +Step 46970/50462 | Loss: 2.5337 | LR: 6.06e-06 | 3196.0ms/step | 18.8 samples/s +Step 46980/50462 | Loss: 2.5324 | LR: 6.03e-06 | 3198.0ms/step | 18.8 samples/s +Step 46990/50462 | Loss: 2.5530 | LR: 5.99e-06 | 3195.1ms/step | 18.8 samples/s +Step 47000/50462 | Loss: 2.5334 | LR: 5.96e-06 | 3198.1ms/step | 18.8 samples/s +Validation at step 47000: Loss = 2.5335, Perplexity = 12.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-47000 diff --git a/epfl_final/log_files/another_sparse_33%.log b/epfl_final/log_files/another_sparse_33%.log new file mode 100644 index 0000000000000000000000000000000000000000..07b014c28992f68932f3d183c5fa8f0af86a508c --- /dev/null +++ b/epfl_final/log_files/another_sparse_33%.log @@ -0,0 +1,552 @@ +New run: Checkpoints will be saved to ./checkpoints/run_20260506_171628 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/the_pile_train/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260506_171628", + "resume_from_checkpoint": null, + "num_epochs": 1, + "max_steps": 20005, + "per_device_batch_size": 3, + "gradient_accumulation_steps": 8, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 250, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 2000, + "eval_steps": 250, + "eval_batches": 200, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": null, + "wandb_tags": [ + "qwen3", + "0.6b" + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 20, + "sparse_mask_pool_size": 1000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run nmzybrc7 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260506_173734-nmzybrc7 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run exalted-wave-16 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/nmzybrc7 + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1024 + num_hidden_layers: 28 + num_attention_heads: 16 + num_key_value_heads: 8 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 20 + sparse_mask_pool_size: 1000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 596,049,920 + Trainable parameters: 596,049,920 + Non-trainable parameters: 0 + Model size (BF16): 1.19 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [3, 8192] -- -- True +├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True +│ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True +├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True +================================================================================================================================================================ +Total params: 751,632,384 +Trainable params: 751,632,384 +Non-trainable params: 0 +Total mult-adds (G): 2.25 +================================================================================================================================================================ +Input size (MB): 0.20 +Forward/backward pass size (MB): 63052.97 +Params size (MB): 1503.26 +Estimated Total Size (MB): 64556.43 +================================================================================================================================================================ +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized... +Loaded 5315146 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1328786) +Dataset split: 1063028 train, 265758 validation +Training examples (packed, train split): 1063028 +Validation examples (packed, val split): 265758 +Per-device batch size: 3 +World size (GPUs): 1 +Gradient accumulation steps: 8 +Effective batch size (seqs/step): 24 +Sequence length: 8192 +Tokens per optimizer step: 196,608 +Micro-batches per epoch (per rank): 354342 +Optimizer steps per epoch: 44292 +Total optimizer steps: 20005 (capped by max_steps; 0.45 epochs) +Starting training... +Step 10/20005 | Loss: 12.0953 | LR: 2.00e-05 | 9747.4ms/step | 2.5 samples/s +Step 20/20005 | Loss: 11.5112 | LR: 4.00e-05 | 2455.1ms/step | 9.8 samples/s +Step 30/20005 | Loss: 10.6615 | LR: 6.00e-05 | 2445.4ms/step | 9.8 samples/s +Step 40/20005 | Loss: 10.1898 | LR: 8.00e-05 | 2439.2ms/step | 9.8 samples/s +Step 50/20005 | Loss: 9.3633 | LR: 1.00e-04 | 2437.9ms/step | 9.8 samples/s +Step 60/20005 | Loss: 8.6099 | LR: 1.20e-04 | 2437.9ms/step | 9.8 samples/s +Step 70/20005 | Loss: 8.0160 | LR: 1.40e-04 | 2443.9ms/step | 9.8 samples/s +Step 80/20005 | Loss: 7.6071 | LR: 1.60e-04 | 2450.5ms/step | 9.8 samples/s +Step 90/20005 | Loss: 7.6196 | LR: 1.80e-04 | 2459.7ms/step | 9.8 samples/s +Step 100/20005 | Loss: 7.3453 | LR: 2.00e-04 | 2461.2ms/step | 9.8 samples/s +Step 110/20005 | Loss: 7.2536 | LR: 2.20e-04 | 2465.1ms/step | 9.7 samples/s +Step 120/20005 | Loss: 7.1495 | LR: 2.40e-04 | 2464.9ms/step | 9.7 samples/s +Step 130/20005 | Loss: 6.9748 | LR: 2.60e-04 | 2466.5ms/step | 9.7 samples/s +Step 140/20005 | Loss: 6.8463 | LR: 2.80e-04 | 2472.3ms/step | 9.7 samples/s +Step 150/20005 | Loss: 6.6894 | LR: 3.00e-04 | 2469.1ms/step | 9.7 samples/s +Step 160/20005 | Loss: 6.5733 | LR: 3.20e-04 | 2472.4ms/step | 9.7 samples/s +Step 170/20005 | Loss: 6.5434 | LR: 3.40e-04 | 2471.1ms/step | 9.7 samples/s +Step 180/20005 | Loss: 6.4437 | LR: 3.60e-04 | 2469.6ms/step | 9.7 samples/s +Step 190/20005 | Loss: 6.2488 | LR: 3.80e-04 | 2469.1ms/step | 9.7 samples/s +Step 200/20005 | Loss: 6.3373 | LR: 4.00e-04 | 2470.4ms/step | 9.7 samples/s +Step 210/20005 | Loss: 6.1415 | LR: 4.20e-04 | 2472.3ms/step | 9.7 samples/s +Step 220/20005 | Loss: 6.0566 | LR: 4.40e-04 | 2472.8ms/step | 9.7 samples/s +Step 230/20005 | Loss: 6.0284 | LR: 4.60e-04 | 2472.8ms/step | 9.7 samples/s +Step 240/20005 | Loss: 6.0281 | LR: 4.80e-04 | 2472.8ms/step | 9.7 samples/s +Step 250/20005 | Loss: 5.8878 | LR: 5.00e-04 | 2474.1ms/step | 9.7 samples/s +Validation at step 250: Loss = 5.9092, Perplexity = 368.41 +Step 260/20005 | Loss: 5.8341 | LR: 5.00e-04 | 23166.1ms/step | 1.0 samples/s +Step 270/20005 | Loss: 5.7961 | LR: 5.00e-04 | 2467.7ms/step | 9.7 samples/s +Step 280/20005 | Loss: 5.8218 | LR: 5.00e-04 | 2469.0ms/step | 9.7 samples/s +Step 290/20005 | Loss: 5.7036 | LR: 5.00e-04 | 2469.6ms/step | 9.7 samples/s +Step 300/20005 | Loss: 5.6739 | LR: 5.00e-04 | 2469.6ms/step | 9.7 samples/s +Step 310/20005 | Loss: 5.6067 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s +Step 320/20005 | Loss: 5.5022 | LR: 5.00e-04 | 2472.2ms/step | 9.7 samples/s +Step 330/20005 | Loss: 5.4496 | LR: 5.00e-04 | 2469.2ms/step | 9.7 samples/s +Step 340/20005 | Loss: 5.2664 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s +Step 350/20005 | Loss: 5.2576 | LR: 5.00e-04 | 2469.4ms/step | 9.7 samples/s +Step 360/20005 | Loss: 5.3474 | LR: 5.00e-04 | 2468.0ms/step | 9.7 samples/s +Step 370/20005 | Loss: 5.0944 | LR: 5.00e-04 | 2468.5ms/step | 9.7 samples/s +Step 380/20005 | Loss: 5.1576 | LR: 5.00e-04 | 2473.3ms/step | 9.7 samples/s +Step 390/20005 | Loss: 5.1447 | LR: 5.00e-04 | 2473.1ms/step | 9.7 samples/s +Step 400/20005 | Loss: 5.0084 | LR: 5.00e-04 | 2475.2ms/step | 9.7 samples/s +Step 410/20005 | Loss: 5.0430 | LR: 5.00e-04 | 2472.0ms/step | 9.7 samples/s +Step 420/20005 | Loss: 4.8453 | LR: 5.00e-04 | 2470.4ms/step | 9.7 samples/s +Step 430/20005 | Loss: 4.9373 | LR: 5.00e-04 | 2472.6ms/step | 9.7 samples/s +Step 440/20005 | Loss: 4.7404 | LR: 5.00e-04 | 2473.2ms/step | 9.7 samples/s +Step 450/20005 | Loss: 4.8040 | LR: 5.00e-04 | 2470.3ms/step | 9.7 samples/s +Step 460/20005 | Loss: 4.8016 | LR: 5.00e-04 | 2468.5ms/step | 9.7 samples/s +Step 470/20005 | Loss: 4.7038 | LR: 5.00e-04 | 2470.9ms/step | 9.7 samples/s +Step 480/20005 | Loss: 4.6288 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s +Step 490/20005 | Loss: 4.6210 | LR: 5.00e-04 | 2470.4ms/step | 9.7 samples/s +Step 500/20005 | Loss: 4.5110 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s +Validation at step 500: Loss = 4.5643, Perplexity = 95.99 +Step 510/20005 | Loss: 4.6094 | LR: 5.00e-04 | 20008.5ms/step | 1.2 samples/s +Step 520/20005 | Loss: 4.5267 | LR: 5.00e-04 | 2473.6ms/step | 9.7 samples/s +Step 530/20005 | Loss: 4.4512 | LR: 5.00e-04 | 2471.4ms/step | 9.7 samples/s +Step 540/20005 | Loss: 4.4637 | LR: 5.00e-04 | 2470.1ms/step | 9.7 samples/s +Step 550/20005 | Loss: 4.3918 | LR: 5.00e-04 | 2469.5ms/step | 9.7 samples/s +Step 560/20005 | Loss: 4.4136 | LR: 5.00e-04 | 2471.2ms/step | 9.7 samples/s +Step 570/20005 | Loss: 4.3111 | LR: 5.00e-04 | 2470.1ms/step | 9.7 samples/s +Step 580/20005 | Loss: 4.2604 | LR: 5.00e-04 | 2466.7ms/step | 9.7 samples/s +Step 590/20005 | Loss: 4.2515 | LR: 5.00e-04 | 2471.6ms/step | 9.7 samples/s +Step 600/20005 | Loss: 4.2258 | LR: 5.00e-04 | 2472.6ms/step | 9.7 samples/s +Step 610/20005 | Loss: 4.1722 | LR: 5.00e-04 | 2471.3ms/step | 9.7 samples/s +Step 620/20005 | Loss: 4.1427 | LR: 5.00e-04 | 2471.6ms/step | 9.7 samples/s +Step 630/20005 | Loss: 4.1107 | LR: 5.00e-04 | 2470.0ms/step | 9.7 samples/s +Step 640/20005 | Loss: 4.1992 | LR: 5.00e-04 | 2469.4ms/step | 9.7 samples/s +Step 650/20005 | Loss: 4.1674 | LR: 4.99e-04 | 2472.4ms/step | 9.7 samples/s +Step 660/20005 | Loss: 4.1263 | LR: 4.99e-04 | 2471.2ms/step | 9.7 samples/s +Step 670/20005 | Loss: 4.0386 | LR: 4.99e-04 | 2474.5ms/step | 9.7 samples/s +Step 680/20005 | Loss: 4.0233 | LR: 4.99e-04 | 2472.6ms/step | 9.7 samples/s +Step 690/20005 | Loss: 3.9074 | LR: 4.99e-04 | 2467.9ms/step | 9.7 samples/s +Step 700/20005 | Loss: 3.9711 | LR: 4.99e-04 | 2467.3ms/step | 9.7 samples/s +Step 710/20005 | Loss: 4.0219 | LR: 4.99e-04 | 2472.1ms/step | 9.7 samples/s +Step 720/20005 | Loss: 3.9824 | LR: 4.99e-04 | 2474.1ms/step | 9.7 samples/s +Step 730/20005 | Loss: 3.9286 | LR: 4.99e-04 | 2473.0ms/step | 9.7 samples/s +Step 740/20005 | Loss: 3.8975 | LR: 4.99e-04 | 2475.8ms/step | 9.7 samples/s +Step 750/20005 | Loss: 3.8868 | LR: 4.99e-04 | 2472.0ms/step | 9.7 samples/s +Validation at step 750: Loss = 3.9122, Perplexity = 50.01 +Step 760/20005 | Loss: 3.9005 | LR: 4.99e-04 | 19998.9ms/step | 1.2 samples/s +Step 770/20005 | Loss: 3.8754 | LR: 4.99e-04 | 2471.5ms/step | 9.7 samples/s +Step 780/20005 | Loss: 3.9505 | LR: 4.99e-04 | 2470.8ms/step | 9.7 samples/s +Step 790/20005 | Loss: 3.8299 | LR: 4.99e-04 | 2472.9ms/step | 9.7 samples/s +Step 800/20005 | Loss: 3.7659 | LR: 4.99e-04 | 2471.3ms/step | 9.7 samples/s +Step 810/20005 | Loss: 3.8095 | LR: 4.99e-04 | 2469.0ms/step | 9.7 samples/s +Step 820/20005 | Loss: 3.8006 | LR: 4.99e-04 | 2470.9ms/step | 9.7 samples/s +Step 830/20005 | Loss: 3.8261 | LR: 4.99e-04 | 2472.9ms/step | 9.7 samples/s +Step 840/20005 | Loss: 3.8026 | LR: 4.99e-04 | 2469.8ms/step | 9.7 samples/s +Step 850/20005 | Loss: 3.7642 | LR: 4.99e-04 | 2472.0ms/step | 9.7 samples/s +Step 860/20005 | Loss: 3.7994 | LR: 4.99e-04 | 2472.7ms/step | 9.7 samples/s +Step 870/20005 | Loss: 3.7685 | LR: 4.99e-04 | 2473.4ms/step | 9.7 samples/s +Step 880/20005 | Loss: 3.7097 | LR: 4.99e-04 | 2470.1ms/step | 9.7 samples/s +Step 890/20005 | Loss: 3.6376 | LR: 4.99e-04 | 2468.6ms/step | 9.7 samples/s +Step 900/20005 | Loss: 3.7063 | LR: 4.99e-04 | 2474.3ms/step | 9.7 samples/s +Step 910/20005 | Loss: 3.6934 | LR: 4.99e-04 | 2471.7ms/step | 9.7 samples/s +Step 920/20005 | Loss: 3.6495 | LR: 4.99e-04 | 2470.7ms/step | 9.7 samples/s +Step 930/20005 | Loss: 3.5923 | LR: 4.99e-04 | 2471.9ms/step | 9.7 samples/s +Step 940/20005 | Loss: 3.7267 | LR: 4.98e-04 | 2470.2ms/step | 9.7 samples/s +Step 950/20005 | Loss: 3.5760 | LR: 4.98e-04 | 2468.5ms/step | 9.7 samples/s +Step 960/20005 | Loss: 3.6459 | LR: 4.98e-04 | 2469.6ms/step | 9.7 samples/s +Step 970/20005 | Loss: 3.5800 | LR: 4.98e-04 | 2471.9ms/step | 9.7 samples/s +Step 980/20005 | Loss: 3.5983 | LR: 4.98e-04 | 2475.0ms/step | 9.7 samples/s +Step 990/20005 | Loss: 3.6347 | LR: 4.98e-04 | 2474.8ms/step | 9.7 samples/s +Step 1000/20005 | Loss: 3.5700 | LR: 4.98e-04 | 2472.7ms/step | 9.7 samples/s +Validation at step 1000: Loss = 3.6054, Perplexity = 36.79 +Step 1010/20005 | Loss: 3.5783 | LR: 4.98e-04 | 20042.8ms/step | 1.2 samples/s +Step 1020/20005 | Loss: 3.5671 | LR: 4.98e-04 | 2473.3ms/step | 9.7 samples/s +Step 1030/20005 | Loss: 3.5786 | LR: 4.98e-04 | 2470.0ms/step | 9.7 samples/s +Step 1040/20005 | Loss: 3.6036 | LR: 4.98e-04 | 2572.7ms/step | 9.3 samples/s +Step 1050/20005 | Loss: 3.4463 | LR: 4.98e-04 | 2481.5ms/step | 9.7 samples/s +Step 1060/20005 | Loss: 3.5629 | LR: 4.98e-04 | 2484.6ms/step | 9.7 samples/s +Step 1070/20005 | Loss: 3.5151 | LR: 4.98e-04 | 2482.3ms/step | 9.7 samples/s +Step 1080/20005 | Loss: 3.6346 | LR: 4.98e-04 | 2482.4ms/step | 9.7 samples/s +Step 1090/20005 | Loss: 3.5463 | LR: 4.98e-04 | 2483.7ms/step | 9.7 samples/s +Step 1100/20005 | Loss: 3.4785 | LR: 4.98e-04 | 2485.4ms/step | 9.7 samples/s +Step 1110/20005 | Loss: 3.5641 | LR: 4.98e-04 | 2482.7ms/step | 9.7 samples/s +Step 1120/20005 | Loss: 3.5929 | LR: 4.98e-04 | 2481.2ms/step | 9.7 samples/s +Step 1130/20005 | Loss: 3.5342 | LR: 4.98e-04 | 2480.7ms/step | 9.7 samples/s +Step 1140/20005 | Loss: 3.5100 | LR: 4.98e-04 | 2483.6ms/step | 9.7 samples/s +Step 1150/20005 | Loss: 3.5179 | LR: 4.97e-04 | 2488.2ms/step | 9.6 samples/s +Step 1160/20005 | Loss: 3.4899 | LR: 4.97e-04 | 2485.5ms/step | 9.7 samples/s +Step 1170/20005 | Loss: 3.5013 | LR: 4.97e-04 | 2483.8ms/step | 9.7 samples/s +Step 1180/20005 | Loss: 3.4978 | LR: 4.97e-04 | 2482.7ms/step | 9.7 samples/s +Step 1190/20005 | Loss: 3.5004 | LR: 4.97e-04 | 2483.5ms/step | 9.7 samples/s +Step 1200/20005 | Loss: 3.4190 | LR: 4.97e-04 | 2484.0ms/step | 9.7 samples/s +Step 1210/20005 | Loss: 3.3502 | LR: 4.97e-04 | 2484.1ms/step | 9.7 samples/s +Step 1220/20005 | Loss: 3.5036 | LR: 4.97e-04 | 2483.9ms/step | 9.7 samples/s +Step 1230/20005 | Loss: 3.4153 | LR: 4.97e-04 | 2481.5ms/step | 9.7 samples/s +Step 1240/20005 | Loss: 3.5046 | LR: 4.97e-04 | 2480.7ms/step | 9.7 samples/s +Step 1250/20005 | Loss: 3.3397 | LR: 4.97e-04 | 2482.6ms/step | 9.7 samples/s +Validation at step 1250: Loss = 3.4208, Perplexity = 30.60 +Step 1260/20005 | Loss: 3.3878 | LR: 4.97e-04 | 20092.1ms/step | 1.2 samples/s +Step 1270/20005 | Loss: 3.4112 | LR: 4.97e-04 | 2480.6ms/step | 9.7 samples/s +Step 1280/20005 | Loss: 3.3534 | LR: 4.97e-04 | 2481.9ms/step | 9.7 samples/s +Step 1290/20005 | Loss: 3.4114 | LR: 4.97e-04 | 2482.4ms/step | 9.7 samples/s +Step 1300/20005 | Loss: 3.3480 | LR: 4.97e-04 | 2481.8ms/step | 9.7 samples/s +Step 1310/20005 | Loss: 3.3368 | LR: 4.96e-04 | 2481.3ms/step | 9.7 samples/s +Step 1320/20005 | Loss: 3.3049 | LR: 4.96e-04 | 2482.6ms/step | 9.7 samples/s +Step 1330/20005 | Loss: 3.4321 | LR: 4.96e-04 | 2480.4ms/step | 9.7 samples/s +Step 1340/20005 | Loss: 3.3693 | LR: 4.96e-04 | 2480.2ms/step | 9.7 samples/s +Step 1350/20005 | Loss: 3.3928 | LR: 4.96e-04 | 2482.5ms/step | 9.7 samples/s +Step 1360/20005 | Loss: 3.3521 | LR: 4.96e-04 | 2482.5ms/step | 9.7 samples/s +Step 1370/20005 | Loss: 3.3616 | LR: 4.96e-04 | 2481.4ms/step | 9.7 samples/s +Step 1380/20005 | Loss: 3.3294 | LR: 4.96e-04 | 2485.7ms/step | 9.7 samples/s +Step 1390/20005 | Loss: 3.2962 | LR: 4.96e-04 | 2482.7ms/step | 9.7 samples/s +Step 1400/20005 | Loss: 3.3390 | LR: 4.96e-04 | 2482.3ms/step | 9.7 samples/s +Step 1410/20005 | Loss: 3.3442 | LR: 4.96e-04 | 2478.7ms/step | 9.7 samples/s +Step 1420/20005 | Loss: 3.3456 | LR: 4.96e-04 | 2480.5ms/step | 9.7 samples/s +Step 1430/20005 | Loss: 3.3379 | LR: 4.96e-04 | 2482.0ms/step | 9.7 samples/s +Step 1440/20005 | Loss: 3.3419 | LR: 4.96e-04 | 2478.3ms/step | 9.7 samples/s +Step 1450/20005 | Loss: 3.3965 | LR: 4.95e-04 | 2480.7ms/step | 9.7 samples/s +Step 1460/20005 | Loss: 3.3223 | LR: 4.95e-04 | 2479.9ms/step | 9.7 samples/s +Step 1470/20005 | Loss: 3.3176 | LR: 4.95e-04 | 2479.8ms/step | 9.7 samples/s +Step 1480/20005 | Loss: 3.3046 | LR: 4.95e-04 | 2480.6ms/step | 9.7 samples/s +Step 1490/20005 | Loss: 3.2875 | LR: 4.95e-04 | 2480.2ms/step | 9.7 samples/s +Step 1500/20005 | Loss: 3.3622 | LR: 4.95e-04 | 2481.7ms/step | 9.7 samples/s +Validation at step 1500: Loss = 3.3429, Perplexity = 28.30 +Step 1510/20005 | Loss: 3.2782 | LR: 4.95e-04 | 20084.7ms/step | 1.2 samples/s +Step 1520/20005 | Loss: 3.3071 | LR: 4.95e-04 | 2483.2ms/step | 9.7 samples/s +Step 1530/20005 | Loss: 3.3001 | LR: 4.95e-04 | 2481.1ms/step | 9.7 samples/s +Step 1540/20005 | Loss: 3.2394 | LR: 4.95e-04 | 2484.8ms/step | 9.7 samples/s +Step 1550/20005 | Loss: 3.3707 | LR: 4.95e-04 | 2484.1ms/step | 9.7 samples/s +Step 1560/20005 | Loss: 3.2867 | LR: 4.95e-04 | 2480.0ms/step | 9.7 samples/s +Step 1570/20005 | Loss: 3.2819 | LR: 4.95e-04 | 2481.8ms/step | 9.7 samples/s +Step 1580/20005 | Loss: 3.2988 | LR: 4.94e-04 | 2480.3ms/step | 9.7 samples/s +Step 1590/20005 | Loss: 3.2332 | LR: 4.94e-04 | 2480.0ms/step | 9.7 samples/s +Step 1600/20005 | Loss: 3.3096 | LR: 4.94e-04 | 2481.9ms/step | 9.7 samples/s +Step 1610/20005 | Loss: 3.2179 | LR: 4.94e-04 | 2484.5ms/step | 9.7 samples/s +Step 1620/20005 | Loss: 3.2699 | LR: 4.94e-04 | 2479.5ms/step | 9.7 samples/s +Step 1630/20005 | Loss: 3.2273 | LR: 4.94e-04 | 2482.2ms/step | 9.7 samples/s +Step 1640/20005 | Loss: 3.3416 | LR: 4.94e-04 | 2481.6ms/step | 9.7 samples/s +Step 1650/20005 | Loss: 3.3088 | LR: 4.94e-04 | 2483.9ms/step | 9.7 samples/s +Step 1660/20005 | Loss: 3.2767 | LR: 4.94e-04 | 2485.8ms/step | 9.7 samples/s +Step 1670/20005 | Loss: 3.1671 | LR: 4.94e-04 | 2485.2ms/step | 9.7 samples/s +Step 1680/20005 | Loss: 3.3102 | LR: 4.94e-04 | 2482.1ms/step | 9.7 samples/s +Step 1690/20005 | Loss: 3.1658 | LR: 4.93e-04 | 2480.9ms/step | 9.7 samples/s +Step 1700/20005 | Loss: 3.2453 | LR: 4.93e-04 | 2484.1ms/step | 9.7 samples/s +Step 1710/20005 | Loss: 3.1070 | LR: 4.93e-04 | 2483.1ms/step | 9.7 samples/s +Step 1720/20005 | Loss: 3.1972 | LR: 4.93e-04 | 2480.9ms/step | 9.7 samples/s +Step 1730/20005 | Loss: 3.2333 | LR: 4.93e-04 | 2480.2ms/step | 9.7 samples/s +Step 1740/20005 | Loss: 3.1906 | LR: 4.93e-04 | 2485.7ms/step | 9.7 samples/s +Step 1750/20005 | Loss: 3.2600 | LR: 4.93e-04 | 2484.0ms/step | 9.7 samples/s +Validation at step 1750: Loss = 3.2076, Perplexity = 24.72 +Step 1760/20005 | Loss: 3.2116 | LR: 4.93e-04 | 20073.0ms/step | 1.2 samples/s +Step 1770/20005 | Loss: 3.2311 | LR: 4.93e-04 | 2482.8ms/step | 9.7 samples/s +Step 1780/20005 | Loss: 3.2018 | LR: 4.93e-04 | 2480.3ms/step | 9.7 samples/s +Step 1790/20005 | Loss: 3.1756 | LR: 4.93e-04 | 2480.4ms/step | 9.7 samples/s +Step 1800/20005 | Loss: 3.1392 | LR: 4.92e-04 | 2482.7ms/step | 9.7 samples/s +Step 1810/20005 | Loss: 3.2799 | LR: 4.92e-04 | 2482.7ms/step | 9.7 samples/s +Step 1820/20005 | Loss: 3.1527 | LR: 4.92e-04 | 2483.2ms/step | 9.7 samples/s +Step 1830/20005 | Loss: 3.2052 | LR: 4.92e-04 | 2582.8ms/step | 9.3 samples/s +Step 1840/20005 | Loss: 3.1211 | LR: 4.92e-04 | 2482.1ms/step | 9.7 samples/s +Step 1850/20005 | Loss: 3.1366 | LR: 4.92e-04 | 2483.4ms/step | 9.7 samples/s +Step 1860/20005 | Loss: 3.1339 | LR: 4.92e-04 | 2485.6ms/step | 9.7 samples/s +Step 1870/20005 | Loss: 3.1281 | LR: 4.92e-04 | 2481.2ms/step | 9.7 samples/s +Step 1880/20005 | Loss: 3.1423 | LR: 4.92e-04 | 2480.9ms/step | 9.7 samples/s +Step 1890/20005 | Loss: 3.1088 | LR: 4.92e-04 | 2484.9ms/step | 9.7 samples/s +Step 1900/20005 | Loss: 3.2010 | LR: 4.91e-04 | 2480.0ms/step | 9.7 samples/s +Step 1910/20005 | Loss: 3.1371 | LR: 4.91e-04 | 2481.1ms/step | 9.7 samples/s +Step 1920/20005 | Loss: 3.1164 | LR: 4.91e-04 | 2482.7ms/step | 9.7 samples/s +Step 1930/20005 | Loss: 3.1960 | LR: 4.91e-04 | 2482.0ms/step | 9.7 samples/s +Step 1940/20005 | Loss: 3.0022 | LR: 4.91e-04 | 2482.1ms/step | 9.7 samples/s +Step 1950/20005 | Loss: 3.1884 | LR: 4.91e-04 | 2484.2ms/step | 9.7 samples/s +Step 1960/20005 | Loss: 3.1131 | LR: 4.91e-04 | 2481.6ms/step | 9.7 samples/s +Step 1970/20005 | Loss: 3.1066 | LR: 4.91e-04 | 2480.0ms/step | 9.7 samples/s +Step 1980/20005 | Loss: 3.0815 | LR: 4.91e-04 | 2481.2ms/step | 9.7 samples/s +Step 1990/20005 | Loss: 3.0854 | LR: 4.90e-04 | 2485.2ms/step | 9.7 samples/s +Step 2000/20005 | Loss: 3.1739 | LR: 4.90e-04 | 2483.9ms/step | 9.7 samples/s +Validation at step 2000: Loss = 3.1330, Perplexity = 22.94 +Saved checkpoint to checkpoints/run_20260506_171628/checkpoint-2000 +Step 2010/20005 | Loss: 3.1899 | LR: 4.90e-04 | 20595.1ms/step | 1.2 samples/s +Step 2020/20005 | Loss: 3.1322 | LR: 4.90e-04 | 2483.0ms/step | 9.7 samples/s +Step 2030/20005 | Loss: 3.1557 | LR: 4.90e-04 | 2483.2ms/step | 9.7 samples/s +Step 2040/20005 | Loss: 3.1301 | LR: 4.90e-04 | 2481.5ms/step | 9.7 samples/s +Step 2050/20005 | Loss: 3.0754 | LR: 4.90e-04 | 2484.8ms/step | 9.7 samples/s +Step 2060/20005 | Loss: 3.0958 | LR: 4.90e-04 | 2482.1ms/step | 9.7 samples/s +Step 2070/20005 | Loss: 3.1173 | LR: 4.90e-04 | 2482.0ms/step | 9.7 samples/s +Step 2080/20005 | Loss: 3.1208 | LR: 4.89e-04 | 2479.9ms/step | 9.7 samples/s +Step 2090/20005 | Loss: 3.0896 | LR: 4.89e-04 | 2479.4ms/step | 9.7 samples/s +Step 2100/20005 | Loss: 3.0643 | LR: 4.89e-04 | 2480.4ms/step | 9.7 samples/s +Step 2110/20005 | Loss: 3.1077 | LR: 4.89e-04 | 2480.6ms/step | 9.7 samples/s +Step 2120/20005 | Loss: 3.0934 | LR: 4.89e-04 | 2481.7ms/step | 9.7 samples/s +Step 2130/20005 | Loss: 3.2026 | LR: 4.89e-04 | 2482.3ms/step | 9.7 samples/s +Step 2140/20005 | Loss: 2.9999 | LR: 4.89e-04 | 2481.5ms/step | 9.7 samples/s +Step 2150/20005 | Loss: 3.1597 | LR: 4.89e-04 | 2478.5ms/step | 9.7 samples/s +Step 2160/20005 | Loss: 3.1149 | LR: 4.89e-04 | 2479.0ms/step | 9.7 samples/s +Step 2170/20005 | Loss: 3.0599 | LR: 4.88e-04 | 2480.0ms/step | 9.7 samples/s +Step 2180/20005 | Loss: 3.0751 | LR: 4.88e-04 | 2478.7ms/step | 9.7 samples/s +Step 2190/20005 | Loss: 3.0508 | LR: 4.88e-04 | 2481.1ms/step | 9.7 samples/s +Step 2200/20005 | Loss: 3.0565 | LR: 4.88e-04 | 2482.9ms/step | 9.7 samples/s +Step 2210/20005 | Loss: 3.0679 | LR: 4.88e-04 | 2479.5ms/step | 9.7 samples/s +Step 2220/20005 | Loss: 3.0740 | LR: 4.88e-04 | 2480.0ms/step | 9.7 samples/s +Step 2230/20005 | Loss: 3.0267 | LR: 4.88e-04 | 2482.2ms/step | 9.7 samples/s +Step 2240/20005 | Loss: 3.0273 | LR: 4.88e-04 | 2480.1ms/step | 9.7 samples/s +Step 2250/20005 | Loss: 3.1010 | LR: 4.87e-04 | 2484.6ms/step | 9.7 samples/s +Validation at step 2250: Loss = 3.0659, Perplexity = 21.45 +Step 2260/20005 | Loss: 3.0495 | LR: 4.87e-04 | 20211.6ms/step | 1.2 samples/s +Step 2270/20005 | Loss: 3.0487 | LR: 4.87e-04 | 2485.8ms/step | 9.7 samples/s +Step 2280/20005 | Loss: 3.0875 | LR: 4.87e-04 | 2485.9ms/step | 9.7 samples/s +Step 2290/20005 | Loss: 3.0081 | LR: 4.87e-04 | 2482.6ms/step | 9.7 samples/s +Step 2300/20005 | Loss: 3.0695 | LR: 4.87e-04 | 2485.7ms/step | 9.7 samples/s +Step 2310/20005 | Loss: 3.0367 | LR: 4.87e-04 | 2482.3ms/step | 9.7 samples/s +Step 2320/20005 | Loss: 2.9226 | LR: 4.87e-04 | 2481.8ms/step | 9.7 samples/s +Step 2330/20005 | Loss: 3.0387 | LR: 4.86e-04 | 2482.7ms/step | 9.7 samples/s +Step 2340/20005 | Loss: 3.0105 | LR: 4.86e-04 | 2484.5ms/step | 9.7 samples/s +Step 2350/20005 | Loss: 3.0209 | LR: 4.86e-04 | 2486.6ms/step | 9.7 samples/s +Step 2360/20005 | Loss: 3.0097 | LR: 4.86e-04 | 2490.4ms/step | 9.6 samples/s +Step 2370/20005 | Loss: 3.0632 | LR: 4.86e-04 | 2485.8ms/step | 9.7 samples/s +Step 2380/20005 | Loss: 3.0618 | LR: 4.86e-04 | 2485.3ms/step | 9.7 samples/s +Step 2390/20005 | Loss: 3.0674 | LR: 4.86e-04 | 2488.0ms/step | 9.6 samples/s +Step 2400/20005 | Loss: 3.0802 | LR: 4.86e-04 | 2485.4ms/step | 9.7 samples/s +Step 2410/20005 | Loss: 2.9627 | LR: 4.85e-04 | 2487.1ms/step | 9.6 samples/s +Step 2420/20005 | Loss: 3.0272 | LR: 4.85e-04 | 2485.9ms/step | 9.7 samples/s +Step 2430/20005 | Loss: 3.0372 | LR: 4.85e-04 | 2486.8ms/step | 9.7 samples/s +Step 2440/20005 | Loss: 3.1485 | LR: 4.85e-04 | 2483.2ms/step | 9.7 samples/s +Step 2450/20005 | Loss: 3.0323 | LR: 4.85e-04 | 2483.8ms/step | 9.7 samples/s +Step 2460/20005 | Loss: 2.9436 | LR: 4.85e-04 | 2484.7ms/step | 9.7 samples/s +Step 2470/20005 | Loss: 2.9838 | LR: 4.85e-04 | 2485.8ms/step | 9.7 samples/s +Step 2480/20005 | Loss: 3.0096 | LR: 4.84e-04 | 2485.2ms/step | 9.7 samples/s +Step 2490/20005 | Loss: 3.0263 | LR: 4.84e-04 | 2483.6ms/step | 9.7 samples/s +Step 2500/20005 | Loss: 3.0297 | LR: 4.84e-04 | 2486.2ms/step | 9.7 samples/s +Validation at step 2500: Loss = 3.0174, Perplexity = 20.44 +Step 2510/20005 | Loss: 3.0233 | LR: 4.84e-04 | 20150.6ms/step | 1.2 samples/s +Step 2520/20005 | Loss: 2.9520 | LR: 4.84e-04 | 2486.3ms/step | 9.7 samples/s +Step 2530/20005 | Loss: 2.9792 | LR: 4.84e-04 | 2487.3ms/step | 9.6 samples/s +Step 2540/20005 | Loss: 3.0410 | LR: 4.84e-04 | 2481.8ms/step | 9.7 samples/s +Step 2550/20005 | Loss: 3.0117 | LR: 4.83e-04 | 2483.7ms/step | 9.7 samples/s +Step 2560/20005 | Loss: 2.9952 | LR: 4.83e-04 | 2482.3ms/step | 9.7 samples/s +Step 2570/20005 | Loss: 3.0182 | LR: 4.83e-04 | 2486.8ms/step | 9.7 samples/s +Step 2580/20005 | Loss: 2.9531 | LR: 4.83e-04 | 2484.5ms/step | 9.7 samples/s +Step 2590/20005 | Loss: 3.0715 | LR: 4.83e-04 | 2483.8ms/step | 9.7 samples/s +Step 2600/20005 | Loss: 2.9748 | LR: 4.83e-04 | 2486.1ms/step | 9.7 samples/s +Step 2610/20005 | Loss: 2.9719 | LR: 4.83e-04 | 2484.1ms/step | 9.7 samples/s +Step 2620/20005 | Loss: 3.0272 | LR: 4.82e-04 | 2641.3ms/step | 9.1 samples/s +Step 2630/20005 | Loss: 2.9623 | LR: 4.82e-04 | 2483.6ms/step | 9.7 samples/s +Step 2640/20005 | Loss: 2.9279 | LR: 4.82e-04 | 2487.8ms/step | 9.6 samples/s +Step 2650/20005 | Loss: 3.0301 | LR: 4.82e-04 | 2486.4ms/step | 9.7 samples/s +Step 2660/20005 | Loss: 2.9855 | LR: 4.82e-04 | 2485.6ms/step | 9.7 samples/s +Step 2670/20005 | Loss: 2.9391 | LR: 4.82e-04 | 2484.7ms/step | 9.7 samples/s +Step 2680/20005 | Loss: 2.9792 | LR: 4.82e-04 | 2491.4ms/step | 9.6 samples/s +Step 2690/20005 | Loss: 2.9522 | LR: 4.81e-04 | 2489.8ms/step | 9.6 samples/s +Step 2700/20005 | Loss: 2.9745 | LR: 4.81e-04 | 2487.5ms/step | 9.6 samples/s +Step 2710/20005 | Loss: 2.9118 | LR: 4.81e-04 | 2491.0ms/step | 9.6 samples/s +Step 2720/20005 | Loss: 2.9993 | LR: 4.81e-04 | 2478.8ms/step | 9.7 samples/s +Step 2730/20005 | Loss: 3.0726 | LR: 4.81e-04 | 2481.5ms/step | 9.7 samples/s +Step 2740/20005 | Loss: 2.9389 | LR: 4.81e-04 | 2486.7ms/step | 9.7 samples/s +Step 2750/20005 | Loss: 2.9551 | LR: 4.81e-04 | 2489.4ms/step | 9.6 samples/s +Validation at step 2750: Loss = 2.9700, Perplexity = 19.49 +Step 2760/20005 | Loss: 2.9470 | LR: 4.80e-04 | 20153.4ms/step | 1.2 samples/s +Step 2770/20005 | Loss: 3.0599 | LR: 4.80e-04 | 2489.6ms/step | 9.6 samples/s +Step 2780/20005 | Loss: 2.9620 | LR: 4.80e-04 | 2484.6ms/step | 9.7 samples/s +Step 2790/20005 | Loss: 3.0053 | LR: 4.80e-04 | 2482.9ms/step | 9.7 samples/s +Step 2800/20005 | Loss: 2.9547 | LR: 4.80e-04 | 2484.7ms/step | 9.7 samples/s +Step 2810/20005 | Loss: 2.9998 | LR: 4.80e-04 | 2487.3ms/step | 9.6 samples/s +Step 2820/20005 | Loss: 3.0059 | LR: 4.79e-04 | 2488.8ms/step | 9.6 samples/s +Step 2830/20005 | Loss: 2.8953 | LR: 4.79e-04 | 2482.8ms/step | 9.7 samples/s +Step 2840/20005 | Loss: 2.9341 | LR: 4.79e-04 | 2487.6ms/step | 9.6 samples/s +Step 2850/20005 | Loss: 2.8997 | LR: 4.79e-04 | 2486.9ms/step | 9.7 samples/s +Step 2860/20005 | Loss: 2.8518 | LR: 4.79e-04 | 2486.5ms/step | 9.7 samples/s +Step 2870/20005 | Loss: 3.0818 | LR: 4.79e-04 | 2485.5ms/step | 9.7 samples/s +Step 2880/20005 | Loss: 2.9127 | LR: 4.78e-04 | 2482.8ms/step | 9.7 samples/s +Step 2890/20005 | Loss: 2.8781 | LR: 4.78e-04 | 2483.6ms/step | 9.7 samples/s +Step 2900/20005 | Loss: 2.9468 | LR: 4.78e-04 | 2484.3ms/step | 9.7 samples/s +Step 2910/20005 | Loss: 2.8944 | LR: 4.78e-04 | 2485.6ms/step | 9.7 samples/s +Step 2920/20005 | Loss: 2.9003 | LR: 4.78e-04 | 2487.8ms/step | 9.6 samples/s +Step 2930/20005 | Loss: 2.9508 | LR: 4.78e-04 | 2487.2ms/step | 9.6 samples/s +Step 2940/20005 | Loss: 2.9581 | LR: 4.77e-04 | 2484.5ms/step | 9.7 samples/s +Step 2950/20005 | Loss: 2.8649 | LR: 4.77e-04 | 2485.0ms/step | 9.7 samples/s +Step 2960/20005 | Loss: 2.9527 | LR: 4.77e-04 | 2488.1ms/step | 9.6 samples/s +Step 2970/20005 | Loss: 2.9560 | LR: 4.77e-04 | 2487.6ms/step | 9.6 samples/s +Step 2980/20005 | Loss: 2.9454 | LR: 4.77e-04 | 2485.0ms/step | 9.7 samples/s +Step 2990/20005 | Loss: 2.9172 | LR: 4.77e-04 | 2483.9ms/step | 9.7 samples/s +Step 3000/20005 | Loss: 2.8768 | LR: 4.76e-04 | 2483.7ms/step | 9.7 samples/s +Validation at step 3000: Loss = 2.9385, Perplexity = 18.89 +Step 3010/20005 | Loss: 2.9027 | LR: 4.76e-04 | 20163.4ms/step | 1.2 samples/s +Step 3020/20005 | Loss: 2.8841 | LR: 4.76e-04 | 2490.1ms/step | 9.6 samples/s +Step 3030/20005 | Loss: 2.8952 | LR: 4.76e-04 | 2490.2ms/step | 9.6 samples/s +Step 3040/20005 | Loss: 2.9458 | LR: 4.76e-04 | 2486.3ms/step | 9.7 samples/s +Step 3050/20005 | Loss: 3.0163 | LR: 4.76e-04 | 2488.3ms/step | 9.6 samples/s +Step 3060/20005 | Loss: 2.8302 | LR: 4.75e-04 | 2483.8ms/step | 9.7 samples/s +Step 3070/20005 | Loss: 2.9645 | LR: 4.75e-04 | 2485.0ms/step | 9.7 samples/s +Step 3080/20005 | Loss: 2.8695 | LR: 4.75e-04 | 2486.8ms/step | 9.7 samples/s +Step 3090/20005 | Loss: 2.9766 | LR: 4.75e-04 | 2487.1ms/step | 9.6 samples/s +Step 3100/20005 | Loss: 2.9351 | LR: 4.75e-04 | 2488.3ms/step | 9.6 samples/s +Step 3110/20005 | Loss: 2.8867 | LR: 4.75e-04 | 2489.0ms/step | 9.6 samples/s +Step 3120/20005 | Loss: 2.9147 | LR: 4.74e-04 | 2486.7ms/step | 9.7 samples/s +Step 3130/20005 | Loss: 2.9249 | LR: 4.74e-04 | 2489.3ms/step | 9.6 samples/s +Step 3140/20005 | Loss: 2.9412 | LR: 4.74e-04 | 2487.1ms/step | 9.6 samples/s +Step 3150/20005 | Loss: 2.9122 | LR: 4.74e-04 | 2490.4ms/step | 9.6 samples/s +Step 3160/20005 | Loss: 2.9904 | LR: 4.74e-04 | 2489.2ms/step | 9.6 samples/s +Step 3170/20005 | Loss: 2.8860 | LR: 4.74e-04 | 2483.5ms/step | 9.7 samples/s +Step 3180/20005 | Loss: 2.8189 | LR: 4.73e-04 | 2489.9ms/step | 9.6 samples/s +Step 3190/20005 | Loss: 2.8932 | LR: 4.73e-04 | 2487.6ms/step | 9.6 samples/s +Step 3200/20005 | Loss: 2.9586 | LR: 4.73e-04 | 2482.2ms/step | 9.7 samples/s +Step 3210/20005 | Loss: 2.7848 | LR: 4.73e-04 | 2484.2ms/step | 9.7 samples/s +Step 3220/20005 | Loss: 2.8996 | LR: 4.73e-04 | 2485.9ms/step | 9.7 samples/s +Step 3230/20005 | Loss: 2.9051 | LR: 4.72e-04 | 2486.1ms/step | 9.7 samples/s +Step 3240/20005 | Loss: 2.8546 | LR: 4.72e-04 | 2484.9ms/step | 9.7 samples/s +Step 3250/20005 | Loss: 2.8317 | LR: 4.72e-04 | 2481.8ms/step | 9.7 samples/s +Validation at step 3250: Loss = 2.8979, Perplexity = 18.14 +Step 3260/20005 | Loss: 2.8391 | LR: 4.72e-04 | 20153.4ms/step | 1.2 samples/s +Step 3270/20005 | Loss: 2.8688 | LR: 4.72e-04 | 2487.9ms/step | 9.6 samples/s +Step 3280/20005 | Loss: 2.8271 | LR: 4.72e-04 | 2486.2ms/step | 9.7 samples/s +Step 3290/20005 | Loss: 2.8906 | LR: 4.71e-04 | 2486.1ms/step | 9.7 samples/s +Step 3300/20005 | Loss: 2.8870 | LR: 4.71e-04 | 2486.2ms/step | 9.7 samples/s +Step 3310/20005 | Loss: 2.8872 | LR: 4.71e-04 | 2486.1ms/step | 9.7 samples/s +Step 3320/20005 | Loss: 2.8777 | LR: 4.71e-04 | 2485.0ms/step | 9.7 samples/s +Step 3330/20005 | Loss: 2.9292 | LR: 4.71e-04 | 2482.9ms/step | 9.7 samples/s +Step 3340/20005 | Loss: 2.9075 | LR: 4.70e-04 | 2486.3ms/step | 9.7 samples/s +Step 3350/20005 | Loss: 2.8418 | LR: 4.70e-04 | 2485.6ms/step | 9.7 samples/s +Step 3360/20005 | Loss: 2.8374 | LR: 4.70e-04 | 2485.8ms/step | 9.7 samples/s +Step 3370/20005 | Loss: 2.8806 | LR: 4.70e-04 | 2486.6ms/step | 9.7 samples/s +Step 3380/20005 | Loss: 2.8556 | LR: 4.70e-04 | 2486.1ms/step | 9.7 samples/s +Step 3390/20005 | Loss: 2.8850 | LR: 4.69e-04 | 2483.9ms/step | 9.7 samples/s +Step 3400/20005 | Loss: 2.7990 | LR: 4.69e-04 | 2487.9ms/step | 9.6 samples/s +Step 3410/20005 | Loss: 2.8183 | LR: 4.69e-04 | 2488.4ms/step | 9.6 samples/s +Step 3420/20005 | Loss: 2.8690 | LR: 4.69e-04 | 2585.4ms/step | 9.3 samples/s +Step 3430/20005 | Loss: 2.9224 | LR: 4.69e-04 | 2487.0ms/step | 9.7 samples/s +Step 3440/20005 | Loss: 2.8586 | LR: 4.69e-04 | 2490.0ms/step | 9.6 samples/s +Step 3450/20005 | Loss: 2.8856 | LR: 4.68e-04 | 2487.9ms/step | 9.6 samples/s +Step 3460/20005 | Loss: 2.8790 | LR: 4.68e-04 | 2486.1ms/step | 9.7 samples/s +Step 3470/20005 | Loss: 2.9077 | LR: 4.68e-04 | 2487.2ms/step | 9.6 samples/s +Step 3480/20005 | Loss: 2.8669 | LR: 4.68e-04 | 2487.2ms/step | 9.6 samples/s +Step 3490/20005 | Loss: 2.8640 | LR: 4.68e-04 | 2488.6ms/step | 9.6 samples/s +Step 3500/20005 | Loss: 2.7557 | LR: 4.67e-04 | 2491.2ms/step | 9.6 samples/s +Validation at step 3500: Loss = 2.8650, Perplexity = 17.55 +Step 3510/20005 | Loss: 2.9039 | LR: 4.67e-04 | 20195.0ms/step | 1.2 samples/s +Step 3520/20005 | Loss: 2.8394 | LR: 4.67e-04 | 2491.3ms/step | 9.6 samples/s +Step 3530/20005 | Loss: 2.7960 | LR: 4.67e-04 | 2486.9ms/step | 9.7 samples/s +Step 3540/20005 | Loss: 2.8816 | LR: 4.67e-04 | 2486.8ms/step | 9.7 samples/s +Step 3550/20005 | Loss: 2.8719 | LR: 4.66e-04 | 2488.1ms/step | 9.6 samples/s +Step 3560/20005 | Loss: 2.7991 | LR: 4.66e-04 | 2488.5ms/step | 9.6 samples/s +Step 3570/20005 | Loss: 2.8148 | LR: 4.66e-04 | 2486.1ms/step | 9.7 samples/s +Step 3580/20005 | Loss: 2.7765 | LR: 4.66e-04 | 2488.7ms/step | 9.6 samples/s +Step 3590/20005 | Loss: 2.8594 | LR: 4.66e-04 | 2487.3ms/step | 9.6 samples/s +Step 3600/20005 | Loss: 2.8868 | LR: 4.65e-04 | 2487.5ms/step | 9.6 samples/s +Step 3610/20005 | Loss: 2.9076 | LR: 4.65e-04 | 2485.3ms/step | 9.7 samples/s +Step 3620/20005 | Loss: 2.8390 | LR: 4.65e-04 | 2490.2ms/step | 9.6 samples/s +Step 3630/20005 | Loss: 2.9418 | LR: 4.65e-04 | 2489.0ms/step | 9.6 samples/s +Step 3640/20005 | Loss: 2.8458 | LR: 4.65e-04 | 2488.1ms/step | 9.6 samples/s +Step 3650/20005 | Loss: 2.9035 | LR: 4.64e-04 | 2487.4ms/step | 9.6 samples/s +Step 3660/20005 | Loss: 2.8621 | LR: 4.64e-04 | 2486.7ms/step | 9.7 samples/s +Step 3670/20005 | Loss: 2.8240 | LR: 4.64e-04 | 2487.5ms/step | 9.6 samples/s +Step 3680/20005 | Loss: 2.7992 | LR: 4.64e-04 | 2486.5ms/step | 9.7 samples/s +Step 3690/20005 | Loss: 2.8206 | LR: 4.64e-04 | 2487.2ms/step | 9.6 samples/s +Step 3700/20005 | Loss: 2.9252 | LR: 4.63e-04 | 2488.6ms/step | 9.6 samples/s +Step 3710/20005 | Loss: 2.8458 | LR: 4.63e-04 | 2487.8ms/step | 9.6 samples/s +Step 3720/20005 | Loss: 2.7925 | LR: 4.63e-04 | 2486.8ms/step | 9.7 samples/s +Step 3730/20005 | Loss: 2.7895 | LR: 4.63e-04 | 2486.2ms/step | 9.7 samples/s +Step 3740/20005 | Loss: 2.8958 | LR: 4.62e-04 | 2486.5ms/step | 9.7 samples/s +Step 3750/20005 | Loss: 2.7859 | LR: 4.62e-04 | 2487.0ms/step | 9.7 samples/s diff --git a/epfl_final/log_files/build_mask_19.log b/epfl_final/log_files/build_mask_19.log new file mode 100644 index 0000000000000000000000000000000000000000..51ed16d120db53434e485a0f4ef279fbb0e5a8cd --- /dev/null +++ b/epfl_final/log_files/build_mask_19.log @@ -0,0 +1,14 @@ +[build_masks] Config: + sparse layers : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27] + seq_len : 8192 + block_size : 128 + num_blocks : 64 + num_random : 18 + mask_pool_size: 2000 + num_workers : 192 + cache_dir : ./mask_cache_19 + force_rebuild : False + +[build_masks] Building 2000 masks × 28 layers on 1 GPU(s). Est. 140s/layer on GPU. + 0%| | 0/28 [00:00 + main() + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 367, in wrapper + return f(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1016, in main + run(args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1007, in run + elastic_launch( + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 191, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 371, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +====================================================== +SparseAtt/train.py FAILED +------------------------------------------------------ +Failures: +[1]: + time : 2026-05-30_02:03:52 + host : h100-wspace-3-gpu-0-0 + rank : 1 (local_rank: 1) + exitcode : -15 (pid: 9082) (SIGTERM) + error_file: + traceback : Signal 15 (SIGTERM) received by PID 9082 +[2]: + time : 2026-05-30_02:03:52 + host : h100-wspace-3-gpu-0-0 + rank : 2 (local_rank: 2) + exitcode : -15 (pid: 9083) (SIGTERM) + error_file: + traceback : Signal 15 (SIGTERM) received by PID 9083 +------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-30_02:03:52 + host : h100-wspace-3-gpu-0-0 + rank : 0 (local_rank: 0) + exitcode : -15 (pid: 9081) (SIGTERM) + error_file: + traceback : Signal 15 (SIGTERM) received by PID 9081 +====================================================== diff --git a/epfl_final/log_files/cont_pretrain_10k.log b/epfl_final/log_files/cont_pretrain_10k.log new file mode 100644 index 0000000000000000000000000000000000000000..167c967d8a8a295372302273d18ef38b7a4483a6 --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_10k.log @@ -0,0 +1,3346 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-21000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 10k", + "wandb_tags": [ + "qwen3", + "1.2b", + "dense-resume" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache", + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Checkpoint activations: False +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-21000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-21000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 21000 (epoch 0, micro_step 210000) +Resuming training from step 21000 +Resuming wandb run ns4ua5fr +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.2 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260606_174407-ns4ua5fr +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run dense continued 10k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ns4ua5fr +Starting training... +Validation at step 21000: Loss = 2.6046, Perplexity = 13.53 +wandb: WARNING Tried to log to step 21000 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 21001: dense (dense step interval) +Step 21010/50462 | Loss: 2.6135 | LR: 3.22e-04 | 48826.1ms/step | 1.2 samples/s +wandb: WARNING Tried to log to step 21010 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21020/50462 | Loss: 2.6203 | LR: 3.21e-04 | 5170.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21020 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21030/50462 | Loss: 2.5691 | LR: 3.21e-04 | 5170.4ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21030 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21040/50462 | Loss: 2.5921 | LR: 3.21e-04 | 5177.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21040 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21050/50462 | Loss: 2.5760 | LR: 3.21e-04 | 5164.1ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21050 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21060/50462 | Loss: 2.5847 | LR: 3.21e-04 | 5185.6ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21060 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21070/50462 | Loss: 2.5893 | LR: 3.21e-04 | 5170.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21070 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21080/50462 | Loss: 2.5925 | LR: 3.21e-04 | 5169.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21080 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21090/50462 | Loss: 2.6132 | LR: 3.20e-04 | 5177.4ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21090 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21100/50462 | Loss: 2.6285 | LR: 3.20e-04 | 5169.0ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21100 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21110/50462 | Loss: 2.6008 | LR: 3.20e-04 | 5169.7ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21110 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21120/50462 | Loss: 2.5722 | LR: 3.20e-04 | 5161.5ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21120 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21130/50462 | Loss: 2.6009 | LR: 3.20e-04 | 5162.0ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21130 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21140/50462 | Loss: 2.6014 | LR: 3.20e-04 | 5155.7ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21140 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21150/50462 | Loss: 2.5730 | LR: 3.19e-04 | 5152.2ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21150 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21160/50462 | Loss: 2.5631 | LR: 3.19e-04 | 5162.0ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21160 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21170/50462 | Loss: 2.5969 | LR: 3.19e-04 | 5170.3ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21170 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21180/50462 | Loss: 2.6122 | LR: 3.19e-04 | 5155.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21180 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21190/50462 | Loss: 2.6209 | LR: 3.19e-04 | 5165.4ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21190 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21200/50462 | Loss: 2.6052 | LR: 3.19e-04 | 5159.5ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21200 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21210/50462 | Loss: 2.6155 | LR: 3.19e-04 | 5152.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21210 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21220/50462 | Loss: 2.6116 | LR: 3.18e-04 | 5151.2ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21220 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21230/50462 | Loss: 2.5664 | LR: 3.18e-04 | 5158.2ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21230 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21240/50462 | Loss: 2.6105 | LR: 3.18e-04 | 5161.5ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21240 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21250/50462 | Loss: 2.5879 | LR: 3.18e-04 | 5171.0ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21250 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 21250: Loss = 2.6017, Perplexity = 13.49 +wandb: WARNING Tried to log to step 21250 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21260/50462 | Loss: 2.5754 | LR: 3.18e-04 | 7707.5ms/step | 7.8 samples/s +wandb: WARNING Tried to log to step 21260 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21270/50462 | Loss: 2.5796 | LR: 3.18e-04 | 5154.4ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21270 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21280/50462 | Loss: 2.6217 | LR: 3.17e-04 | 5154.7ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21280 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21290/50462 | Loss: 2.6058 | LR: 3.17e-04 | 5142.2ms/step | 11.7 samples/s +wandb: WARNING Tried to log to step 21290 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21300/50462 | Loss: 2.5857 | LR: 3.17e-04 | 5145.0ms/step | 11.7 samples/s +wandb: WARNING Tried to log to step 21300 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21310/50462 | Loss: 2.5981 | LR: 3.17e-04 | 5152.2ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21310 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21320/50462 | Loss: 2.5810 | LR: 3.17e-04 | 5154.1ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 21320 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21330/50462 | Loss: 2.5928 | LR: 3.17e-04 | 5149.9ms/step | 11.7 samples/s +wandb: WARNING Tried to log to step 21330 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21340/50462 | Loss: 2.5954 | LR: 3.17e-04 | 5150.0ms/step | 11.7 samples/s +wandb: WARNING Tried to log to step 21340 that is less than the current step 21341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21350/50462 | Loss: 2.5664 | LR: 3.16e-04 | 5152.3ms/step | 11.6 samples/s +Step 21360/50462 | Loss: 2.5828 | LR: 3.16e-04 | 5141.9ms/step | 11.7 samples/s +Step 21370/50462 | Loss: 2.6041 | LR: 3.16e-04 | 5151.2ms/step | 11.6 samples/s +Step 21380/50462 | Loss: 2.5829 | LR: 3.16e-04 | 5140.9ms/step | 11.7 samples/s +Step 21390/50462 | Loss: 2.6217 | LR: 3.16e-04 | 5164.5ms/step | 11.6 samples/s +Step 21400/50462 | Loss: 2.6071 | LR: 3.16e-04 | 5157.1ms/step | 11.6 samples/s +Step 21410/50462 | Loss: 2.6090 | LR: 3.16e-04 | 5161.6ms/step | 11.6 samples/s +Step 21420/50462 | Loss: 2.5815 | LR: 3.15e-04 | 5169.5ms/step | 11.6 samples/s +Step 21430/50462 | Loss: 2.5935 | LR: 3.15e-04 | 5144.9ms/step | 11.7 samples/s +Step 21440/50462 | Loss: 2.5817 | LR: 3.15e-04 | 5147.5ms/step | 11.7 samples/s +Step 21450/50462 | Loss: 2.5832 | LR: 3.15e-04 | 5157.0ms/step | 11.6 samples/s +Step 21460/50462 | Loss: 2.5762 | LR: 3.15e-04 | 5168.9ms/step | 11.6 samples/s +Step 21470/50462 | Loss: 2.5868 | LR: 3.15e-04 | 5167.9ms/step | 11.6 samples/s +Step 21480/50462 | Loss: 2.5920 | LR: 3.14e-04 | 5155.4ms/step | 11.6 samples/s +Step 21490/50462 | Loss: 2.5922 | LR: 3.14e-04 | 5155.7ms/step | 11.6 samples/s +Step 21500/50462 | Loss: 2.5679 | LR: 3.14e-04 | 5149.9ms/step | 11.7 samples/s +Validation at step 21500: Loss = 2.5991, Perplexity = 13.45 +Step 21510/50462 | Loss: 2.5904 | LR: 3.14e-04 | 7718.7ms/step | 7.8 samples/s +Step 21520/50462 | Loss: 2.5551 | LR: 3.14e-04 | 5160.0ms/step | 11.6 samples/s +Step 21530/50462 | Loss: 2.6097 | LR: 3.14e-04 | 5164.7ms/step | 11.6 samples/s +Step 21540/50462 | Loss: 2.5700 | LR: 3.14e-04 | 5170.9ms/step | 11.6 samples/s +Step 21550/50462 | Loss: 2.5863 | LR: 3.13e-04 | 5158.2ms/step | 11.6 samples/s +Step 21560/50462 | Loss: 2.5695 | LR: 3.13e-04 | 5171.8ms/step | 11.6 samples/s +Step 21570/50462 | Loss: 2.5923 | LR: 3.13e-04 | 5173.2ms/step | 11.6 samples/s +Step 21580/50462 | Loss: 2.5676 | LR: 3.13e-04 | 5162.6ms/step | 11.6 samples/s +Step 21590/50462 | Loss: 2.5826 | LR: 3.13e-04 | 5158.5ms/step | 11.6 samples/s +Step 21600/50462 | Loss: 2.5572 | LR: 3.13e-04 | 5153.7ms/step | 11.6 samples/s +Step 21610/50462 | Loss: 2.5996 | LR: 3.12e-04 | 5177.0ms/step | 11.6 samples/s +Step 21620/50462 | Loss: 2.5985 | LR: 3.12e-04 | 5157.8ms/step | 11.6 samples/s +Step 21630/50462 | Loss: 2.5863 | LR: 3.12e-04 | 5162.5ms/step | 11.6 samples/s +Step 21640/50462 | Loss: 2.5507 | LR: 3.12e-04 | 5158.9ms/step | 11.6 samples/s +Step 21650/50462 | Loss: 2.6120 | LR: 3.12e-04 | 5176.6ms/step | 11.6 samples/s +Step 21660/50462 | Loss: 2.5547 | LR: 3.12e-04 | 5164.1ms/step | 11.6 samples/s +Step 21670/50462 | Loss: 2.5571 | LR: 3.12e-04 | 5151.9ms/step | 11.6 samples/s +Step 21680/50462 | Loss: 2.5880 | LR: 3.11e-04 | 5152.4ms/step | 11.6 samples/s +Step 21690/50462 | Loss: 2.5652 | LR: 3.11e-04 | 5152.3ms/step | 11.6 samples/s +Step 21700/50462 | Loss: 2.6110 | LR: 3.11e-04 | 5141.8ms/step | 11.7 samples/s +Step 21710/50462 | Loss: 2.6158 | LR: 3.11e-04 | 5148.0ms/step | 11.7 samples/s +Step 21720/50462 | Loss: 2.5641 | LR: 3.11e-04 | 5132.3ms/step | 11.7 samples/s +Step 21730/50462 | Loss: 2.5927 | LR: 3.11e-04 | 5147.3ms/step | 11.7 samples/s +Step 21740/50462 | Loss: 2.5657 | LR: 3.10e-04 | 5160.3ms/step | 11.6 samples/s +Step 21750/50462 | Loss: 2.5586 | LR: 3.10e-04 | 5152.6ms/step | 11.6 samples/s +Validation at step 21750: Loss = 2.5963, Perplexity = 13.41 +Step 21760/50462 | Loss: 2.5713 | LR: 3.10e-04 | 7714.3ms/step | 7.8 samples/s +Step 21770/50462 | Loss: 2.5521 | LR: 3.10e-04 | 5146.5ms/step | 11.7 samples/s +Step 21780/50462 | Loss: 2.5497 | LR: 3.10e-04 | 5149.0ms/step | 11.7 samples/s +Step 21790/50462 | Loss: 2.5690 | LR: 3.10e-04 | 5156.6ms/step | 11.6 samples/s +Step 21800/50462 | Loss: 2.5945 | LR: 3.10e-04 | 5164.2ms/step | 11.6 samples/s +Step 21810/50462 | Loss: 2.6145 | LR: 3.09e-04 | 5154.8ms/step | 11.6 samples/s +Step 21820/50462 | Loss: 2.5666 | LR: 3.09e-04 | 5160.7ms/step | 11.6 samples/s +Step 21830/50462 | Loss: 2.5890 | LR: 3.09e-04 | 5154.1ms/step | 11.6 samples/s +Step 21840/50462 | Loss: 2.5833 | LR: 3.09e-04 | 5160.2ms/step | 11.6 samples/s +Step 21850/50462 | Loss: 2.5705 | LR: 3.09e-04 | 5163.2ms/step | 11.6 samples/s +Step 21860/50462 | Loss: 2.5941 | LR: 3.09e-04 | 5164.8ms/step | 11.6 samples/s +Step 21870/50462 | Loss: 2.5772 | LR: 3.08e-04 | 5150.8ms/step | 11.6 samples/s +Step 21880/50462 | Loss: 2.5822 | LR: 3.08e-04 | 5154.6ms/step | 11.6 samples/s +Step 21890/50462 | Loss: 2.5921 | LR: 3.08e-04 | 5149.2ms/step | 11.7 samples/s +Step 21900/50462 | Loss: 2.5663 | LR: 3.08e-04 | 5160.6ms/step | 11.6 samples/s +Step 21910/50462 | Loss: 2.5658 | LR: 3.08e-04 | 5163.9ms/step | 11.6 samples/s +Step 21920/50462 | Loss: 2.5759 | LR: 3.08e-04 | 5160.7ms/step | 11.6 samples/s +Step 21930/50462 | Loss: 2.5972 | LR: 3.08e-04 | 5155.0ms/step | 11.6 samples/s +Step 21940/50462 | Loss: 2.6073 | LR: 3.07e-04 | 5150.4ms/step | 11.6 samples/s +Step 21950/50462 | Loss: 2.5856 | LR: 3.07e-04 | 5153.9ms/step | 11.6 samples/s +Step 21960/50462 | Loss: 2.5786 | LR: 3.07e-04 | 5151.4ms/step | 11.6 samples/s +Step 21970/50462 | Loss: 2.5759 | LR: 3.07e-04 | 5158.9ms/step | 11.6 samples/s +Step 21980/50462 | Loss: 2.5649 | LR: 3.07e-04 | 5157.6ms/step | 11.6 samples/s +Step 21990/50462 | Loss: 2.5784 | LR: 3.07e-04 | 5158.1ms/step | 11.6 samples/s +Step 22000/50462 | Loss: 2.6025 | LR: 3.06e-04 | 5175.0ms/step | 11.6 samples/s +Validation at step 22000: Loss = 2.5937, Perplexity = 13.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-22000 +Step 22010/50462 | Loss: 2.5666 | LR: 3.06e-04 | 8551.8ms/step | 7.0 samples/s +Step 22020/50462 | Loss: 2.5721 | LR: 3.06e-04 | 5636.0ms/step | 10.6 samples/s +Step 22030/50462 | Loss: 2.5754 | LR: 3.06e-04 | 5166.7ms/step | 11.6 samples/s +Step 22040/50462 | Loss: 2.5999 | LR: 3.06e-04 | 5150.9ms/step | 11.6 samples/s +Step 22050/50462 | Loss: 2.5865 | LR: 3.06e-04 | 5165.4ms/step | 11.6 samples/s +Step 22060/50462 | Loss: 2.5642 | LR: 3.06e-04 | 5648.9ms/step | 10.6 samples/s +Step 22070/50462 | Loss: 2.6019 | LR: 3.05e-04 | 5166.0ms/step | 11.6 samples/s +Step 22080/50462 | Loss: 2.5662 | LR: 3.05e-04 | 5675.0ms/step | 10.6 samples/s +Step 22090/50462 | Loss: 2.5850 | LR: 3.05e-04 | 5160.0ms/step | 11.6 samples/s +Step 22100/50462 | Loss: 2.5747 | LR: 3.05e-04 | 5102.2ms/step | 11.8 samples/s +Step 22110/50462 | Loss: 2.5921 | LR: 3.05e-04 | 5086.0ms/step | 11.8 samples/s +Step 22120/50462 | Loss: 2.5741 | LR: 3.05e-04 | 5096.7ms/step | 11.8 samples/s +Step 22130/50462 | Loss: 2.5867 | LR: 3.04e-04 | 5088.4ms/step | 11.8 samples/s +Step 22140/50462 | Loss: 2.5644 | LR: 3.04e-04 | 5091.7ms/step | 11.8 samples/s +Step 22150/50462 | Loss: 2.5896 | LR: 3.04e-04 | 5099.8ms/step | 11.8 samples/s +Step 22160/50462 | Loss: 2.6127 | LR: 3.04e-04 | 5092.8ms/step | 11.8 samples/s +Step 22170/50462 | Loss: 2.5687 | LR: 3.04e-04 | 5097.8ms/step | 11.8 samples/s +Step 22180/50462 | Loss: 2.5905 | LR: 3.04e-04 | 5101.7ms/step | 11.8 samples/s +Step 22190/50462 | Loss: 2.5981 | LR: 3.04e-04 | 5102.0ms/step | 11.8 samples/s +Step 22200/50462 | Loss: 2.5890 | LR: 3.03e-04 | 5087.9ms/step | 11.8 samples/s +Step 22210/50462 | Loss: 2.6081 | LR: 3.03e-04 | 5103.4ms/step | 11.8 samples/s +Step 22220/50462 | Loss: 2.5973 | LR: 3.03e-04 | 5100.9ms/step | 11.8 samples/s +Step 22230/50462 | Loss: 2.5673 | LR: 3.03e-04 | 5096.7ms/step | 11.8 samples/s +Step 22240/50462 | Loss: 2.5717 | LR: 3.03e-04 | 5094.5ms/step | 11.8 samples/s +Step 22250/50462 | Loss: 2.5707 | LR: 3.03e-04 | 5113.1ms/step | 11.7 samples/s +Validation at step 22250: Loss = 2.5915, Perplexity = 13.35 +Step 22260/50462 | Loss: 2.5783 | LR: 3.02e-04 | 7650.4ms/step | 7.8 samples/s +Step 22270/50462 | Loss: 2.6118 | LR: 3.02e-04 | 5120.9ms/step | 11.7 samples/s +Step 22280/50462 | Loss: 2.5491 | LR: 3.02e-04 | 5122.0ms/step | 11.7 samples/s +Step 22290/50462 | Loss: 2.5867 | LR: 3.02e-04 | 5102.6ms/step | 11.8 samples/s +Step 22300/50462 | Loss: 2.5588 | LR: 3.02e-04 | 5110.0ms/step | 11.7 samples/s +Step 22310/50462 | Loss: 2.5939 | LR: 3.02e-04 | 5114.4ms/step | 11.7 samples/s +Step 22320/50462 | Loss: 2.5592 | LR: 3.02e-04 | 5108.3ms/step | 11.7 samples/s +Step 22330/50462 | Loss: 2.6045 | LR: 3.01e-04 | 5115.8ms/step | 11.7 samples/s +Step 22340/50462 | Loss: 2.5768 | LR: 3.01e-04 | 5123.0ms/step | 11.7 samples/s +Step 22350/50462 | Loss: 2.5905 | LR: 3.01e-04 | 5102.2ms/step | 11.8 samples/s +Step 22360/50462 | Loss: 2.5789 | LR: 3.01e-04 | 5107.9ms/step | 11.7 samples/s +Step 22370/50462 | Loss: 2.5798 | LR: 3.01e-04 | 5127.1ms/step | 11.7 samples/s +Step 22380/50462 | Loss: 2.5550 | LR: 3.01e-04 | 5125.7ms/step | 11.7 samples/s +Step 22390/50462 | Loss: 2.5943 | LR: 3.00e-04 | 5125.9ms/step | 11.7 samples/s +Step 22400/50462 | Loss: 2.5638 | LR: 3.00e-04 | 5141.2ms/step | 11.7 samples/s +Step 22410/50462 | Loss: 2.5843 | LR: 3.00e-04 | 5123.8ms/step | 11.7 samples/s +Step 22420/50462 | Loss: 2.5685 | LR: 3.00e-04 | 5133.1ms/step | 11.7 samples/s +Step 22430/50462 | Loss: 2.5857 | LR: 3.00e-04 | 5136.9ms/step | 11.7 samples/s +Step 22440/50462 | Loss: 2.5952 | LR: 3.00e-04 | 5125.2ms/step | 11.7 samples/s +Step 22450/50462 | Loss: 2.5793 | LR: 3.00e-04 | 5127.2ms/step | 11.7 samples/s +Step 22460/50462 | Loss: 2.5476 | LR: 2.99e-04 | 5138.7ms/step | 11.7 samples/s +Step 22470/50462 | Loss: 2.5686 | LR: 2.99e-04 | 5110.2ms/step | 11.7 samples/s +Step 22480/50462 | Loss: 2.5786 | LR: 2.99e-04 | 5113.9ms/step | 11.7 samples/s +Step 22490/50462 | Loss: 2.5643 | LR: 2.99e-04 | 5120.0ms/step | 11.7 samples/s +Step 22500/50462 | Loss: 2.5652 | LR: 2.99e-04 | 5116.0ms/step | 11.7 samples/s +Validation at step 22500: Loss = 2.5891, Perplexity = 13.32 +Step 22510/50462 | Loss: 2.5839 | LR: 2.99e-04 | 7657.0ms/step | 7.8 samples/s +Step 22520/50462 | Loss: 2.5921 | LR: 2.98e-04 | 5093.7ms/step | 11.8 samples/s +Step 22530/50462 | Loss: 2.5636 | LR: 2.98e-04 | 5119.0ms/step | 11.7 samples/s +Step 22540/50462 | Loss: 2.5925 | LR: 2.98e-04 | 5118.7ms/step | 11.7 samples/s +Step 22550/50462 | Loss: 2.5666 | LR: 2.98e-04 | 5111.9ms/step | 11.7 samples/s +Step 22560/50462 | Loss: 2.5747 | LR: 2.98e-04 | 5101.5ms/step | 11.8 samples/s +Step 22570/50462 | Loss: 2.5855 | LR: 2.98e-04 | 5098.4ms/step | 11.8 samples/s +Step 22580/50462 | Loss: 2.5355 | LR: 2.98e-04 | 5097.6ms/step | 11.8 samples/s +Step 22590/50462 | Loss: 2.6067 | LR: 2.97e-04 | 5127.8ms/step | 11.7 samples/s +Step 22600/50462 | Loss: 2.5769 | LR: 2.97e-04 | 5107.8ms/step | 11.7 samples/s +Step 22610/50462 | Loss: 2.5946 | LR: 2.97e-04 | 5101.3ms/step | 11.8 samples/s +Step 22620/50462 | Loss: 2.5559 | LR: 2.97e-04 | 5088.8ms/step | 11.8 samples/s +Step 22630/50462 | Loss: 2.5515 | LR: 2.97e-04 | 5095.6ms/step | 11.8 samples/s +Step 22640/50462 | Loss: 2.5604 | LR: 2.97e-04 | 5099.1ms/step | 11.8 samples/s +Step 22650/50462 | Loss: 2.6182 | LR: 2.96e-04 | 5107.7ms/step | 11.7 samples/s +Step 22660/50462 | Loss: 2.5788 | LR: 2.96e-04 | 5106.0ms/step | 11.8 samples/s +Step 22670/50462 | Loss: 2.5611 | LR: 2.96e-04 | 5094.6ms/step | 11.8 samples/s +Step 22680/50462 | Loss: 2.5910 | LR: 2.96e-04 | 5115.6ms/step | 11.7 samples/s +Step 22690/50462 | Loss: 2.5695 | LR: 2.96e-04 | 5106.8ms/step | 11.7 samples/s +Step 22700/50462 | Loss: 2.5858 | LR: 2.96e-04 | 5120.0ms/step | 11.7 samples/s +Step 22710/50462 | Loss: 2.5655 | LR: 2.95e-04 | 5119.1ms/step | 11.7 samples/s +Step 22720/50462 | Loss: 2.6083 | LR: 2.95e-04 | 5112.6ms/step | 11.7 samples/s +Step 22730/50462 | Loss: 2.5636 | LR: 2.95e-04 | 5122.8ms/step | 11.7 samples/s +Step 22740/50462 | Loss: 2.5866 | LR: 2.95e-04 | 5129.2ms/step | 11.7 samples/s +Step 22750/50462 | Loss: 2.5984 | LR: 2.95e-04 | 5126.9ms/step | 11.7 samples/s +Validation at step 22750: Loss = 2.5873, Perplexity = 13.29 +Step 22760/50462 | Loss: 2.5904 | LR: 2.95e-04 | 7720.1ms/step | 7.8 samples/s +Step 22770/50462 | Loss: 2.5794 | LR: 2.95e-04 | 5162.5ms/step | 11.6 samples/s +Step 22780/50462 | Loss: 2.5733 | LR: 2.94e-04 | 5161.0ms/step | 11.6 samples/s +Step 22790/50462 | Loss: 2.5716 | LR: 2.94e-04 | 5157.8ms/step | 11.6 samples/s +Step 22800/50462 | Loss: 2.5956 | LR: 2.94e-04 | 5171.2ms/step | 11.6 samples/s +Step 22810/50462 | Loss: 2.5753 | LR: 2.94e-04 | 5165.2ms/step | 11.6 samples/s +Step 22820/50462 | Loss: 2.5686 | LR: 2.94e-04 | 5160.9ms/step | 11.6 samples/s +Step 22830/50462 | Loss: 2.5778 | LR: 2.94e-04 | 5158.9ms/step | 11.6 samples/s +Step 22840/50462 | Loss: 2.5582 | LR: 2.93e-04 | 5165.4ms/step | 11.6 samples/s +Step 22850/50462 | Loss: 2.5542 | LR: 2.93e-04 | 5161.7ms/step | 11.6 samples/s +Step 22860/50462 | Loss: 2.5819 | LR: 2.93e-04 | 5169.3ms/step | 11.6 samples/s +Step 22870/50462 | Loss: 2.5938 | LR: 2.93e-04 | 5168.0ms/step | 11.6 samples/s +Step 22880/50462 | Loss: 2.5974 | LR: 2.93e-04 | 5159.9ms/step | 11.6 samples/s +Step 22890/50462 | Loss: 2.5549 | LR: 2.93e-04 | 5172.0ms/step | 11.6 samples/s +Step 22900/50462 | Loss: 2.5583 | LR: 2.93e-04 | 5159.8ms/step | 11.6 samples/s +Step 22910/50462 | Loss: 2.5761 | LR: 2.92e-04 | 5151.3ms/step | 11.6 samples/s +Step 22920/50462 | Loss: 2.5393 | LR: 2.92e-04 | 5164.5ms/step | 11.6 samples/s +Step 22930/50462 | Loss: 2.5789 | LR: 2.92e-04 | 5165.1ms/step | 11.6 samples/s +Step 22940/50462 | Loss: 2.5544 | LR: 2.92e-04 | 5175.6ms/step | 11.6 samples/s +Step 22950/50462 | Loss: 2.5554 | LR: 2.92e-04 | 5152.2ms/step | 11.6 samples/s +Step 22960/50462 | Loss: 2.5575 | LR: 2.92e-04 | 5171.5ms/step | 11.6 samples/s +Step 22970/50462 | Loss: 2.5726 | LR: 2.91e-04 | 5157.5ms/step | 11.6 samples/s +Step 22980/50462 | Loss: 2.5743 | LR: 2.91e-04 | 5150.1ms/step | 11.7 samples/s +Step 22990/50462 | Loss: 2.5862 | LR: 2.91e-04 | 5163.4ms/step | 11.6 samples/s +Step 23000/50462 | Loss: 2.5572 | LR: 2.91e-04 | 5142.2ms/step | 11.7 samples/s +Validation at step 23000: Loss = 2.5845, Perplexity = 13.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-23000 +Step 23010/50462 | Loss: 2.5805 | LR: 2.91e-04 | 8609.0ms/step | 7.0 samples/s +Step 23020/50462 | Loss: 2.5399 | LR: 2.91e-04 | 5108.7ms/step | 11.7 samples/s +Step 23030/50462 | Loss: 2.5713 | LR: 2.91e-04 | 5115.3ms/step | 11.7 samples/s +Step 23040/50462 | Loss: 2.5718 | LR: 2.90e-04 | 5091.4ms/step | 11.8 samples/s +Step 23050/50462 | Loss: 2.5669 | LR: 2.90e-04 | 5115.7ms/step | 11.7 samples/s +Step 23060/50462 | Loss: 2.5297 | LR: 2.90e-04 | 5104.1ms/step | 11.8 samples/s +Step 23070/50462 | Loss: 2.5619 | LR: 2.90e-04 | 5109.5ms/step | 11.7 samples/s +Step 23080/50462 | Loss: 2.5755 | LR: 2.90e-04 | 5105.7ms/step | 11.8 samples/s +Step 23090/50462 | Loss: 2.5524 | LR: 2.90e-04 | 5109.3ms/step | 11.7 samples/s +Step 23100/50462 | Loss: 2.5524 | LR: 2.89e-04 | 5100.3ms/step | 11.8 samples/s +Step 23110/50462 | Loss: 2.5569 | LR: 2.89e-04 | 5123.7ms/step | 11.7 samples/s +Step 23120/50462 | Loss: 2.6075 | LR: 2.89e-04 | 5102.1ms/step | 11.8 samples/s +Step 23130/50462 | Loss: 2.5684 | LR: 2.89e-04 | 5111.7ms/step | 11.7 samples/s +Step 23140/50462 | Loss: 2.5718 | LR: 2.89e-04 | 5113.6ms/step | 11.7 samples/s +Step 23150/50462 | Loss: 2.5948 | LR: 2.89e-04 | 5109.2ms/step | 11.7 samples/s +Step 23160/50462 | Loss: 2.5669 | LR: 2.88e-04 | 5120.2ms/step | 11.7 samples/s +Step 23170/50462 | Loss: 2.5836 | LR: 2.88e-04 | 5109.2ms/step | 11.7 samples/s +Step 23180/50462 | Loss: 2.5606 | LR: 2.88e-04 | 5104.5ms/step | 11.8 samples/s +Step 23190/50462 | Loss: 2.5870 | LR: 2.88e-04 | 5112.0ms/step | 11.7 samples/s +Step 23200/50462 | Loss: 2.5788 | LR: 2.88e-04 | 5114.2ms/step | 11.7 samples/s +Step 23210/50462 | Loss: 2.5652 | LR: 2.88e-04 | 5105.4ms/step | 11.8 samples/s +Step 23220/50462 | Loss: 2.5777 | LR: 2.88e-04 | 5122.8ms/step | 11.7 samples/s +Step 23230/50462 | Loss: 2.5877 | LR: 2.87e-04 | 5114.6ms/step | 11.7 samples/s +Step 23240/50462 | Loss: 2.5434 | LR: 2.87e-04 | 5125.5ms/step | 11.7 samples/s +Step 23250/50462 | Loss: 2.5817 | LR: 2.87e-04 | 5103.7ms/step | 11.8 samples/s +Validation at step 23250: Loss = 2.5824, Perplexity = 13.23 +Step 23260/50462 | Loss: 2.5909 | LR: 2.87e-04 | 7670.8ms/step | 7.8 samples/s +Step 23270/50462 | Loss: 2.5675 | LR: 2.87e-04 | 5119.9ms/step | 11.7 samples/s +Step 23280/50462 | Loss: 2.5600 | LR: 2.87e-04 | 5130.2ms/step | 11.7 samples/s +Step 23290/50462 | Loss: 2.5823 | LR: 2.86e-04 | 5113.4ms/step | 11.7 samples/s +Step 23300/50462 | Loss: 2.5658 | LR: 2.86e-04 | 5128.0ms/step | 11.7 samples/s +Step 23310/50462 | Loss: 2.5903 | LR: 2.86e-04 | 5124.6ms/step | 11.7 samples/s +Step 23320/50462 | Loss: 2.5891 | LR: 2.86e-04 | 5127.0ms/step | 11.7 samples/s +Step 23330/50462 | Loss: 2.5752 | LR: 2.86e-04 | 5119.8ms/step | 11.7 samples/s +Step 23340/50462 | Loss: 2.5739 | LR: 2.86e-04 | 5124.7ms/step | 11.7 samples/s +Step 23350/50462 | Loss: 2.5697 | LR: 2.86e-04 | 5127.8ms/step | 11.7 samples/s +Step 23360/50462 | Loss: 2.5273 | LR: 2.85e-04 | 5122.2ms/step | 11.7 samples/s +Step 23370/50462 | Loss: 2.5794 | LR: 2.85e-04 | 5120.1ms/step | 11.7 samples/s +Step 23380/50462 | Loss: 2.5635 | LR: 2.85e-04 | 5121.3ms/step | 11.7 samples/s +Step 23390/50462 | Loss: 2.5563 | LR: 2.85e-04 | 5120.2ms/step | 11.7 samples/s +Step 23400/50462 | Loss: 2.5890 | LR: 2.85e-04 | 5130.1ms/step | 11.7 samples/s +Step 23410/50462 | Loss: 2.5528 | LR: 2.85e-04 | 5118.8ms/step | 11.7 samples/s +Step 23420/50462 | Loss: 2.5815 | LR: 2.84e-04 | 5105.9ms/step | 11.8 samples/s +Step 23430/50462 | Loss: 2.5610 | LR: 2.84e-04 | 5115.1ms/step | 11.7 samples/s +Step 23440/50462 | Loss: 2.5625 | LR: 2.84e-04 | 5133.2ms/step | 11.7 samples/s +Step 23450/50462 | Loss: 2.5530 | LR: 2.84e-04 | 5128.6ms/step | 11.7 samples/s +Step 23460/50462 | Loss: 2.6011 | LR: 2.84e-04 | 5117.7ms/step | 11.7 samples/s +Step 23470/50462 | Loss: 2.6080 | LR: 2.84e-04 | 5114.3ms/step | 11.7 samples/s +Step 23480/50462 | Loss: 2.5966 | LR: 2.83e-04 | 5119.2ms/step | 11.7 samples/s +Step 23490/50462 | Loss: 2.5475 | LR: 2.83e-04 | 5110.6ms/step | 11.7 samples/s +Step 23500/50462 | Loss: 2.5929 | LR: 2.83e-04 | 5114.9ms/step | 11.7 samples/s +Validation at step 23500: Loss = 2.5809, Perplexity = 13.21 +Step 23510/50462 | Loss: 2.5517 | LR: 2.83e-04 | 7652.8ms/step | 7.8 samples/s +Step 23520/50462 | Loss: 2.5706 | LR: 2.83e-04 | 5110.2ms/step | 11.7 samples/s +Step 23530/50462 | Loss: 2.5751 | LR: 2.83e-04 | 5118.5ms/step | 11.7 samples/s +Step 23540/50462 | Loss: 2.5852 | LR: 2.83e-04 | 5116.1ms/step | 11.7 samples/s +Step 23550/50462 | Loss: 2.5441 | LR: 2.82e-04 | 5099.1ms/step | 11.8 samples/s +Step 23560/50462 | Loss: 2.5532 | LR: 2.82e-04 | 5111.2ms/step | 11.7 samples/s +Step 23570/50462 | Loss: 2.5784 | LR: 2.82e-04 | 5102.8ms/step | 11.8 samples/s +Step 23580/50462 | Loss: 2.5383 | LR: 2.82e-04 | 5105.0ms/step | 11.8 samples/s +Step 23590/50462 | Loss: 2.5580 | LR: 2.82e-04 | 5109.5ms/step | 11.7 samples/s +Step 23600/50462 | Loss: 2.5891 | LR: 2.82e-04 | 5103.9ms/step | 11.8 samples/s +Step 23610/50462 | Loss: 2.5750 | LR: 2.81e-04 | 5106.1ms/step | 11.8 samples/s +Step 23620/50462 | Loss: 2.5892 | LR: 2.81e-04 | 5109.5ms/step | 11.7 samples/s +Step 23630/50462 | Loss: 2.5574 | LR: 2.81e-04 | 5131.4ms/step | 11.7 samples/s +Step 23640/50462 | Loss: 2.5797 | LR: 2.81e-04 | 5118.1ms/step | 11.7 samples/s +Step 23650/50462 | Loss: 2.5761 | LR: 2.81e-04 | 5108.9ms/step | 11.7 samples/s +Step 23660/50462 | Loss: 2.5473 | LR: 2.81e-04 | 5119.7ms/step | 11.7 samples/s +Step 23670/50462 | Loss: 2.5748 | LR: 2.81e-04 | 5120.8ms/step | 11.7 samples/s +Step 23680/50462 | Loss: 2.5535 | LR: 2.80e-04 | 5117.9ms/step | 11.7 samples/s +Step 23690/50462 | Loss: 2.5521 | LR: 2.80e-04 | 5121.8ms/step | 11.7 samples/s +Step 23700/50462 | Loss: 2.5619 | LR: 2.80e-04 | 5127.6ms/step | 11.7 samples/s +Step 23710/50462 | Loss: 2.5963 | LR: 2.80e-04 | 5127.2ms/step | 11.7 samples/s +Step 23720/50462 | Loss: 2.5196 | LR: 2.80e-04 | 5125.4ms/step | 11.7 samples/s +Step 23730/50462 | Loss: 2.5569 | LR: 2.80e-04 | 5113.3ms/step | 11.7 samples/s +Step 23740/50462 | Loss: 2.5814 | LR: 2.79e-04 | 5117.8ms/step | 11.7 samples/s +Step 23750/50462 | Loss: 2.5521 | LR: 2.79e-04 | 5122.4ms/step | 11.7 samples/s +Validation at step 23750: Loss = 2.5789, Perplexity = 13.18 +Step 23760/50462 | Loss: 2.5981 | LR: 2.79e-04 | 7668.4ms/step | 7.8 samples/s +Step 23770/50462 | Loss: 2.5481 | LR: 2.79e-04 | 5137.0ms/step | 11.7 samples/s +Step 23780/50462 | Loss: 2.5448 | LR: 2.79e-04 | 5134.6ms/step | 11.7 samples/s +Step 23790/50462 | Loss: 2.5314 | LR: 2.79e-04 | 5133.0ms/step | 11.7 samples/s +Step 23800/50462 | Loss: 2.5630 | LR: 2.78e-04 | 5136.8ms/step | 11.7 samples/s +Step 23810/50462 | Loss: 2.5592 | LR: 2.78e-04 | 5135.2ms/step | 11.7 samples/s +Step 23820/50462 | Loss: 2.5703 | LR: 2.78e-04 | 5136.6ms/step | 11.7 samples/s +Step 23830/50462 | Loss: 2.5350 | LR: 2.78e-04 | 5135.6ms/step | 11.7 samples/s +Step 23840/50462 | Loss: 2.5657 | LR: 2.78e-04 | 5134.9ms/step | 11.7 samples/s +Step 23850/50462 | Loss: 2.5796 | LR: 2.78e-04 | 5131.2ms/step | 11.7 samples/s +Step 23860/50462 | Loss: 2.5868 | LR: 2.78e-04 | 5139.6ms/step | 11.7 samples/s +Step 23870/50462 | Loss: 2.5804 | LR: 2.77e-04 | 5140.1ms/step | 11.7 samples/s +Step 23880/50462 | Loss: 2.5887 | LR: 2.77e-04 | 5136.0ms/step | 11.7 samples/s +Step 23890/50462 | Loss: 2.5653 | LR: 2.77e-04 | 5128.4ms/step | 11.7 samples/s +Step 23900/50462 | Loss: 2.5725 | LR: 2.77e-04 | 5147.6ms/step | 11.7 samples/s +Step 23910/50462 | Loss: 2.5556 | LR: 2.77e-04 | 5129.9ms/step | 11.7 samples/s +Step 23920/50462 | Loss: 2.5520 | LR: 2.77e-04 | 5124.9ms/step | 11.7 samples/s +Step 23930/50462 | Loss: 2.5688 | LR: 2.76e-04 | 5121.6ms/step | 11.7 samples/s +Step 23940/50462 | Loss: 2.5622 | LR: 2.76e-04 | 5127.0ms/step | 11.7 samples/s +Step 23950/50462 | Loss: 2.5635 | LR: 2.76e-04 | 5126.6ms/step | 11.7 samples/s +Step 23960/50462 | Loss: 2.5626 | LR: 2.76e-04 | 5124.5ms/step | 11.7 samples/s +Step 23970/50462 | Loss: 2.5331 | LR: 2.76e-04 | 5139.8ms/step | 11.7 samples/s +Step 23980/50462 | Loss: 2.5963 | LR: 2.76e-04 | 5129.0ms/step | 11.7 samples/s +Step 23990/50462 | Loss: 2.5716 | LR: 2.75e-04 | 5121.0ms/step | 11.7 samples/s +Step 24000/50462 | Loss: 2.5714 | LR: 2.75e-04 | 5123.2ms/step | 11.7 samples/s +Validation at step 24000: Loss = 2.5770, Perplexity = 13.16 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-24000 +Step 24010/50462 | Loss: 2.5618 | LR: 2.75e-04 | 8633.0ms/step | 7.0 samples/s +Step 24020/50462 | Loss: 2.5610 | LR: 2.75e-04 | 5118.8ms/step | 11.7 samples/s +Step 24030/50462 | Loss: 2.5369 | LR: 2.75e-04 | 5121.0ms/step | 11.7 samples/s +Step 24040/50462 | Loss: 2.5825 | LR: 2.75e-04 | 5129.3ms/step | 11.7 samples/s +Step 24050/50462 | Loss: 2.5439 | LR: 2.75e-04 | 5117.7ms/step | 11.7 samples/s +Step 24060/50462 | Loss: 2.5669 | LR: 2.74e-04 | 5137.1ms/step | 11.7 samples/s +Step 24070/50462 | Loss: 2.5811 | LR: 2.74e-04 | 5151.1ms/step | 11.6 samples/s +Step 24080/50462 | Loss: 2.5834 | LR: 2.74e-04 | 5130.8ms/step | 11.7 samples/s +Step 24090/50462 | Loss: 2.5616 | LR: 2.74e-04 | 5547.2ms/step | 10.8 samples/s +Step 24100/50462 | Loss: 2.5404 | LR: 2.74e-04 | 5145.6ms/step | 11.7 samples/s +Step 24110/50462 | Loss: 2.5630 | LR: 2.74e-04 | 5162.1ms/step | 11.6 samples/s +Step 24120/50462 | Loss: 2.5769 | LR: 2.73e-04 | 5169.9ms/step | 11.6 samples/s +Step 24130/50462 | Loss: 2.5704 | LR: 2.73e-04 | 5146.4ms/step | 11.7 samples/s +Step 24140/50462 | Loss: 2.5699 | LR: 2.73e-04 | 5164.2ms/step | 11.6 samples/s +Step 24150/50462 | Loss: 2.5619 | LR: 2.73e-04 | 5170.9ms/step | 11.6 samples/s +Step 24160/50462 | Loss: 2.5760 | LR: 2.73e-04 | 5166.5ms/step | 11.6 samples/s +Step 24170/50462 | Loss: 2.5377 | LR: 2.73e-04 | 5149.7ms/step | 11.7 samples/s +Step 24180/50462 | Loss: 2.5557 | LR: 2.72e-04 | 5576.1ms/step | 10.8 samples/s +Step 24190/50462 | Loss: 2.5628 | LR: 2.72e-04 | 5172.7ms/step | 11.6 samples/s +Step 24200/50462 | Loss: 2.5452 | LR: 2.72e-04 | 5622.5ms/step | 10.7 samples/s +Step 24210/50462 | Loss: 2.5608 | LR: 2.72e-04 | 5176.2ms/step | 11.6 samples/s +Step 24220/50462 | Loss: 2.5300 | LR: 2.72e-04 | 5190.8ms/step | 11.6 samples/s +Step 24230/50462 | Loss: 2.5807 | LR: 2.72e-04 | 5190.6ms/step | 11.6 samples/s +Step 24240/50462 | Loss: 2.5688 | LR: 2.72e-04 | 5180.1ms/step | 11.6 samples/s +Step 24250/50462 | Loss: 2.5562 | LR: 2.71e-04 | 5185.2ms/step | 11.6 samples/s +Validation at step 24250: Loss = 2.5744, Perplexity = 13.12 +Step 24260/50462 | Loss: 2.5632 | LR: 2.71e-04 | 7745.1ms/step | 7.7 samples/s +Step 24270/50462 | Loss: 2.5627 | LR: 2.71e-04 | 5177.1ms/step | 11.6 samples/s +Step 24280/50462 | Loss: 2.5719 | LR: 2.71e-04 | 5188.0ms/step | 11.6 samples/s +Step 24290/50462 | Loss: 2.5515 | LR: 2.71e-04 | 5181.0ms/step | 11.6 samples/s +Step 24300/50462 | Loss: 2.5470 | LR: 2.71e-04 | 5174.8ms/step | 11.6 samples/s +Step 24310/50462 | Loss: 2.5646 | LR: 2.70e-04 | 5190.4ms/step | 11.6 samples/s +Step 24320/50462 | Loss: 2.5637 | LR: 2.70e-04 | 5182.5ms/step | 11.6 samples/s +Step 24330/50462 | Loss: 2.5856 | LR: 2.70e-04 | 5175.0ms/step | 11.6 samples/s +Step 24340/50462 | Loss: 2.5298 | LR: 2.70e-04 | 5174.8ms/step | 11.6 samples/s +Step 24350/50462 | Loss: 2.5668 | LR: 2.70e-04 | 5171.8ms/step | 11.6 samples/s +Step 24360/50462 | Loss: 2.5563 | LR: 2.70e-04 | 5160.6ms/step | 11.6 samples/s +Step 24370/50462 | Loss: 2.5604 | LR: 2.70e-04 | 5163.8ms/step | 11.6 samples/s +Step 24380/50462 | Loss: 2.5841 | LR: 2.69e-04 | 5174.4ms/step | 11.6 samples/s +Step 24390/50462 | Loss: 2.5442 | LR: 2.69e-04 | 5182.4ms/step | 11.6 samples/s +Step 24400/50462 | Loss: 2.5667 | LR: 2.69e-04 | 5178.1ms/step | 11.6 samples/s +Step 24410/50462 | Loss: 2.5643 | LR: 2.69e-04 | 5174.7ms/step | 11.6 samples/s +Step 24420/50462 | Loss: 2.5604 | LR: 2.69e-04 | 5178.9ms/step | 11.6 samples/s +Step 24430/50462 | Loss: 2.5375 | LR: 2.69e-04 | 5167.7ms/step | 11.6 samples/s +Step 24440/50462 | Loss: 2.6017 | LR: 2.68e-04 | 5181.5ms/step | 11.6 samples/s +Step 24450/50462 | Loss: 2.5722 | LR: 2.68e-04 | 5176.6ms/step | 11.6 samples/s +Step 24460/50462 | Loss: 2.5483 | LR: 2.68e-04 | 5178.2ms/step | 11.6 samples/s +Step 24470/50462 | Loss: 2.5432 | LR: 2.68e-04 | 5171.6ms/step | 11.6 samples/s +Step 24480/50462 | Loss: 2.5559 | LR: 2.68e-04 | 5176.6ms/step | 11.6 samples/s +Step 24490/50462 | Loss: 2.5790 | LR: 2.68e-04 | 5175.1ms/step | 11.6 samples/s +Step 24500/50462 | Loss: 2.5722 | LR: 2.67e-04 | 5183.3ms/step | 11.6 samples/s +Validation at step 24500: Loss = 2.5720, Perplexity = 13.09 +Step 24510/50462 | Loss: 2.5585 | LR: 2.67e-04 | 7741.3ms/step | 7.8 samples/s +Step 24520/50462 | Loss: 2.5819 | LR: 2.67e-04 | 5189.2ms/step | 11.6 samples/s +Step 24530/50462 | Loss: 2.5611 | LR: 2.67e-04 | 5174.5ms/step | 11.6 samples/s +Step 24540/50462 | Loss: 2.5866 | LR: 2.67e-04 | 5183.4ms/step | 11.6 samples/s +Step 24550/50462 | Loss: 2.5597 | LR: 2.67e-04 | 5182.8ms/step | 11.6 samples/s +Step 24560/50462 | Loss: 2.6101 | LR: 2.67e-04 | 5182.7ms/step | 11.6 samples/s +Step 24570/50462 | Loss: 2.5623 | LR: 2.66e-04 | 5175.0ms/step | 11.6 samples/s +Step 24580/50462 | Loss: 2.5711 | LR: 2.66e-04 | 5179.1ms/step | 11.6 samples/s +Step 24590/50462 | Loss: 2.5742 | LR: 2.66e-04 | 5176.7ms/step | 11.6 samples/s +Step 24600/50462 | Loss: 2.5793 | LR: 2.66e-04 | 5168.7ms/step | 11.6 samples/s +Step 24610/50462 | Loss: 2.5451 | LR: 2.66e-04 | 5184.0ms/step | 11.6 samples/s +Step 24620/50462 | Loss: 2.5861 | LR: 2.66e-04 | 5168.8ms/step | 11.6 samples/s +Step 24630/50462 | Loss: 2.5335 | LR: 2.65e-04 | 5187.9ms/step | 11.6 samples/s +Step 24640/50462 | Loss: 2.5759 | LR: 2.65e-04 | 5190.0ms/step | 11.6 samples/s +Step 24650/50462 | Loss: 2.5347 | LR: 2.65e-04 | 5165.8ms/step | 11.6 samples/s +Step 24660/50462 | Loss: 2.5758 | LR: 2.65e-04 | 5146.3ms/step | 11.7 samples/s +Step 24670/50462 | Loss: 2.5654 | LR: 2.65e-04 | 5153.1ms/step | 11.6 samples/s +Step 24680/50462 | Loss: 2.5678 | LR: 2.65e-04 | 5184.9ms/step | 11.6 samples/s +Step 24690/50462 | Loss: 2.5618 | LR: 2.64e-04 | 5189.5ms/step | 11.6 samples/s +Step 24700/50462 | Loss: 2.5862 | LR: 2.64e-04 | 5174.8ms/step | 11.6 samples/s +Step 24710/50462 | Loss: 2.5581 | LR: 2.64e-04 | 5166.6ms/step | 11.6 samples/s +Step 24720/50462 | Loss: 2.5343 | LR: 2.64e-04 | 5171.8ms/step | 11.6 samples/s +Step 24730/50462 | Loss: 2.5506 | LR: 2.64e-04 | 5183.0ms/step | 11.6 samples/s +Step 24740/50462 | Loss: 2.5753 | LR: 2.64e-04 | 5171.7ms/step | 11.6 samples/s +Step 24750/50462 | Loss: 2.5514 | LR: 2.64e-04 | 5188.8ms/step | 11.6 samples/s +Validation at step 24750: Loss = 2.5695, Perplexity = 13.06 +Step 24760/50462 | Loss: 2.5741 | LR: 2.63e-04 | 7719.0ms/step | 7.8 samples/s +Step 24770/50462 | Loss: 2.5403 | LR: 2.63e-04 | 5168.8ms/step | 11.6 samples/s +Step 24780/50462 | Loss: 2.5444 | LR: 2.63e-04 | 5175.7ms/step | 11.6 samples/s +Step 24790/50462 | Loss: 2.5556 | LR: 2.63e-04 | 5182.8ms/step | 11.6 samples/s +Step 24800/50462 | Loss: 2.5819 | LR: 2.63e-04 | 5149.6ms/step | 11.7 samples/s +Step 24810/50462 | Loss: 2.5743 | LR: 2.63e-04 | 5147.4ms/step | 11.7 samples/s +Step 24820/50462 | Loss: 2.5699 | LR: 2.62e-04 | 5157.0ms/step | 11.6 samples/s +Step 24830/50462 | Loss: 2.5627 | LR: 2.62e-04 | 5151.6ms/step | 11.6 samples/s +Step 24840/50462 | Loss: 2.5363 | LR: 2.62e-04 | 5168.1ms/step | 11.6 samples/s +Step 24850/50462 | Loss: 2.5507 | LR: 2.62e-04 | 5163.8ms/step | 11.6 samples/s +Step 24860/50462 | Loss: 2.5788 | LR: 2.62e-04 | 5162.4ms/step | 11.6 samples/s +Step 24870/50462 | Loss: 2.5670 | LR: 2.62e-04 | 5142.2ms/step | 11.7 samples/s +Step 24880/50462 | Loss: 2.5128 | LR: 2.61e-04 | 5171.2ms/step | 11.6 samples/s +Step 24890/50462 | Loss: 2.5283 | LR: 2.61e-04 | 5182.9ms/step | 11.6 samples/s +Step 24900/50462 | Loss: 2.5825 | LR: 2.61e-04 | 5171.5ms/step | 11.6 samples/s +Step 24910/50462 | Loss: 2.5229 | LR: 2.61e-04 | 5179.6ms/step | 11.6 samples/s +Step 24920/50462 | Loss: 2.5309 | LR: 2.61e-04 | 5185.7ms/step | 11.6 samples/s +Step 24930/50462 | Loss: 2.5443 | LR: 2.61e-04 | 5163.5ms/step | 11.6 samples/s +Step 24940/50462 | Loss: 2.5773 | LR: 2.61e-04 | 5170.2ms/step | 11.6 samples/s +Step 24950/50462 | Loss: 2.5706 | LR: 2.60e-04 | 5173.3ms/step | 11.6 samples/s +Step 24960/50462 | Loss: 2.5279 | LR: 2.60e-04 | 5182.5ms/step | 11.6 samples/s +Step 24970/50462 | Loss: 2.5297 | LR: 2.60e-04 | 5167.8ms/step | 11.6 samples/s +Step 24980/50462 | Loss: 2.5655 | LR: 2.60e-04 | 5177.1ms/step | 11.6 samples/s +Step 24990/50462 | Loss: 2.5449 | LR: 2.60e-04 | 5178.6ms/step | 11.6 samples/s +Step 25000/50462 | Loss: 2.5650 | LR: 2.60e-04 | 5180.1ms/step | 11.6 samples/s +Validation at step 25000: Loss = 2.5676, Perplexity = 13.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-25000 +Step 25010/50462 | Loss: 2.5472 | LR: 2.59e-04 | 8621.6ms/step | 7.0 samples/s +Step 25020/50462 | Loss: 2.5542 | LR: 2.59e-04 | 5180.3ms/step | 11.6 samples/s +Step 25030/50462 | Loss: 2.5437 | LR: 2.59e-04 | 5192.1ms/step | 11.6 samples/s +Step 25040/50462 | Loss: 2.5653 | LR: 2.59e-04 | 5178.9ms/step | 11.6 samples/s +Step 25050/50462 | Loss: 2.5748 | LR: 2.59e-04 | 5189.1ms/step | 11.6 samples/s +Step 25060/50462 | Loss: 2.5548 | LR: 2.59e-04 | 5184.3ms/step | 11.6 samples/s +Step 25070/50462 | Loss: 2.5396 | LR: 2.58e-04 | 5191.5ms/step | 11.6 samples/s +Step 25080/50462 | Loss: 2.5294 | LR: 2.58e-04 | 5176.0ms/step | 11.6 samples/s +Step 25090/50462 | Loss: 2.5507 | LR: 2.58e-04 | 5192.2ms/step | 11.6 samples/s +Step 25100/50462 | Loss: 2.5579 | LR: 2.58e-04 | 5186.9ms/step | 11.6 samples/s +Step 25110/50462 | Loss: 2.5510 | LR: 2.58e-04 | 5178.1ms/step | 11.6 samples/s +Step 25120/50462 | Loss: 2.5738 | LR: 2.58e-04 | 5182.0ms/step | 11.6 samples/s +Step 25130/50462 | Loss: 2.5781 | LR: 2.58e-04 | 5177.0ms/step | 11.6 samples/s +Step 25140/50462 | Loss: 2.5626 | LR: 2.57e-04 | 5180.7ms/step | 11.6 samples/s +Step 25150/50462 | Loss: 2.5316 | LR: 2.57e-04 | 5177.6ms/step | 11.6 samples/s +Step 25160/50462 | Loss: 2.5334 | LR: 2.57e-04 | 5182.0ms/step | 11.6 samples/s +Step 25170/50462 | Loss: 2.5479 | LR: 2.57e-04 | 5170.5ms/step | 11.6 samples/s +Step 25180/50462 | Loss: 2.5505 | LR: 2.57e-04 | 5190.1ms/step | 11.6 samples/s +Step 25190/50462 | Loss: 2.5453 | LR: 2.57e-04 | 5181.2ms/step | 11.6 samples/s +Step 25200/50462 | Loss: 2.5791 | LR: 2.56e-04 | 5171.4ms/step | 11.6 samples/s +Step 25210/50462 | Loss: 2.5605 | LR: 2.56e-04 | 5178.1ms/step | 11.6 samples/s +Step 25220/50462 | Loss: 2.5984 | LR: 2.56e-04 | 5191.3ms/step | 11.6 samples/s +Step 25230/50462 | Loss: 2.5792 | LR: 2.56e-04 | 5173.4ms/step | 11.6 samples/s +Step 25240/50462 | Loss: 2.5249 | LR: 2.56e-04 | 5181.0ms/step | 11.6 samples/s +Step 25250/50462 | Loss: 2.5591 | LR: 2.56e-04 | 5175.8ms/step | 11.6 samples/s +Validation at step 25250: Loss = 2.5653, Perplexity = 13.00 +Step 25260/50462 | Loss: 2.5340 | LR: 2.55e-04 | 7730.3ms/step | 7.8 samples/s +Step 25270/50462 | Loss: 2.5598 | LR: 2.55e-04 | 5158.4ms/step | 11.6 samples/s +Step 25280/50462 | Loss: 2.5872 | LR: 2.55e-04 | 5174.8ms/step | 11.6 samples/s +Step 25290/50462 | Loss: 2.5562 | LR: 2.55e-04 | 5173.4ms/step | 11.6 samples/s +Step 25300/50462 | Loss: 2.5412 | LR: 2.55e-04 | 5187.3ms/step | 11.6 samples/s +Step 25310/50462 | Loss: 2.5290 | LR: 2.55e-04 | 5171.7ms/step | 11.6 samples/s +Step 25320/50462 | Loss: 2.5631 | LR: 2.55e-04 | 5171.3ms/step | 11.6 samples/s +Step 25330/50462 | Loss: 2.5368 | LR: 2.54e-04 | 5161.3ms/step | 11.6 samples/s +Step 25340/50462 | Loss: 2.5703 | LR: 2.54e-04 | 5174.7ms/step | 11.6 samples/s +Step 25350/50462 | Loss: 2.5105 | LR: 2.54e-04 | 5189.5ms/step | 11.6 samples/s +Step 25360/50462 | Loss: 2.5613 | LR: 2.54e-04 | 5169.7ms/step | 11.6 samples/s +Step 25370/50462 | Loss: 2.5633 | LR: 2.54e-04 | 5186.7ms/step | 11.6 samples/s +Step 25380/50462 | Loss: 2.6032 | LR: 2.54e-04 | 5185.3ms/step | 11.6 samples/s +Step 25390/50462 | Loss: 2.5653 | LR: 2.53e-04 | 5169.6ms/step | 11.6 samples/s +Step 25400/50462 | Loss: 2.5401 | LR: 2.53e-04 | 5172.5ms/step | 11.6 samples/s +Step 25410/50462 | Loss: 2.5332 | LR: 2.53e-04 | 5171.0ms/step | 11.6 samples/s +Step 25420/50462 | Loss: 2.5587 | LR: 2.53e-04 | 5179.1ms/step | 11.6 samples/s +Step 25430/50462 | Loss: 2.5481 | LR: 2.53e-04 | 5174.0ms/step | 11.6 samples/s +Step 25440/50462 | Loss: 2.5409 | LR: 2.53e-04 | 5174.4ms/step | 11.6 samples/s +Step 25450/50462 | Loss: 2.5452 | LR: 2.52e-04 | 5183.3ms/step | 11.6 samples/s +Step 25460/50462 | Loss: 2.5565 | LR: 2.52e-04 | 5173.9ms/step | 11.6 samples/s +Step 25470/50462 | Loss: 2.5560 | LR: 2.52e-04 | 5196.8ms/step | 11.5 samples/s +Step 25480/50462 | Loss: 2.5397 | LR: 2.52e-04 | 5180.4ms/step | 11.6 samples/s +Step 25490/50462 | Loss: 2.5445 | LR: 2.52e-04 | 5184.3ms/step | 11.6 samples/s +Step 25500/50462 | Loss: 2.5460 | LR: 2.52e-04 | 5182.8ms/step | 11.6 samples/s +Validation at step 25500: Loss = 2.5632, Perplexity = 12.98 +Step 25510/50462 | Loss: 2.5406 | LR: 2.52e-04 | 7736.9ms/step | 7.8 samples/s +Step 25520/50462 | Loss: 2.5477 | LR: 2.51e-04 | 5180.1ms/step | 11.6 samples/s +Step 25530/50462 | Loss: 2.5760 | LR: 2.51e-04 | 5161.7ms/step | 11.6 samples/s +Step 25540/50462 | Loss: 2.5709 | LR: 2.51e-04 | 5169.8ms/step | 11.6 samples/s +Step 25550/50462 | Loss: 2.5650 | LR: 2.51e-04 | 5168.3ms/step | 11.6 samples/s +Step 25560/50462 | Loss: 2.5625 | LR: 2.51e-04 | 5151.8ms/step | 11.6 samples/s +Step 25570/50462 | Loss: 2.5661 | LR: 2.51e-04 | 5149.1ms/step | 11.7 samples/s +Step 25580/50462 | Loss: 2.5891 | LR: 2.50e-04 | 5156.9ms/step | 11.6 samples/s +Step 25590/50462 | Loss: 2.5791 | LR: 2.50e-04 | 5175.2ms/step | 11.6 samples/s +Step 25600/50462 | Loss: 2.5467 | LR: 2.50e-04 | 5149.7ms/step | 11.7 samples/s +Step 25610/50462 | Loss: 2.5632 | LR: 2.50e-04 | 5150.0ms/step | 11.7 samples/s +Step 25620/50462 | Loss: 2.5511 | LR: 2.50e-04 | 5174.9ms/step | 11.6 samples/s +Step 25630/50462 | Loss: 2.5452 | LR: 2.50e-04 | 5177.6ms/step | 11.6 samples/s +Step 25640/50462 | Loss: 2.5777 | LR: 2.49e-04 | 5178.4ms/step | 11.6 samples/s +Step 25650/50462 | Loss: 2.5393 | LR: 2.49e-04 | 5152.6ms/step | 11.6 samples/s +Step 25660/50462 | Loss: 2.5177 | LR: 2.49e-04 | 5156.7ms/step | 11.6 samples/s +Step 25670/50462 | Loss: 2.5437 | LR: 2.49e-04 | 5154.1ms/step | 11.6 samples/s +Step 25680/50462 | Loss: 2.5617 | LR: 2.49e-04 | 5144.9ms/step | 11.7 samples/s +Step 25690/50462 | Loss: 2.5642 | LR: 2.49e-04 | 5154.8ms/step | 11.6 samples/s +Step 25700/50462 | Loss: 2.5543 | LR: 2.49e-04 | 5181.6ms/step | 11.6 samples/s +Step 25710/50462 | Loss: 2.5298 | LR: 2.48e-04 | 5143.7ms/step | 11.7 samples/s +Step 25720/50462 | Loss: 2.5322 | LR: 2.48e-04 | 5161.8ms/step | 11.6 samples/s +Step 25730/50462 | Loss: 2.5629 | LR: 2.48e-04 | 5154.4ms/step | 11.6 samples/s +Step 25740/50462 | Loss: 2.5588 | LR: 2.48e-04 | 5169.9ms/step | 11.6 samples/s +Step 25750/50462 | Loss: 2.5696 | LR: 2.48e-04 | 5165.6ms/step | 11.6 samples/s +Validation at step 25750: Loss = 2.5613, Perplexity = 12.95 +Step 25760/50462 | Loss: 2.5447 | LR: 2.48e-04 | 7707.0ms/step | 7.8 samples/s +Step 25770/50462 | Loss: 2.5499 | LR: 2.47e-04 | 5158.7ms/step | 11.6 samples/s +Step 25780/50462 | Loss: 2.5761 | LR: 2.47e-04 | 5160.2ms/step | 11.6 samples/s +Step 25790/50462 | Loss: 2.5461 | LR: 2.47e-04 | 5145.1ms/step | 11.7 samples/s +Step 25800/50462 | Loss: 2.5369 | LR: 2.47e-04 | 5154.3ms/step | 11.6 samples/s +Step 25810/50462 | Loss: 2.5475 | LR: 2.47e-04 | 5149.1ms/step | 11.7 samples/s +Step 25820/50462 | Loss: 2.5503 | LR: 2.47e-04 | 5154.8ms/step | 11.6 samples/s +Step 25830/50462 | Loss: 2.5621 | LR: 2.46e-04 | 5160.0ms/step | 11.6 samples/s +Step 25840/50462 | Loss: 2.5814 | LR: 2.46e-04 | 5144.7ms/step | 11.7 samples/s +Step 25850/50462 | Loss: 2.5515 | LR: 2.46e-04 | 5174.8ms/step | 11.6 samples/s +Step 25860/50462 | Loss: 2.5402 | LR: 2.46e-04 | 5175.7ms/step | 11.6 samples/s +Step 25870/50462 | Loss: 2.5576 | LR: 2.46e-04 | 5160.1ms/step | 11.6 samples/s +Step 25880/50462 | Loss: 2.5606 | LR: 2.46e-04 | 5161.3ms/step | 11.6 samples/s +Step 25890/50462 | Loss: 2.5326 | LR: 2.46e-04 | 5158.9ms/step | 11.6 samples/s +Step 25900/50462 | Loss: 2.5228 | LR: 2.45e-04 | 5161.3ms/step | 11.6 samples/s +Step 25910/50462 | Loss: 2.5260 | LR: 2.45e-04 | 5171.8ms/step | 11.6 samples/s +Step 25920/50462 | Loss: 2.5578 | LR: 2.45e-04 | 5158.5ms/step | 11.6 samples/s +Step 25930/50462 | Loss: 2.5483 | LR: 2.45e-04 | 5164.5ms/step | 11.6 samples/s +Step 25940/50462 | Loss: 2.5444 | LR: 2.45e-04 | 5167.7ms/step | 11.6 samples/s +Step 25950/50462 | Loss: 2.5699 | LR: 2.45e-04 | 5169.5ms/step | 11.6 samples/s +Step 25960/50462 | Loss: 2.5550 | LR: 2.44e-04 | 5167.5ms/step | 11.6 samples/s +Step 25970/50462 | Loss: 2.5506 | LR: 2.44e-04 | 5144.8ms/step | 11.7 samples/s +Step 25980/50462 | Loss: 2.5340 | LR: 2.44e-04 | 5157.9ms/step | 11.6 samples/s +Step 25990/50462 | Loss: 2.5377 | LR: 2.44e-04 | 5158.3ms/step | 11.6 samples/s +Step 26000/50462 | Loss: 2.5361 | LR: 2.44e-04 | 5169.0ms/step | 11.6 samples/s +Validation at step 26000: Loss = 2.5596, Perplexity = 12.93 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-26000 +Step 26010/50462 | Loss: 2.5687 | LR: 2.44e-04 | 8598.3ms/step | 7.0 samples/s +Step 26020/50462 | Loss: 2.5689 | LR: 2.43e-04 | 5176.6ms/step | 11.6 samples/s +Step 26030/50462 | Loss: 2.5548 | LR: 2.43e-04 | 5196.4ms/step | 11.5 samples/s +Step 26040/50462 | Loss: 2.5358 | LR: 2.43e-04 | 5183.3ms/step | 11.6 samples/s +Step 26050/50462 | Loss: 2.5330 | LR: 2.43e-04 | 5179.2ms/step | 11.6 samples/s +Step 26060/50462 | Loss: 2.5385 | LR: 2.43e-04 | 5186.5ms/step | 11.6 samples/s +Step 26070/50462 | Loss: 2.5710 | LR: 2.43e-04 | 5180.3ms/step | 11.6 samples/s +Step 26080/50462 | Loss: 2.5234 | LR: 2.43e-04 | 5186.4ms/step | 11.6 samples/s +Step 26090/50462 | Loss: 2.5608 | LR: 2.42e-04 | 5174.8ms/step | 11.6 samples/s +Step 26100/50462 | Loss: 2.5621 | LR: 2.42e-04 | 5174.8ms/step | 11.6 samples/s +Step 26110/50462 | Loss: 2.5647 | LR: 2.42e-04 | 5187.8ms/step | 11.6 samples/s +Step 26120/50462 | Loss: 2.5686 | LR: 2.42e-04 | 5184.9ms/step | 11.6 samples/s +Step 26130/50462 | Loss: 2.5395 | LR: 2.42e-04 | 5172.7ms/step | 11.6 samples/s +Step 26140/50462 | Loss: 2.5532 | LR: 2.42e-04 | 5179.1ms/step | 11.6 samples/s +Step 26150/50462 | Loss: 2.5494 | LR: 2.41e-04 | 5189.4ms/step | 11.6 samples/s +Step 26160/50462 | Loss: 2.5405 | LR: 2.41e-04 | 5583.1ms/step | 10.7 samples/s +Step 26170/50462 | Loss: 2.5574 | LR: 2.41e-04 | 5183.2ms/step | 11.6 samples/s +Step 26180/50462 | Loss: 2.5488 | LR: 2.41e-04 | 5169.8ms/step | 11.6 samples/s +Step 26190/50462 | Loss: 2.5370 | LR: 2.41e-04 | 5166.7ms/step | 11.6 samples/s +Step 26200/50462 | Loss: 2.5482 | LR: 2.41e-04 | 5169.0ms/step | 11.6 samples/s +Step 26210/50462 | Loss: 2.5326 | LR: 2.40e-04 | 5153.1ms/step | 11.6 samples/s +Step 26220/50462 | Loss: 2.5396 | LR: 2.40e-04 | 5149.5ms/step | 11.7 samples/s +Step 26230/50462 | Loss: 2.5447 | LR: 2.40e-04 | 5149.3ms/step | 11.7 samples/s +Step 26240/50462 | Loss: 2.5528 | LR: 2.40e-04 | 5180.7ms/step | 11.6 samples/s +Step 26250/50462 | Loss: 2.5632 | LR: 2.40e-04 | 5115.9ms/step | 11.7 samples/s +Validation at step 26250: Loss = 2.5574, Perplexity = 12.90 +Step 26260/50462 | Loss: 2.5477 | LR: 2.40e-04 | 7658.0ms/step | 7.8 samples/s +Step 26270/50462 | Loss: 2.5169 | LR: 2.40e-04 | 5129.2ms/step | 11.7 samples/s +Step 26280/50462 | Loss: 2.5392 | LR: 2.39e-04 | 5134.7ms/step | 11.7 samples/s +Step 26290/50462 | Loss: 2.5466 | LR: 2.39e-04 | 5557.2ms/step | 10.8 samples/s +Step 26300/50462 | Loss: 2.5485 | LR: 2.39e-04 | 5180.1ms/step | 11.6 samples/s +Step 26310/50462 | Loss: 2.5400 | LR: 2.39e-04 | 5621.7ms/step | 10.7 samples/s +Step 26320/50462 | Loss: 2.5514 | LR: 2.39e-04 | 5160.4ms/step | 11.6 samples/s +Step 26330/50462 | Loss: 2.5501 | LR: 2.39e-04 | 5139.6ms/step | 11.7 samples/s +Step 26340/50462 | Loss: 2.5643 | LR: 2.38e-04 | 5118.9ms/step | 11.7 samples/s +Step 26350/50462 | Loss: 2.5647 | LR: 2.38e-04 | 5130.6ms/step | 11.7 samples/s +Step 26360/50462 | Loss: 2.5589 | LR: 2.38e-04 | 5171.1ms/step | 11.6 samples/s +Step 26370/50462 | Loss: 2.5311 | LR: 2.38e-04 | 5124.1ms/step | 11.7 samples/s +Step 26380/50462 | Loss: 2.5044 | LR: 2.38e-04 | 5119.2ms/step | 11.7 samples/s +Step 26390/50462 | Loss: 2.5429 | LR: 2.38e-04 | 5147.1ms/step | 11.7 samples/s +Step 26400/50462 | Loss: 2.5461 | LR: 2.37e-04 | 5187.3ms/step | 11.6 samples/s +Step 26410/50462 | Loss: 2.5566 | LR: 2.37e-04 | 5166.9ms/step | 11.6 samples/s +Step 26420/50462 | Loss: 2.5436 | LR: 2.37e-04 | 5149.4ms/step | 11.7 samples/s +Step 26430/50462 | Loss: 2.5421 | LR: 2.37e-04 | 5124.0ms/step | 11.7 samples/s +Step 26440/50462 | Loss: 2.5612 | LR: 2.37e-04 | 5134.3ms/step | 11.7 samples/s +Step 26450/50462 | Loss: 2.5575 | LR: 2.37e-04 | 5134.1ms/step | 11.7 samples/s +Step 26460/50462 | Loss: 2.5462 | LR: 2.37e-04 | 5124.1ms/step | 11.7 samples/s +Step 26470/50462 | Loss: 2.5834 | LR: 2.36e-04 | 5136.9ms/step | 11.7 samples/s +Step 26480/50462 | Loss: 2.5303 | LR: 2.36e-04 | 5121.1ms/step | 11.7 samples/s +Step 26490/50462 | Loss: 2.5400 | LR: 2.36e-04 | 5138.9ms/step | 11.7 samples/s +Step 26500/50462 | Loss: 2.5457 | LR: 2.36e-04 | 5121.1ms/step | 11.7 samples/s +Validation at step 26500: Loss = 2.5551, Perplexity = 12.87 +Step 26510/50462 | Loss: 2.5541 | LR: 2.36e-04 | 7673.4ms/step | 7.8 samples/s +Step 26520/50462 | Loss: 2.5530 | LR: 2.36e-04 | 5148.0ms/step | 11.7 samples/s +Step 26530/50462 | Loss: 2.5395 | LR: 2.35e-04 | 5144.2ms/step | 11.7 samples/s +Step 26540/50462 | Loss: 2.5216 | LR: 2.35e-04 | 5143.1ms/step | 11.7 samples/s +Step 26550/50462 | Loss: 2.5699 | LR: 2.35e-04 | 5133.6ms/step | 11.7 samples/s +Step 26560/50462 | Loss: 2.5644 | LR: 2.35e-04 | 5131.7ms/step | 11.7 samples/s +Step 26570/50462 | Loss: 2.5458 | LR: 2.35e-04 | 5127.4ms/step | 11.7 samples/s +Step 26580/50462 | Loss: 2.5438 | LR: 2.35e-04 | 5131.8ms/step | 11.7 samples/s +Step 26590/50462 | Loss: 2.5273 | LR: 2.34e-04 | 5124.5ms/step | 11.7 samples/s +Step 26600/50462 | Loss: 2.5278 | LR: 2.34e-04 | 5125.5ms/step | 11.7 samples/s +Step 26610/50462 | Loss: 2.5726 | LR: 2.34e-04 | 5125.5ms/step | 11.7 samples/s +Step 26620/50462 | Loss: 2.5411 | LR: 2.34e-04 | 5168.2ms/step | 11.6 samples/s +Step 26630/50462 | Loss: 2.5470 | LR: 2.34e-04 | 5175.2ms/step | 11.6 samples/s +Step 26640/50462 | Loss: 2.5520 | LR: 2.34e-04 | 5171.4ms/step | 11.6 samples/s +Step 26650/50462 | Loss: 2.5365 | LR: 2.34e-04 | 5179.5ms/step | 11.6 samples/s +Step 26660/50462 | Loss: 2.5122 | LR: 2.33e-04 | 5154.3ms/step | 11.6 samples/s +Step 26670/50462 | Loss: 2.5106 | LR: 2.33e-04 | 5152.2ms/step | 11.6 samples/s +Step 26680/50462 | Loss: 2.5066 | LR: 2.33e-04 | 5114.7ms/step | 11.7 samples/s +Step 26690/50462 | Loss: 2.5298 | LR: 2.33e-04 | 5132.2ms/step | 11.7 samples/s +Step 26700/50462 | Loss: 2.5452 | LR: 2.33e-04 | 5129.0ms/step | 11.7 samples/s +Step 26710/50462 | Loss: 2.5503 | LR: 2.33e-04 | 5124.7ms/step | 11.7 samples/s +Step 26720/50462 | Loss: 2.4937 | LR: 2.32e-04 | 5121.6ms/step | 11.7 samples/s +Step 26730/50462 | Loss: 2.5271 | LR: 2.32e-04 | 5117.3ms/step | 11.7 samples/s +Step 26740/50462 | Loss: 2.5442 | LR: 2.32e-04 | 5112.5ms/step | 11.7 samples/s +Step 26750/50462 | Loss: 2.5547 | LR: 2.32e-04 | 5116.0ms/step | 11.7 samples/s +Validation at step 26750: Loss = 2.5540, Perplexity = 12.86 +Step 26760/50462 | Loss: 2.5513 | LR: 2.32e-04 | 7652.6ms/step | 7.8 samples/s +Step 26770/50462 | Loss: 2.5246 | LR: 2.32e-04 | 5112.0ms/step | 11.7 samples/s +Step 26780/50462 | Loss: 2.5131 | LR: 2.31e-04 | 5120.4ms/step | 11.7 samples/s +Step 26790/50462 | Loss: 2.5800 | LR: 2.31e-04 | 5156.6ms/step | 11.6 samples/s +Step 26800/50462 | Loss: 2.5669 | LR: 2.31e-04 | 5157.8ms/step | 11.6 samples/s +Step 26810/50462 | Loss: 2.5778 | LR: 2.31e-04 | 5127.7ms/step | 11.7 samples/s +Step 26820/50462 | Loss: 2.5327 | LR: 2.31e-04 | 5120.4ms/step | 11.7 samples/s +Step 26830/50462 | Loss: 2.5781 | LR: 2.31e-04 | 5142.4ms/step | 11.7 samples/s +Step 26840/50462 | Loss: 2.5045 | LR: 2.31e-04 | 5114.7ms/step | 11.7 samples/s +Step 26850/50462 | Loss: 2.5702 | LR: 2.30e-04 | 5128.3ms/step | 11.7 samples/s +Step 26860/50462 | Loss: 2.5505 | LR: 2.30e-04 | 5131.6ms/step | 11.7 samples/s +Step 26870/50462 | Loss: 2.5307 | LR: 2.30e-04 | 5125.6ms/step | 11.7 samples/s +Step 26880/50462 | Loss: 2.5233 | LR: 2.30e-04 | 5122.9ms/step | 11.7 samples/s +Step 26890/50462 | Loss: 2.5309 | LR: 2.30e-04 | 5134.0ms/step | 11.7 samples/s +Step 26900/50462 | Loss: 2.5377 | LR: 2.30e-04 | 5132.2ms/step | 11.7 samples/s +Step 26910/50462 | Loss: 2.5521 | LR: 2.29e-04 | 5130.4ms/step | 11.7 samples/s +Step 26920/50462 | Loss: 2.5797 | LR: 2.29e-04 | 5129.0ms/step | 11.7 samples/s +Step 26930/50462 | Loss: 2.5346 | LR: 2.29e-04 | 5129.1ms/step | 11.7 samples/s +Step 26940/50462 | Loss: 2.5380 | LR: 2.29e-04 | 5128.6ms/step | 11.7 samples/s +Step 26950/50462 | Loss: 2.5384 | LR: 2.29e-04 | 5138.1ms/step | 11.7 samples/s +Step 26960/50462 | Loss: 2.5395 | LR: 2.29e-04 | 5144.9ms/step | 11.7 samples/s +Step 26970/50462 | Loss: 2.5481 | LR: 2.28e-04 | 5157.5ms/step | 11.6 samples/s +Step 26980/50462 | Loss: 2.5114 | LR: 2.28e-04 | 5129.7ms/step | 11.7 samples/s +Step 26990/50462 | Loss: 2.5513 | LR: 2.28e-04 | 5126.3ms/step | 11.7 samples/s +Step 27000/50462 | Loss: 2.5593 | LR: 2.28e-04 | 5138.3ms/step | 11.7 samples/s +Validation at step 27000: Loss = 2.5517, Perplexity = 12.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-27000 +Step 27010/50462 | Loss: 2.5197 | LR: 2.28e-04 | 8650.6ms/step | 6.9 samples/s +Step 27020/50462 | Loss: 2.5553 | LR: 2.28e-04 | 5172.2ms/step | 11.6 samples/s +Step 27030/50462 | Loss: 2.5659 | LR: 2.28e-04 | 5173.3ms/step | 11.6 samples/s +Step 27040/50462 | Loss: 2.5529 | LR: 2.27e-04 | 5186.6ms/step | 11.6 samples/s +Step 27050/50462 | Loss: 2.5453 | LR: 2.27e-04 | 5184.2ms/step | 11.6 samples/s +Step 27060/50462 | Loss: 2.5374 | LR: 2.27e-04 | 5172.4ms/step | 11.6 samples/s +Step 27070/50462 | Loss: 2.5377 | LR: 2.27e-04 | 5168.4ms/step | 11.6 samples/s +Step 27080/50462 | Loss: 2.5469 | LR: 2.27e-04 | 5184.1ms/step | 11.6 samples/s +Step 27090/50462 | Loss: 2.5296 | LR: 2.27e-04 | 5179.0ms/step | 11.6 samples/s +Step 27100/50462 | Loss: 2.5463 | LR: 2.26e-04 | 5178.9ms/step | 11.6 samples/s +Step 27110/50462 | Loss: 2.5380 | LR: 2.26e-04 | 5196.1ms/step | 11.5 samples/s +Step 27120/50462 | Loss: 2.5473 | LR: 2.26e-04 | 5176.0ms/step | 11.6 samples/s +Step 27130/50462 | Loss: 2.5615 | LR: 2.26e-04 | 5173.9ms/step | 11.6 samples/s +Step 27140/50462 | Loss: 2.5230 | LR: 2.26e-04 | 5183.4ms/step | 11.6 samples/s +Step 27150/50462 | Loss: 2.5297 | LR: 2.26e-04 | 5184.5ms/step | 11.6 samples/s +Step 27160/50462 | Loss: 2.5379 | LR: 2.25e-04 | 5187.0ms/step | 11.6 samples/s +Step 27170/50462 | Loss: 2.5679 | LR: 2.25e-04 | 5177.0ms/step | 11.6 samples/s +Step 27180/50462 | Loss: 2.5514 | LR: 2.25e-04 | 5185.5ms/step | 11.6 samples/s +Step 27190/50462 | Loss: 2.5572 | LR: 2.25e-04 | 5171.1ms/step | 11.6 samples/s +Step 27200/50462 | Loss: 2.5031 | LR: 2.25e-04 | 5175.6ms/step | 11.6 samples/s +Step 27210/50462 | Loss: 2.5413 | LR: 2.25e-04 | 5153.8ms/step | 11.6 samples/s +Step 27220/50462 | Loss: 2.5279 | LR: 2.25e-04 | 5169.8ms/step | 11.6 samples/s +Step 27230/50462 | Loss: 2.5719 | LR: 2.24e-04 | 5153.2ms/step | 11.6 samples/s +Step 27240/50462 | Loss: 2.5425 | LR: 2.24e-04 | 5147.3ms/step | 11.7 samples/s +Step 27250/50462 | Loss: 2.5573 | LR: 2.24e-04 | 5163.6ms/step | 11.6 samples/s +Validation at step 27250: Loss = 2.5498, Perplexity = 12.80 +Step 27260/50462 | Loss: 2.5451 | LR: 2.24e-04 | 7724.2ms/step | 7.8 samples/s +Step 27270/50462 | Loss: 2.5502 | LR: 2.24e-04 | 5176.4ms/step | 11.6 samples/s +Step 27280/50462 | Loss: 2.5222 | LR: 2.24e-04 | 5182.6ms/step | 11.6 samples/s +Step 27290/50462 | Loss: 2.5435 | LR: 2.23e-04 | 5174.4ms/step | 11.6 samples/s +Step 27300/50462 | Loss: 2.5034 | LR: 2.23e-04 | 5176.1ms/step | 11.6 samples/s +Step 27310/50462 | Loss: 2.4974 | LR: 2.23e-04 | 5164.1ms/step | 11.6 samples/s +Step 27320/50462 | Loss: 2.5408 | LR: 2.23e-04 | 5179.3ms/step | 11.6 samples/s +Step 27330/50462 | Loss: 2.5690 | LR: 2.23e-04 | 5173.6ms/step | 11.6 samples/s +Step 27340/50462 | Loss: 2.5308 | LR: 2.23e-04 | 5175.2ms/step | 11.6 samples/s +Step 27350/50462 | Loss: 2.5195 | LR: 2.23e-04 | 5185.9ms/step | 11.6 samples/s +Step 27360/50462 | Loss: 2.4901 | LR: 2.22e-04 | 5175.0ms/step | 11.6 samples/s +Step 27370/50462 | Loss: 2.5291 | LR: 2.22e-04 | 5177.7ms/step | 11.6 samples/s +Step 27380/50462 | Loss: 2.5219 | LR: 2.22e-04 | 5178.2ms/step | 11.6 samples/s +Step 27390/50462 | Loss: 2.5375 | LR: 2.22e-04 | 5175.5ms/step | 11.6 samples/s +Step 27400/50462 | Loss: 2.5529 | LR: 2.22e-04 | 5175.2ms/step | 11.6 samples/s +Step 27410/50462 | Loss: 2.5184 | LR: 2.22e-04 | 5176.8ms/step | 11.6 samples/s +Step 27420/50462 | Loss: 2.5204 | LR: 2.21e-04 | 5176.1ms/step | 11.6 samples/s +Step 27430/50462 | Loss: 2.5438 | LR: 2.21e-04 | 5161.2ms/step | 11.6 samples/s +Step 27440/50462 | Loss: 2.5279 | LR: 2.21e-04 | 5179.2ms/step | 11.6 samples/s +Step 27450/50462 | Loss: 2.5259 | LR: 2.21e-04 | 5180.4ms/step | 11.6 samples/s +Step 27460/50462 | Loss: 2.5619 | LR: 2.21e-04 | 5164.0ms/step | 11.6 samples/s +Step 27470/50462 | Loss: 2.5165 | LR: 2.21e-04 | 5181.3ms/step | 11.6 samples/s +Step 27480/50462 | Loss: 2.5330 | LR: 2.20e-04 | 5181.0ms/step | 11.6 samples/s +Step 27490/50462 | Loss: 2.5054 | LR: 2.20e-04 | 5161.0ms/step | 11.6 samples/s +Step 27500/50462 | Loss: 2.5430 | LR: 2.20e-04 | 5175.0ms/step | 11.6 samples/s +Validation at step 27500: Loss = 2.5481, Perplexity = 12.78 +Step 27510/50462 | Loss: 2.5351 | LR: 2.20e-04 | 7737.8ms/step | 7.8 samples/s +Step 27520/50462 | Loss: 2.5380 | LR: 2.20e-04 | 5179.0ms/step | 11.6 samples/s +Step 27530/50462 | Loss: 2.5607 | LR: 2.20e-04 | 5174.5ms/step | 11.6 samples/s +Step 27540/50462 | Loss: 2.5543 | LR: 2.20e-04 | 5187.5ms/step | 11.6 samples/s +Step 27550/50462 | Loss: 2.5333 | LR: 2.19e-04 | 5180.5ms/step | 11.6 samples/s +Step 27560/50462 | Loss: 2.5432 | LR: 2.19e-04 | 5180.7ms/step | 11.6 samples/s +Step 27570/50462 | Loss: 2.5370 | LR: 2.19e-04 | 5200.4ms/step | 11.5 samples/s +Step 27580/50462 | Loss: 2.5209 | LR: 2.19e-04 | 5170.3ms/step | 11.6 samples/s +Step 27590/50462 | Loss: 2.5446 | LR: 2.19e-04 | 5183.2ms/step | 11.6 samples/s +Step 27600/50462 | Loss: 2.5417 | LR: 2.19e-04 | 5174.4ms/step | 11.6 samples/s +Step 27610/50462 | Loss: 2.5259 | LR: 2.18e-04 | 5187.2ms/step | 11.6 samples/s +Step 27620/50462 | Loss: 2.5483 | LR: 2.18e-04 | 5175.1ms/step | 11.6 samples/s +Step 27630/50462 | Loss: 2.5262 | LR: 2.18e-04 | 5172.5ms/step | 11.6 samples/s +Step 27640/50462 | Loss: 2.5289 | LR: 2.18e-04 | 5178.2ms/step | 11.6 samples/s +Step 27650/50462 | Loss: 2.5160 | LR: 2.18e-04 | 5179.8ms/step | 11.6 samples/s +Step 27660/50462 | Loss: 2.5300 | LR: 2.18e-04 | 5203.0ms/step | 11.5 samples/s +Step 27670/50462 | Loss: 2.5355 | LR: 2.17e-04 | 5171.9ms/step | 11.6 samples/s +Step 27680/50462 | Loss: 2.5288 | LR: 2.17e-04 | 5191.9ms/step | 11.6 samples/s +Step 27690/50462 | Loss: 2.5191 | LR: 2.17e-04 | 5178.2ms/step | 11.6 samples/s +Step 27700/50462 | Loss: 2.5326 | LR: 2.17e-04 | 5178.4ms/step | 11.6 samples/s +Step 27710/50462 | Loss: 2.5313 | LR: 2.17e-04 | 5165.2ms/step | 11.6 samples/s +Step 27720/50462 | Loss: 2.5481 | LR: 2.17e-04 | 5197.6ms/step | 11.5 samples/s +Step 27730/50462 | Loss: 2.5483 | LR: 2.17e-04 | 5169.0ms/step | 11.6 samples/s +Step 27740/50462 | Loss: 2.5329 | LR: 2.16e-04 | 5185.5ms/step | 11.6 samples/s +Step 27750/50462 | Loss: 2.5406 | LR: 2.16e-04 | 5170.8ms/step | 11.6 samples/s +Validation at step 27750: Loss = 2.5459, Perplexity = 12.75 +Step 27760/50462 | Loss: 2.5294 | LR: 2.16e-04 | 7730.5ms/step | 7.8 samples/s +Step 27770/50462 | Loss: 2.5133 | LR: 2.16e-04 | 5179.9ms/step | 11.6 samples/s +Step 27780/50462 | Loss: 2.5513 | LR: 2.16e-04 | 5174.9ms/step | 11.6 samples/s +Step 27790/50462 | Loss: 2.5027 | LR: 2.16e-04 | 5176.0ms/step | 11.6 samples/s +Step 27800/50462 | Loss: 2.5355 | LR: 2.15e-04 | 5178.6ms/step | 11.6 samples/s +Step 27810/50462 | Loss: 2.5424 | LR: 2.15e-04 | 5170.7ms/step | 11.6 samples/s +Step 27820/50462 | Loss: 2.5275 | LR: 2.15e-04 | 5179.3ms/step | 11.6 samples/s +Step 27830/50462 | Loss: 2.5409 | LR: 2.15e-04 | 5172.2ms/step | 11.6 samples/s +Step 27840/50462 | Loss: 2.5223 | LR: 2.15e-04 | 5165.3ms/step | 11.6 samples/s +Step 27850/50462 | Loss: 2.5270 | LR: 2.15e-04 | 5166.9ms/step | 11.6 samples/s +Step 27860/50462 | Loss: 2.5311 | LR: 2.15e-04 | 5187.4ms/step | 11.6 samples/s +Step 27870/50462 | Loss: 2.4878 | LR: 2.14e-04 | 5168.6ms/step | 11.6 samples/s +Step 27880/50462 | Loss: 2.5464 | LR: 2.14e-04 | 5166.9ms/step | 11.6 samples/s +Step 27890/50462 | Loss: 2.5164 | LR: 2.14e-04 | 5177.1ms/step | 11.6 samples/s +Step 27900/50462 | Loss: 2.5167 | LR: 2.14e-04 | 5173.7ms/step | 11.6 samples/s +Step 27910/50462 | Loss: 2.5871 | LR: 2.14e-04 | 5170.6ms/step | 11.6 samples/s +Step 27920/50462 | Loss: 2.5555 | LR: 2.14e-04 | 5186.5ms/step | 11.6 samples/s +Step 27930/50462 | Loss: 2.5232 | LR: 2.13e-04 | 5175.5ms/step | 11.6 samples/s +Step 27940/50462 | Loss: 2.5127 | LR: 2.13e-04 | 5170.0ms/step | 11.6 samples/s +Step 27950/50462 | Loss: 2.5584 | LR: 2.13e-04 | 5179.1ms/step | 11.6 samples/s +Step 27960/50462 | Loss: 2.5384 | LR: 2.13e-04 | 5192.7ms/step | 11.6 samples/s +Step 27970/50462 | Loss: 2.5423 | LR: 2.13e-04 | 5187.6ms/step | 11.6 samples/s +Step 27980/50462 | Loss: 2.5354 | LR: 2.13e-04 | 5171.7ms/step | 11.6 samples/s +Step 27990/50462 | Loss: 2.5420 | LR: 2.12e-04 | 5180.1ms/step | 11.6 samples/s +Step 28000/50462 | Loss: 2.5544 | LR: 2.12e-04 | 5171.9ms/step | 11.6 samples/s +Validation at step 28000: Loss = 2.5444, Perplexity = 12.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-28000 +Step 28010/50462 | Loss: 2.5458 | LR: 2.12e-04 | 8611.8ms/step | 7.0 samples/s +Step 28020/50462 | Loss: 2.5539 | LR: 2.12e-04 | 5180.5ms/step | 11.6 samples/s +Step 28030/50462 | Loss: 2.5571 | LR: 2.12e-04 | 5178.9ms/step | 11.6 samples/s +Step 28040/50462 | Loss: 2.5539 | LR: 2.12e-04 | 5177.3ms/step | 11.6 samples/s +Step 28050/50462 | Loss: 2.5168 | LR: 2.12e-04 | 5174.7ms/step | 11.6 samples/s +Step 28060/50462 | Loss: 2.5594 | LR: 2.11e-04 | 5183.6ms/step | 11.6 samples/s +Step 28070/50462 | Loss: 2.5341 | LR: 2.11e-04 | 5181.7ms/step | 11.6 samples/s +Step 28080/50462 | Loss: 2.5240 | LR: 2.11e-04 | 5158.7ms/step | 11.6 samples/s +Step 28090/50462 | Loss: 2.5386 | LR: 2.11e-04 | 5174.6ms/step | 11.6 samples/s +Step 28100/50462 | Loss: 2.5007 | LR: 2.11e-04 | 5169.4ms/step | 11.6 samples/s +Step 28110/50462 | Loss: 2.5369 | LR: 2.11e-04 | 5174.9ms/step | 11.6 samples/s +Step 28120/50462 | Loss: 2.5231 | LR: 2.10e-04 | 5175.3ms/step | 11.6 samples/s +Step 28130/50462 | Loss: 2.5548 | LR: 2.10e-04 | 5170.5ms/step | 11.6 samples/s +Step 28140/50462 | Loss: 2.5244 | LR: 2.10e-04 | 5183.8ms/step | 11.6 samples/s +Step 28150/50462 | Loss: 2.5452 | LR: 2.10e-04 | 5178.9ms/step | 11.6 samples/s +Step 28160/50462 | Loss: 2.5344 | LR: 2.10e-04 | 5168.7ms/step | 11.6 samples/s +Step 28170/50462 | Loss: 2.5122 | LR: 2.10e-04 | 5168.2ms/step | 11.6 samples/s +Step 28180/50462 | Loss: 2.5477 | LR: 2.10e-04 | 5166.5ms/step | 11.6 samples/s +Step 28190/50462 | Loss: 2.5348 | LR: 2.09e-04 | 5171.8ms/step | 11.6 samples/s +Step 28200/50462 | Loss: 2.5404 | LR: 2.09e-04 | 5191.8ms/step | 11.6 samples/s +Step 28210/50462 | Loss: 2.5285 | LR: 2.09e-04 | 5177.7ms/step | 11.6 samples/s +Step 28220/50462 | Loss: 2.5478 | LR: 2.09e-04 | 5588.9ms/step | 10.7 samples/s +Step 28230/50462 | Loss: 2.5176 | LR: 2.09e-04 | 5172.4ms/step | 11.6 samples/s +Step 28240/50462 | Loss: 2.5221 | LR: 2.09e-04 | 5163.9ms/step | 11.6 samples/s +Step 28250/50462 | Loss: 2.5403 | LR: 2.08e-04 | 5182.7ms/step | 11.6 samples/s +Validation at step 28250: Loss = 2.5430, Perplexity = 12.72 +Step 28260/50462 | Loss: 2.5166 | LR: 2.08e-04 | 7727.2ms/step | 7.8 samples/s +Step 28270/50462 | Loss: 2.5567 | LR: 2.08e-04 | 5177.0ms/step | 11.6 samples/s +Step 28280/50462 | Loss: 2.5078 | LR: 2.08e-04 | 5174.9ms/step | 11.6 samples/s +Step 28290/50462 | Loss: 2.5266 | LR: 2.08e-04 | 5166.8ms/step | 11.6 samples/s +Step 28300/50462 | Loss: 2.5161 | LR: 2.08e-04 | 5164.5ms/step | 11.6 samples/s +Step 28310/50462 | Loss: 2.5494 | LR: 2.07e-04 | 5171.3ms/step | 11.6 samples/s +Step 28320/50462 | Loss: 2.5405 | LR: 2.07e-04 | 5182.2ms/step | 11.6 samples/s +Step 28330/50462 | Loss: 2.5493 | LR: 2.07e-04 | 5167.3ms/step | 11.6 samples/s +Step 28340/50462 | Loss: 2.5310 | LR: 2.07e-04 | 5171.3ms/step | 11.6 samples/s +Step 28350/50462 | Loss: 2.5183 | LR: 2.07e-04 | 5175.7ms/step | 11.6 samples/s +Step 28360/50462 | Loss: 2.5212 | LR: 2.07e-04 | 5184.4ms/step | 11.6 samples/s +Step 28370/50462 | Loss: 2.5162 | LR: 2.07e-04 | 5183.6ms/step | 11.6 samples/s +Step 28380/50462 | Loss: 2.5242 | LR: 2.06e-04 | 5181.5ms/step | 11.6 samples/s +Step 28390/50462 | Loss: 2.5180 | LR: 2.06e-04 | 5184.1ms/step | 11.6 samples/s +Step 28400/50462 | Loss: 2.5521 | LR: 2.06e-04 | 5174.5ms/step | 11.6 samples/s +Step 28410/50462 | Loss: 2.5297 | LR: 2.06e-04 | 5612.8ms/step | 10.7 samples/s +Step 28420/50462 | Loss: 2.5513 | LR: 2.06e-04 | 5174.2ms/step | 11.6 samples/s +Step 28430/50462 | Loss: 2.4988 | LR: 2.06e-04 | 5619.4ms/step | 10.7 samples/s +Step 28440/50462 | Loss: 2.5356 | LR: 2.05e-04 | 5177.7ms/step | 11.6 samples/s +Step 28450/50462 | Loss: 2.5054 | LR: 2.05e-04 | 5181.2ms/step | 11.6 samples/s +Step 28460/50462 | Loss: 2.5348 | LR: 2.05e-04 | 5164.2ms/step | 11.6 samples/s +Step 28470/50462 | Loss: 2.5368 | LR: 2.05e-04 | 5135.7ms/step | 11.7 samples/s +Step 28480/50462 | Loss: 2.5277 | LR: 2.05e-04 | 5148.2ms/step | 11.7 samples/s +Step 28490/50462 | Loss: 2.5217 | LR: 2.05e-04 | 5144.5ms/step | 11.7 samples/s +Step 28500/50462 | Loss: 2.5317 | LR: 2.05e-04 | 5162.3ms/step | 11.6 samples/s +Validation at step 28500: Loss = 2.5411, Perplexity = 12.69 +Step 28510/50462 | Loss: 2.5194 | LR: 2.04e-04 | 7704.4ms/step | 7.8 samples/s +Step 28520/50462 | Loss: 2.5327 | LR: 2.04e-04 | 5179.6ms/step | 11.6 samples/s +Step 28530/50462 | Loss: 2.4830 | LR: 2.04e-04 | 5144.5ms/step | 11.7 samples/s +Step 28540/50462 | Loss: 2.5264 | LR: 2.04e-04 | 5143.1ms/step | 11.7 samples/s +Step 28550/50462 | Loss: 2.5775 | LR: 2.04e-04 | 5123.0ms/step | 11.7 samples/s +Step 28560/50462 | Loss: 2.5179 | LR: 2.04e-04 | 5118.5ms/step | 11.7 samples/s +Step 28570/50462 | Loss: 2.5338 | LR: 2.03e-04 | 5133.7ms/step | 11.7 samples/s +Step 28580/50462 | Loss: 2.5294 | LR: 2.03e-04 | 5127.3ms/step | 11.7 samples/s +Step 28590/50462 | Loss: 2.5460 | LR: 2.03e-04 | 5127.6ms/step | 11.7 samples/s +Step 28600/50462 | Loss: 2.5156 | LR: 2.03e-04 | 5131.0ms/step | 11.7 samples/s +Step 28610/50462 | Loss: 2.5312 | LR: 2.03e-04 | 5136.8ms/step | 11.7 samples/s +Step 28620/50462 | Loss: 2.5112 | LR: 2.03e-04 | 5132.9ms/step | 11.7 samples/s +Step 28630/50462 | Loss: 2.5250 | LR: 2.03e-04 | 5142.2ms/step | 11.7 samples/s +Step 28640/50462 | Loss: 2.5388 | LR: 2.02e-04 | 5122.9ms/step | 11.7 samples/s +Step 28650/50462 | Loss: 2.5278 | LR: 2.02e-04 | 5168.5ms/step | 11.6 samples/s +Step 28660/50462 | Loss: 2.5301 | LR: 2.02e-04 | 5159.3ms/step | 11.6 samples/s +Step 28670/50462 | Loss: 2.5272 | LR: 2.02e-04 | 5154.2ms/step | 11.6 samples/s +Step 28680/50462 | Loss: 2.5424 | LR: 2.02e-04 | 5123.5ms/step | 11.7 samples/s +Step 28690/50462 | Loss: 2.5440 | LR: 2.02e-04 | 5132.7ms/step | 11.7 samples/s +Step 28700/50462 | Loss: 2.5254 | LR: 2.01e-04 | 5140.2ms/step | 11.7 samples/s +Step 28710/50462 | Loss: 2.5093 | LR: 2.01e-04 | 5154.8ms/step | 11.6 samples/s +Step 28720/50462 | Loss: 2.5420 | LR: 2.01e-04 | 5134.3ms/step | 11.7 samples/s +Step 28730/50462 | Loss: 2.5135 | LR: 2.01e-04 | 5125.5ms/step | 11.7 samples/s +Step 28740/50462 | Loss: 2.5325 | LR: 2.01e-04 | 5127.3ms/step | 11.7 samples/s +Step 28750/50462 | Loss: 2.5176 | LR: 2.01e-04 | 5124.3ms/step | 11.7 samples/s +Validation at step 28750: Loss = 2.5390, Perplexity = 12.67 +Step 28760/50462 | Loss: 2.5704 | LR: 2.00e-04 | 7675.5ms/step | 7.8 samples/s +Step 28770/50462 | Loss: 2.5599 | LR: 2.00e-04 | 5129.0ms/step | 11.7 samples/s +Step 28780/50462 | Loss: 2.5299 | LR: 2.00e-04 | 5138.4ms/step | 11.7 samples/s +Step 28790/50462 | Loss: 2.5044 | LR: 2.00e-04 | 5131.7ms/step | 11.7 samples/s +Step 28800/50462 | Loss: 2.5661 | LR: 2.00e-04 | 5128.3ms/step | 11.7 samples/s +Step 28810/50462 | Loss: 2.5006 | LR: 2.00e-04 | 5134.3ms/step | 11.7 samples/s +Step 28820/50462 | Loss: 2.5148 | LR: 2.00e-04 | 5116.8ms/step | 11.7 samples/s +Step 28830/50462 | Loss: 2.5233 | LR: 1.99e-04 | 5135.6ms/step | 11.7 samples/s +Step 28840/50462 | Loss: 2.5308 | LR: 1.99e-04 | 5130.6ms/step | 11.7 samples/s +Step 28850/50462 | Loss: 2.5190 | LR: 1.99e-04 | 5136.3ms/step | 11.7 samples/s +Step 28860/50462 | Loss: 2.5127 | LR: 1.99e-04 | 5131.8ms/step | 11.7 samples/s +Step 28870/50462 | Loss: 2.5349 | LR: 1.99e-04 | 5119.1ms/step | 11.7 samples/s +Step 28880/50462 | Loss: 2.5541 | LR: 1.99e-04 | 5129.0ms/step | 11.7 samples/s +Step 28890/50462 | Loss: 2.5064 | LR: 1.98e-04 | 5124.4ms/step | 11.7 samples/s +Step 28900/50462 | Loss: 2.5222 | LR: 1.98e-04 | 5121.2ms/step | 11.7 samples/s +Step 28910/50462 | Loss: 2.5290 | LR: 1.98e-04 | 5124.6ms/step | 11.7 samples/s +Step 28920/50462 | Loss: 2.5178 | LR: 1.98e-04 | 5128.4ms/step | 11.7 samples/s +Step 28930/50462 | Loss: 2.5433 | LR: 1.98e-04 | 5134.8ms/step | 11.7 samples/s +Step 28940/50462 | Loss: 2.5192 | LR: 1.98e-04 | 5115.7ms/step | 11.7 samples/s +Step 28950/50462 | Loss: 2.5296 | LR: 1.98e-04 | 5111.1ms/step | 11.7 samples/s +Step 28960/50462 | Loss: 2.5317 | LR: 1.97e-04 | 5132.9ms/step | 11.7 samples/s +Step 28970/50462 | Loss: 2.5353 | LR: 1.97e-04 | 5118.0ms/step | 11.7 samples/s +Step 28980/50462 | Loss: 2.5260 | LR: 1.97e-04 | 5116.5ms/step | 11.7 samples/s +Step 28990/50462 | Loss: 2.5198 | LR: 1.97e-04 | 5112.1ms/step | 11.7 samples/s +Step 29000/50462 | Loss: 2.5265 | LR: 1.97e-04 | 5107.9ms/step | 11.7 samples/s +Validation at step 29000: Loss = 2.5377, Perplexity = 12.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-29000 +Step 29010/50462 | Loss: 2.5332 | LR: 1.97e-04 | 8603.6ms/step | 7.0 samples/s +Step 29020/50462 | Loss: 2.5579 | LR: 1.96e-04 | 5170.2ms/step | 11.6 samples/s +Step 29030/50462 | Loss: 2.5296 | LR: 1.96e-04 | 5164.8ms/step | 11.6 samples/s +Step 29040/50462 | Loss: 2.5070 | LR: 1.96e-04 | 5166.4ms/step | 11.6 samples/s +Step 29050/50462 | Loss: 2.5173 | LR: 1.96e-04 | 5169.5ms/step | 11.6 samples/s +Step 29060/50462 | Loss: 2.5470 | LR: 1.96e-04 | 5159.2ms/step | 11.6 samples/s +Step 29070/50462 | Loss: 2.5282 | LR: 1.96e-04 | 5165.7ms/step | 11.6 samples/s +Step 29080/50462 | Loss: 2.5396 | LR: 1.96e-04 | 5172.2ms/step | 11.6 samples/s +Step 29090/50462 | Loss: 2.5347 | LR: 1.95e-04 | 5164.5ms/step | 11.6 samples/s +Step 29100/50462 | Loss: 2.4962 | LR: 1.95e-04 | 5163.0ms/step | 11.6 samples/s +Step 29110/50462 | Loss: 2.5353 | LR: 1.95e-04 | 5168.3ms/step | 11.6 samples/s +Step 29120/50462 | Loss: 2.5436 | LR: 1.95e-04 | 5175.8ms/step | 11.6 samples/s +Step 29130/50462 | Loss: 2.5249 | LR: 1.95e-04 | 5163.2ms/step | 11.6 samples/s +Step 29140/50462 | Loss: 2.5220 | LR: 1.95e-04 | 5163.9ms/step | 11.6 samples/s +Step 29150/50462 | Loss: 2.5558 | LR: 1.94e-04 | 5155.2ms/step | 11.6 samples/s +Step 29160/50462 | Loss: 2.5416 | LR: 1.94e-04 | 5165.3ms/step | 11.6 samples/s +Step 29170/50462 | Loss: 2.5149 | LR: 1.94e-04 | 5149.9ms/step | 11.7 samples/s +Step 29180/50462 | Loss: 2.5160 | LR: 1.94e-04 | 5163.4ms/step | 11.6 samples/s +Step 29190/50462 | Loss: 2.5303 | LR: 1.94e-04 | 5164.2ms/step | 11.6 samples/s +Step 29200/50462 | Loss: 2.5071 | LR: 1.94e-04 | 5194.2ms/step | 11.6 samples/s +Step 29210/50462 | Loss: 2.5380 | LR: 1.94e-04 | 5184.6ms/step | 11.6 samples/s +Step 29220/50462 | Loss: 2.5503 | LR: 1.93e-04 | 5176.1ms/step | 11.6 samples/s +Step 29230/50462 | Loss: 2.5158 | LR: 1.93e-04 | 5182.1ms/step | 11.6 samples/s +Step 29240/50462 | Loss: 2.5356 | LR: 1.93e-04 | 5171.1ms/step | 11.6 samples/s +Step 29250/50462 | Loss: 2.5589 | LR: 1.93e-04 | 5186.3ms/step | 11.6 samples/s +Validation at step 29250: Loss = 2.5361, Perplexity = 12.63 +Step 29260/50462 | Loss: 2.5280 | LR: 1.93e-04 | 7733.9ms/step | 7.8 samples/s +Step 29270/50462 | Loss: 2.5297 | LR: 1.93e-04 | 5163.9ms/step | 11.6 samples/s +Step 29280/50462 | Loss: 2.5207 | LR: 1.92e-04 | 5161.0ms/step | 11.6 samples/s +Step 29290/50462 | Loss: 2.5428 | LR: 1.92e-04 | 5154.8ms/step | 11.6 samples/s +Step 29300/50462 | Loss: 2.5281 | LR: 1.92e-04 | 5163.2ms/step | 11.6 samples/s +Step 29310/50462 | Loss: 2.5297 | LR: 1.92e-04 | 5178.2ms/step | 11.6 samples/s +Step 29320/50462 | Loss: 2.5295 | LR: 1.92e-04 | 5186.6ms/step | 11.6 samples/s +Step 29330/50462 | Loss: 2.5175 | LR: 1.92e-04 | 5167.5ms/step | 11.6 samples/s +Step 29340/50462 | Loss: 2.5400 | LR: 1.92e-04 | 5173.9ms/step | 11.6 samples/s +Step 29350/50462 | Loss: 2.5510 | LR: 1.91e-04 | 5168.9ms/step | 11.6 samples/s +Step 29360/50462 | Loss: 2.5023 | LR: 1.91e-04 | 5184.0ms/step | 11.6 samples/s +Step 29370/50462 | Loss: 2.5505 | LR: 1.91e-04 | 5169.3ms/step | 11.6 samples/s +Step 29380/50462 | Loss: 2.5268 | LR: 1.91e-04 | 5160.5ms/step | 11.6 samples/s +Step 29390/50462 | Loss: 2.5188 | LR: 1.91e-04 | 5178.6ms/step | 11.6 samples/s +Step 29400/50462 | Loss: 2.5492 | LR: 1.91e-04 | 5171.7ms/step | 11.6 samples/s +Step 29410/50462 | Loss: 2.5262 | LR: 1.90e-04 | 5176.0ms/step | 11.6 samples/s +Step 29420/50462 | Loss: 2.5237 | LR: 1.90e-04 | 5169.3ms/step | 11.6 samples/s +Step 29430/50462 | Loss: 2.4952 | LR: 1.90e-04 | 5178.6ms/step | 11.6 samples/s +Step 29440/50462 | Loss: 2.5185 | LR: 1.90e-04 | 5579.8ms/step | 10.8 samples/s +Step 29450/50462 | Loss: 2.5049 | LR: 1.90e-04 | 5636.3ms/step | 10.6 samples/s +Step 29460/50462 | Loss: 2.5440 | LR: 1.90e-04 | 5643.9ms/step | 10.6 samples/s +Step 29470/50462 | Loss: 2.5336 | LR: 1.90e-04 | 5642.0ms/step | 10.6 samples/s +Step 29480/50462 | Loss: 2.5149 | LR: 1.89e-04 | 5636.6ms/step | 10.6 samples/s +Step 29490/50462 | Loss: 2.4966 | LR: 1.89e-04 | 5638.4ms/step | 10.6 samples/s +Step 29500/50462 | Loss: 2.5129 | LR: 1.89e-04 | 5634.1ms/step | 10.6 samples/s +Validation at step 29500: Loss = 2.5350, Perplexity = 12.62 +Step 29510/50462 | Loss: 2.5126 | LR: 1.89e-04 | 8434.4ms/step | 7.1 samples/s +Step 29520/50462 | Loss: 2.5372 | LR: 1.89e-04 | 5633.9ms/step | 10.6 samples/s +Step 29530/50462 | Loss: 2.5432 | LR: 1.89e-04 | 5631.8ms/step | 10.7 samples/s +Step 29540/50462 | Loss: 2.5206 | LR: 1.88e-04 | 5632.9ms/step | 10.7 samples/s +Step 29550/50462 | Loss: 2.4922 | LR: 1.88e-04 | 5631.4ms/step | 10.7 samples/s +Step 29560/50462 | Loss: 2.5179 | LR: 1.88e-04 | 5622.9ms/step | 10.7 samples/s +Step 29570/50462 | Loss: 2.5088 | LR: 1.88e-04 | 5623.6ms/step | 10.7 samples/s +Step 29580/50462 | Loss: 2.5651 | LR: 1.88e-04 | 5623.6ms/step | 10.7 samples/s +Step 29590/50462 | Loss: 2.5120 | LR: 1.88e-04 | 5628.2ms/step | 10.7 samples/s +Step 29600/50462 | Loss: 2.5415 | LR: 1.88e-04 | 5624.2ms/step | 10.7 samples/s +Step 29610/50462 | Loss: 2.5458 | LR: 1.87e-04 | 5621.9ms/step | 10.7 samples/s +Step 29620/50462 | Loss: 2.5184 | LR: 1.87e-04 | 5624.2ms/step | 10.7 samples/s +Step 29630/50462 | Loss: 2.5096 | LR: 1.87e-04 | 5625.2ms/step | 10.7 samples/s +Step 29640/50462 | Loss: 2.5257 | LR: 1.87e-04 | 5622.8ms/step | 10.7 samples/s +Step 29650/50462 | Loss: 2.5387 | LR: 1.87e-04 | 5625.4ms/step | 10.7 samples/s +Step 29660/50462 | Loss: 2.4902 | LR: 1.87e-04 | 5624.6ms/step | 10.7 samples/s +Step 29670/50462 | Loss: 2.5285 | LR: 1.86e-04 | 5621.5ms/step | 10.7 samples/s +Step 29680/50462 | Loss: 2.5297 | LR: 1.86e-04 | 5626.1ms/step | 10.7 samples/s +Step 29690/50462 | Loss: 2.5058 | LR: 1.86e-04 | 5625.5ms/step | 10.7 samples/s +Step 29700/50462 | Loss: 2.5369 | LR: 1.86e-04 | 5624.8ms/step | 10.7 samples/s +Step 29710/50462 | Loss: 2.5498 | LR: 1.86e-04 | 5622.7ms/step | 10.7 samples/s +Step 29720/50462 | Loss: 2.5291 | LR: 1.86e-04 | 5620.8ms/step | 10.7 samples/s +Step 29730/50462 | Loss: 2.5244 | LR: 1.86e-04 | 5620.0ms/step | 10.7 samples/s +Step 29740/50462 | Loss: 2.5371 | LR: 1.85e-04 | 5621.9ms/step | 10.7 samples/s +Step 29750/50462 | Loss: 2.5287 | LR: 1.85e-04 | 5622.7ms/step | 10.7 samples/s +Validation at step 29750: Loss = 2.5338, Perplexity = 12.60 +Step 29760/50462 | Loss: 2.5315 | LR: 1.85e-04 | 8422.0ms/step | 7.1 samples/s +Step 29770/50462 | Loss: 2.5284 | LR: 1.85e-04 | 5619.3ms/step | 10.7 samples/s +Step 29780/50462 | Loss: 2.5127 | LR: 1.85e-04 | 5622.9ms/step | 10.7 samples/s +Step 29790/50462 | Loss: 2.5252 | LR: 1.85e-04 | 5489.0ms/step | 10.9 samples/s +Step 29800/50462 | Loss: 2.4971 | LR: 1.85e-04 | 5162.1ms/step | 11.6 samples/s +Step 29810/50462 | Loss: 2.4923 | LR: 1.84e-04 | 5175.1ms/step | 11.6 samples/s +Step 29820/50462 | Loss: 2.5158 | LR: 1.84e-04 | 5167.0ms/step | 11.6 samples/s +Step 29830/50462 | Loss: 2.5318 | LR: 1.84e-04 | 5180.9ms/step | 11.6 samples/s +Step 29840/50462 | Loss: 2.5112 | LR: 1.84e-04 | 5173.4ms/step | 11.6 samples/s +Step 29850/50462 | Loss: 2.5123 | LR: 1.84e-04 | 5189.3ms/step | 11.6 samples/s +Step 29860/50462 | Loss: 2.5192 | LR: 1.84e-04 | 5176.7ms/step | 11.6 samples/s +Step 29870/50462 | Loss: 2.5246 | LR: 1.83e-04 | 5175.0ms/step | 11.6 samples/s +Step 29880/50462 | Loss: 2.5284 | LR: 1.83e-04 | 5177.0ms/step | 11.6 samples/s +Step 29890/50462 | Loss: 2.5257 | LR: 1.83e-04 | 5171.3ms/step | 11.6 samples/s +Step 29900/50462 | Loss: 2.5240 | LR: 1.83e-04 | 5182.2ms/step | 11.6 samples/s +Step 29910/50462 | Loss: 2.5077 | LR: 1.83e-04 | 5384.9ms/step | 11.1 samples/s +Step 29920/50462 | Loss: 2.5169 | LR: 1.83e-04 | 5620.7ms/step | 10.7 samples/s +Step 29930/50462 | Loss: 2.5406 | LR: 1.83e-04 | 5638.1ms/step | 10.6 samples/s +Step 29940/50462 | Loss: 2.5169 | LR: 1.82e-04 | 5640.6ms/step | 10.6 samples/s +Step 29950/50462 | Loss: 2.5577 | LR: 1.82e-04 | 5632.2ms/step | 10.7 samples/s +Step 29960/50462 | Loss: 2.5348 | LR: 1.82e-04 | 5630.8ms/step | 10.7 samples/s +Step 29970/50462 | Loss: 2.5515 | LR: 1.82e-04 | 5633.6ms/step | 10.7 samples/s +Step 29980/50462 | Loss: 2.4990 | LR: 1.82e-04 | 5632.5ms/step | 10.7 samples/s +Step 29990/50462 | Loss: 2.4950 | LR: 1.82e-04 | 5630.1ms/step | 10.7 samples/s +Step 30000/50462 | Loss: 2.4724 | LR: 1.81e-04 | 5629.2ms/step | 10.7 samples/s +Validation at step 30000: Loss = 2.5326, Perplexity = 12.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-30000 +Step 30010/50462 | Loss: 2.4898 | LR: 1.81e-04 | 9231.7ms/step | 6.5 samples/s +Step 30020/50462 | Loss: 2.5031 | LR: 1.81e-04 | 5626.8ms/step | 10.7 samples/s +Step 30030/50462 | Loss: 2.5227 | LR: 1.81e-04 | 5624.7ms/step | 10.7 samples/s +Step 30040/50462 | Loss: 2.5791 | LR: 1.81e-04 | 5624.5ms/step | 10.7 samples/s +Step 30050/50462 | Loss: 2.5294 | LR: 1.81e-04 | 5626.2ms/step | 10.7 samples/s +Step 30060/50462 | Loss: 2.5296 | LR: 1.81e-04 | 5627.2ms/step | 10.7 samples/s +Step 30070/50462 | Loss: 2.5361 | LR: 1.80e-04 | 5624.3ms/step | 10.7 samples/s +Step 30080/50462 | Loss: 2.5282 | LR: 1.80e-04 | 5621.3ms/step | 10.7 samples/s +Step 30090/50462 | Loss: 2.5335 | LR: 1.80e-04 | 5625.5ms/step | 10.7 samples/s +Step 30100/50462 | Loss: 2.5590 | LR: 1.80e-04 | 5621.1ms/step | 10.7 samples/s +Step 30110/50462 | Loss: 2.5518 | LR: 1.80e-04 | 5619.9ms/step | 10.7 samples/s +Step 30120/50462 | Loss: 2.5273 | LR: 1.80e-04 | 5621.9ms/step | 10.7 samples/s +Step 30130/50462 | Loss: 2.4877 | LR: 1.79e-04 | 5621.1ms/step | 10.7 samples/s +Step 30140/50462 | Loss: 2.5407 | LR: 1.79e-04 | 5621.2ms/step | 10.7 samples/s +Step 30150/50462 | Loss: 2.5305 | LR: 1.79e-04 | 5626.4ms/step | 10.7 samples/s +Step 30160/50462 | Loss: 2.5306 | LR: 1.79e-04 | 5623.7ms/step | 10.7 samples/s +Step 30170/50462 | Loss: 2.5100 | LR: 1.79e-04 | 5621.4ms/step | 10.7 samples/s +Step 30180/50462 | Loss: 2.4980 | LR: 1.79e-04 | 5622.5ms/step | 10.7 samples/s +Step 30190/50462 | Loss: 2.4950 | LR: 1.79e-04 | 5617.6ms/step | 10.7 samples/s +Step 30200/50462 | Loss: 2.5232 | LR: 1.78e-04 | 5620.6ms/step | 10.7 samples/s +Step 30210/50462 | Loss: 2.5443 | LR: 1.78e-04 | 5617.9ms/step | 10.7 samples/s +Step 30220/50462 | Loss: 2.5051 | LR: 1.78e-04 | 5617.2ms/step | 10.7 samples/s +Step 30230/50462 | Loss: 2.5197 | LR: 1.78e-04 | 5618.2ms/step | 10.7 samples/s +Step 30240/50462 | Loss: 2.5386 | LR: 1.78e-04 | 5617.7ms/step | 10.7 samples/s +Step 30250/50462 | Loss: 2.5016 | LR: 1.78e-04 | 5618.5ms/step | 10.7 samples/s +Validation at step 30250: Loss = 2.5314, Perplexity = 12.57 +Step 30260/50462 | Loss: 2.5223 | LR: 1.78e-04 | 8394.3ms/step | 7.1 samples/s +Step 30270/50462 | Loss: 2.5372 | LR: 1.77e-04 | 5174.8ms/step | 11.6 samples/s +Step 30280/50462 | Loss: 2.5335 | LR: 1.77e-04 | 5167.5ms/step | 11.6 samples/s +Step 30290/50462 | Loss: 2.5253 | LR: 1.77e-04 | 5573.8ms/step | 10.8 samples/s +Step 30300/50462 | Loss: 2.5264 | LR: 1.77e-04 | 5159.7ms/step | 11.6 samples/s +Step 30310/50462 | Loss: 2.5069 | LR: 1.77e-04 | 5163.0ms/step | 11.6 samples/s +Step 30320/50462 | Loss: 2.5138 | LR: 1.77e-04 | 5428.2ms/step | 11.1 samples/s +Step 30330/50462 | Loss: 2.5286 | LR: 1.76e-04 | 5760.9ms/step | 10.4 samples/s +Step 30340/50462 | Loss: 2.5020 | LR: 1.76e-04 | 5740.7ms/step | 10.5 samples/s +Step 30350/50462 | Loss: 2.5519 | LR: 1.76e-04 | 5764.4ms/step | 10.4 samples/s +Step 30360/50462 | Loss: 2.5328 | LR: 1.76e-04 | 5761.9ms/step | 10.4 samples/s +Step 30370/50462 | Loss: 2.5251 | LR: 1.76e-04 | 5758.9ms/step | 10.4 samples/s +Step 30380/50462 | Loss: 2.5214 | LR: 1.76e-04 | 5725.3ms/step | 10.5 samples/s +Step 30390/50462 | Loss: 2.5069 | LR: 1.76e-04 | 5678.9ms/step | 10.6 samples/s +Step 30400/50462 | Loss: 2.4941 | LR: 1.75e-04 | 5684.7ms/step | 10.6 samples/s +Step 30410/50462 | Loss: 2.5147 | LR: 1.75e-04 | 5655.4ms/step | 10.6 samples/s +Step 30420/50462 | Loss: 2.4754 | LR: 1.75e-04 | 5660.2ms/step | 10.6 samples/s +Step 30430/50462 | Loss: 2.5308 | LR: 1.75e-04 | 5667.7ms/step | 10.6 samples/s +Step 30440/50462 | Loss: 2.5316 | LR: 1.75e-04 | 5683.3ms/step | 10.6 samples/s +Step 30450/50462 | Loss: 2.5429 | LR: 1.75e-04 | 5683.0ms/step | 10.6 samples/s +Step 30460/50462 | Loss: 2.4955 | LR: 1.75e-04 | 5667.6ms/step | 10.6 samples/s +Step 30470/50462 | Loss: 2.5198 | LR: 1.74e-04 | 5670.8ms/step | 10.6 samples/s +Step 30480/50462 | Loss: 2.5402 | LR: 1.74e-04 | 5668.8ms/step | 10.6 samples/s +Step 30490/50462 | Loss: 2.5343 | LR: 1.74e-04 | 5670.5ms/step | 10.6 samples/s +Step 30500/50462 | Loss: 2.5137 | LR: 1.74e-04 | 5665.6ms/step | 10.6 samples/s +Validation at step 30500: Loss = 2.5300, Perplexity = 12.55 +Step 30510/50462 | Loss: 2.5130 | LR: 1.74e-04 | 8479.7ms/step | 7.1 samples/s +Step 30520/50462 | Loss: 2.5142 | LR: 1.74e-04 | 6117.3ms/step | 9.8 samples/s +Step 30530/50462 | Loss: 2.5360 | LR: 1.73e-04 | 5647.2ms/step | 10.6 samples/s +Step 30540/50462 | Loss: 2.4926 | LR: 1.73e-04 | 6159.0ms/step | 9.7 samples/s +Step 30550/50462 | Loss: 2.5392 | LR: 1.73e-04 | 5653.3ms/step | 10.6 samples/s +Step 30560/50462 | Loss: 2.4919 | LR: 1.73e-04 | 5654.0ms/step | 10.6 samples/s +Step 30570/50462 | Loss: 2.5256 | LR: 1.73e-04 | 5650.0ms/step | 10.6 samples/s +Step 30580/50462 | Loss: 2.5262 | LR: 1.73e-04 | 5652.4ms/step | 10.6 samples/s +Step 30590/50462 | Loss: 2.4934 | LR: 1.73e-04 | 5657.6ms/step | 10.6 samples/s +Step 30600/50462 | Loss: 2.4985 | LR: 1.72e-04 | 5649.5ms/step | 10.6 samples/s +Step 30610/50462 | Loss: 2.5445 | LR: 1.72e-04 | 5651.4ms/step | 10.6 samples/s +Step 30620/50462 | Loss: 2.5065 | LR: 1.72e-04 | 5643.5ms/step | 10.6 samples/s +Step 30630/50462 | Loss: 2.4925 | LR: 1.72e-04 | 5628.9ms/step | 10.7 samples/s +Step 30640/50462 | Loss: 2.4972 | LR: 1.72e-04 | 5636.0ms/step | 10.6 samples/s +Step 30650/50462 | Loss: 2.5274 | LR: 1.72e-04 | 5643.6ms/step | 10.6 samples/s +Step 30660/50462 | Loss: 2.4836 | LR: 1.72e-04 | 5643.3ms/step | 10.6 samples/s +Step 30670/50462 | Loss: 2.5045 | LR: 1.71e-04 | 5649.9ms/step | 10.6 samples/s +Step 30680/50462 | Loss: 2.5056 | LR: 1.71e-04 | 5645.3ms/step | 10.6 samples/s +Step 30690/50462 | Loss: 2.5036 | LR: 1.71e-04 | 5645.6ms/step | 10.6 samples/s +Step 30700/50462 | Loss: 2.5323 | LR: 1.71e-04 | 5641.4ms/step | 10.6 samples/s +Step 30710/50462 | Loss: 2.5139 | LR: 1.71e-04 | 5316.5ms/step | 11.3 samples/s +Step 30720/50462 | Loss: 2.5431 | LR: 1.71e-04 | 5308.3ms/step | 11.3 samples/s +Step 30730/50462 | Loss: 2.5161 | LR: 1.70e-04 | 5648.4ms/step | 10.6 samples/s +Step 30740/50462 | Loss: 2.5143 | LR: 1.70e-04 | 5650.3ms/step | 10.6 samples/s +Step 30750/50462 | Loss: 2.5066 | LR: 1.70e-04 | 5647.6ms/step | 10.6 samples/s +Validation at step 30750: Loss = 2.5284, Perplexity = 12.53 +Step 30760/50462 | Loss: 2.5290 | LR: 1.70e-04 | 8452.3ms/step | 7.1 samples/s +Step 30770/50462 | Loss: 2.5213 | LR: 1.70e-04 | 5645.0ms/step | 10.6 samples/s +Step 30780/50462 | Loss: 2.5148 | LR: 1.70e-04 | 5643.9ms/step | 10.6 samples/s +Step 30790/50462 | Loss: 2.5288 | LR: 1.70e-04 | 5642.9ms/step | 10.6 samples/s +Step 30800/50462 | Loss: 2.4980 | LR: 1.69e-04 | 5645.9ms/step | 10.6 samples/s +Step 30810/50462 | Loss: 2.4953 | LR: 1.69e-04 | 5643.6ms/step | 10.6 samples/s +Step 30820/50462 | Loss: 2.5265 | LR: 1.69e-04 | 5643.7ms/step | 10.6 samples/s +Step 30830/50462 | Loss: 2.5230 | LR: 1.69e-04 | 5642.6ms/step | 10.6 samples/s +Step 30840/50462 | Loss: 2.5004 | LR: 1.69e-04 | 5644.6ms/step | 10.6 samples/s +Step 30850/50462 | Loss: 2.5208 | LR: 1.69e-04 | 5644.5ms/step | 10.6 samples/s +Step 30860/50462 | Loss: 2.5359 | LR: 1.69e-04 | 5646.8ms/step | 10.6 samples/s +Step 30870/50462 | Loss: 2.5044 | LR: 1.68e-04 | 5648.5ms/step | 10.6 samples/s +Step 30880/50462 | Loss: 2.4805 | LR: 1.68e-04 | 5649.2ms/step | 10.6 samples/s +Step 30890/50462 | Loss: 2.5044 | LR: 1.68e-04 | 5646.1ms/step | 10.6 samples/s +Step 30900/50462 | Loss: 2.4907 | LR: 1.68e-04 | 5643.9ms/step | 10.6 samples/s +Step 30910/50462 | Loss: 2.4928 | LR: 1.68e-04 | 5650.1ms/step | 10.6 samples/s +Step 30920/50462 | Loss: 2.5184 | LR: 1.68e-04 | 5665.0ms/step | 10.6 samples/s +Step 30930/50462 | Loss: 2.4969 | LR: 1.67e-04 | 5650.7ms/step | 10.6 samples/s +Step 30940/50462 | Loss: 2.4743 | LR: 1.67e-04 | 5650.1ms/step | 10.6 samples/s +Step 30950/50462 | Loss: 2.5381 | LR: 1.67e-04 | 5650.9ms/step | 10.6 samples/s +Step 30960/50462 | Loss: 2.5132 | LR: 1.67e-04 | 5663.0ms/step | 10.6 samples/s +Step 30970/50462 | Loss: 2.5352 | LR: 1.67e-04 | 5656.3ms/step | 10.6 samples/s +Step 30980/50462 | Loss: 2.5330 | LR: 1.67e-04 | 5649.6ms/step | 10.6 samples/s +Step 30990/50462 | Loss: 2.5127 | LR: 1.67e-04 | 5654.5ms/step | 10.6 samples/s +Step 31000/50462 | Loss: 2.5064 | LR: 1.66e-04 | 5663.7ms/step | 10.6 samples/s +Validation at step 31000: Loss = 2.5275, Perplexity = 12.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-31000 +Step 31010/50462 | Loss: 2.5472 | LR: 1.66e-04 | 9334.5ms/step | 6.4 samples/s +Step 31020/50462 | Loss: 2.5140 | LR: 1.66e-04 | 5671.6ms/step | 10.6 samples/s +Step 31030/50462 | Loss: 2.5065 | LR: 1.66e-04 | 5671.2ms/step | 10.6 samples/s +Step 31040/50462 | Loss: 2.4836 | LR: 1.66e-04 | 5670.8ms/step | 10.6 samples/s +Step 31050/50462 | Loss: 2.5092 | LR: 1.66e-04 | 5665.3ms/step | 10.6 samples/s +Step 31060/50462 | Loss: 2.5100 | LR: 1.66e-04 | 5667.1ms/step | 10.6 samples/s +Step 31070/50462 | Loss: 2.4996 | LR: 1.65e-04 | 5666.9ms/step | 10.6 samples/s +Step 31080/50462 | Loss: 2.4763 | LR: 1.65e-04 | 5671.6ms/step | 10.6 samples/s +Step 31090/50462 | Loss: 2.5004 | LR: 1.65e-04 | 5670.3ms/step | 10.6 samples/s +Step 31100/50462 | Loss: 2.5366 | LR: 1.65e-04 | 5668.1ms/step | 10.6 samples/s +Step 31110/50462 | Loss: 2.5011 | LR: 1.65e-04 | 5664.4ms/step | 10.6 samples/s +Step 31120/50462 | Loss: 2.5396 | LR: 1.65e-04 | 5659.4ms/step | 10.6 samples/s +Step 31130/50462 | Loss: 2.5313 | LR: 1.64e-04 | 5665.9ms/step | 10.6 samples/s +Step 31140/50462 | Loss: 2.4823 | LR: 1.64e-04 | 5670.8ms/step | 10.6 samples/s +Step 31150/50462 | Loss: 2.5431 | LR: 1.64e-04 | 5671.2ms/step | 10.6 samples/s +Step 31160/50462 | Loss: 2.5076 | LR: 1.64e-04 | 5669.3ms/step | 10.6 samples/s +Step 31170/50462 | Loss: 2.5263 | LR: 1.64e-04 | 5668.7ms/step | 10.6 samples/s +Step 31180/50462 | Loss: 2.5091 | LR: 1.64e-04 | 5668.2ms/step | 10.6 samples/s +Step 31190/50462 | Loss: 2.5187 | LR: 1.64e-04 | 5670.2ms/step | 10.6 samples/s +Step 31200/50462 | Loss: 2.4910 | LR: 1.63e-04 | 5669.3ms/step | 10.6 samples/s +Step 31210/50462 | Loss: 2.5107 | LR: 1.63e-04 | 5669.1ms/step | 10.6 samples/s +Step 31220/50462 | Loss: 2.4949 | LR: 1.63e-04 | 5668.7ms/step | 10.6 samples/s +Step 31230/50462 | Loss: 2.5112 | LR: 1.63e-04 | 5669.2ms/step | 10.6 samples/s +Step 31240/50462 | Loss: 2.4899 | LR: 1.63e-04 | 5669.1ms/step | 10.6 samples/s +Step 31250/50462 | Loss: 2.5035 | LR: 1.63e-04 | 5665.5ms/step | 10.6 samples/s +Validation at step 31250: Loss = 2.5262, Perplexity = 12.51 +Step 31260/50462 | Loss: 2.4938 | LR: 1.63e-04 | 8466.3ms/step | 7.1 samples/s +Step 31270/50462 | Loss: 2.5032 | LR: 1.62e-04 | 5661.0ms/step | 10.6 samples/s +Step 31280/50462 | Loss: 2.5397 | LR: 1.62e-04 | 5647.1ms/step | 10.6 samples/s +Step 31290/50462 | Loss: 2.5342 | LR: 1.62e-04 | 5651.7ms/step | 10.6 samples/s +Step 31300/50462 | Loss: 2.5179 | LR: 1.62e-04 | 5661.4ms/step | 10.6 samples/s +Step 31310/50462 | Loss: 2.5156 | LR: 1.62e-04 | 5658.6ms/step | 10.6 samples/s +Step 31320/50462 | Loss: 2.5297 | LR: 1.62e-04 | 5652.5ms/step | 10.6 samples/s +Step 31330/50462 | Loss: 2.4693 | LR: 1.62e-04 | 5648.9ms/step | 10.6 samples/s +Step 31340/50462 | Loss: 2.5184 | LR: 1.61e-04 | 5656.0ms/step | 10.6 samples/s +Step 31350/50462 | Loss: 2.5373 | LR: 1.61e-04 | 5657.6ms/step | 10.6 samples/s +Step 31360/50462 | Loss: 2.5298 | LR: 1.61e-04 | 5647.7ms/step | 10.6 samples/s +Step 31370/50462 | Loss: 2.5077 | LR: 1.61e-04 | 5647.9ms/step | 10.6 samples/s +Step 31380/50462 | Loss: 2.4889 | LR: 1.61e-04 | 5655.6ms/step | 10.6 samples/s +Step 31390/50462 | Loss: 2.4815 | LR: 1.61e-04 | 5644.2ms/step | 10.6 samples/s +Step 31400/50462 | Loss: 2.5075 | LR: 1.60e-04 | 5646.5ms/step | 10.6 samples/s +Step 31410/50462 | Loss: 2.5158 | LR: 1.60e-04 | 5645.1ms/step | 10.6 samples/s +Step 31420/50462 | Loss: 2.5212 | LR: 1.60e-04 | 5643.9ms/step | 10.6 samples/s +Step 31430/50462 | Loss: 2.5212 | LR: 1.60e-04 | 5642.3ms/step | 10.6 samples/s +Step 31440/50462 | Loss: 2.5413 | LR: 1.60e-04 | 5655.9ms/step | 10.6 samples/s +Step 31450/50462 | Loss: 2.5064 | LR: 1.60e-04 | 5650.1ms/step | 10.6 samples/s +Step 31460/50462 | Loss: 2.5202 | LR: 1.60e-04 | 5646.5ms/step | 10.6 samples/s +Step 31470/50462 | Loss: 2.4962 | LR: 1.59e-04 | 5643.7ms/step | 10.6 samples/s +Step 31480/50462 | Loss: 2.5360 | LR: 1.59e-04 | 5645.2ms/step | 10.6 samples/s +Step 31490/50462 | Loss: 2.5431 | LR: 1.59e-04 | 5646.4ms/step | 10.6 samples/s +Step 31500/50462 | Loss: 2.5140 | LR: 1.59e-04 | 5646.0ms/step | 10.6 samples/s +Validation at step 31500: Loss = 2.5247, Perplexity = 12.49 +Step 31510/50462 | Loss: 2.5297 | LR: 1.59e-04 | 8461.2ms/step | 7.1 samples/s +Step 31520/50462 | Loss: 2.5592 | LR: 1.59e-04 | 5669.1ms/step | 10.6 samples/s +Step 31530/50462 | Loss: 2.4853 | LR: 1.59e-04 | 5669.6ms/step | 10.6 samples/s +Step 31540/50462 | Loss: 2.5012 | LR: 1.58e-04 | 5667.9ms/step | 10.6 samples/s +Step 31550/50462 | Loss: 2.4960 | LR: 1.58e-04 | 5667.7ms/step | 10.6 samples/s +Step 31560/50462 | Loss: 2.5277 | LR: 1.58e-04 | 5665.0ms/step | 10.6 samples/s +Step 31570/50462 | Loss: 2.5021 | LR: 1.58e-04 | 5666.0ms/step | 10.6 samples/s +Step 31580/50462 | Loss: 2.4827 | LR: 1.58e-04 | 5663.3ms/step | 10.6 samples/s +Step 31590/50462 | Loss: 2.5119 | LR: 1.58e-04 | 5665.8ms/step | 10.6 samples/s +Step 31600/50462 | Loss: 2.5063 | LR: 1.58e-04 | 5666.3ms/step | 10.6 samples/s +Step 31610/50462 | Loss: 2.4859 | LR: 1.57e-04 | 5665.6ms/step | 10.6 samples/s +Step 31620/50462 | Loss: 2.5364 | LR: 1.57e-04 | 5664.7ms/step | 10.6 samples/s +Step 31630/50462 | Loss: 2.5077 | LR: 1.57e-04 | 5666.7ms/step | 10.6 samples/s +Step 31640/50462 | Loss: 2.5301 | LR: 1.57e-04 | 5663.1ms/step | 10.6 samples/s +Step 31650/50462 | Loss: 2.5170 | LR: 1.57e-04 | 5664.0ms/step | 10.6 samples/s +Step 31660/50462 | Loss: 2.4967 | LR: 1.57e-04 | 5667.1ms/step | 10.6 samples/s +Step 31670/50462 | Loss: 2.5193 | LR: 1.57e-04 | 5668.8ms/step | 10.6 samples/s +Step 31680/50462 | Loss: 2.5218 | LR: 1.56e-04 | 5495.6ms/step | 10.9 samples/s +Step 31690/50462 | Loss: 2.5188 | LR: 1.56e-04 | 5166.8ms/step | 11.6 samples/s +Step 31700/50462 | Loss: 2.4736 | LR: 1.56e-04 | 5165.2ms/step | 11.6 samples/s +Step 31710/50462 | Loss: 2.4807 | LR: 1.56e-04 | 5161.1ms/step | 11.6 samples/s +Step 31720/50462 | Loss: 2.5261 | LR: 1.56e-04 | 5402.9ms/step | 11.1 samples/s +Step 31730/50462 | Loss: 2.5316 | LR: 1.56e-04 | 5738.4ms/step | 10.5 samples/s +Step 31740/50462 | Loss: 2.5255 | LR: 1.55e-04 | 5745.5ms/step | 10.4 samples/s +Step 31750/50462 | Loss: 2.5177 | LR: 1.55e-04 | 5741.3ms/step | 10.5 samples/s +Validation at step 31750: Loss = 2.5235, Perplexity = 12.47 +Step 31760/50462 | Loss: 2.5081 | LR: 1.55e-04 | 8594.2ms/step | 7.0 samples/s +Step 31770/50462 | Loss: 2.5297 | LR: 1.55e-04 | 5742.3ms/step | 10.4 samples/s +Step 31780/50462 | Loss: 2.4992 | LR: 1.55e-04 | 5747.8ms/step | 10.4 samples/s +Step 31790/50462 | Loss: 2.5216 | LR: 1.55e-04 | 5743.3ms/step | 10.4 samples/s +Step 31800/50462 | Loss: 2.5126 | LR: 1.55e-04 | 5745.1ms/step | 10.4 samples/s +Step 31810/50462 | Loss: 2.5061 | LR: 1.54e-04 | 5739.2ms/step | 10.5 samples/s +Step 31820/50462 | Loss: 2.5190 | LR: 1.54e-04 | 5747.9ms/step | 10.4 samples/s +Step 31830/50462 | Loss: 2.4929 | LR: 1.54e-04 | 5744.9ms/step | 10.4 samples/s +Step 31840/50462 | Loss: 2.5122 | LR: 1.54e-04 | 5745.1ms/step | 10.4 samples/s +Step 31850/50462 | Loss: 2.5052 | LR: 1.54e-04 | 5740.4ms/step | 10.5 samples/s +Step 31860/50462 | Loss: 2.4984 | LR: 1.54e-04 | 5743.5ms/step | 10.4 samples/s +Step 31870/50462 | Loss: 2.5128 | LR: 1.54e-04 | 5749.5ms/step | 10.4 samples/s +Step 31880/50462 | Loss: 2.5116 | LR: 1.53e-04 | 5744.8ms/step | 10.4 samples/s +Step 31890/50462 | Loss: 2.5129 | LR: 1.53e-04 | 5747.3ms/step | 10.4 samples/s +Step 31900/50462 | Loss: 2.5332 | LR: 1.53e-04 | 5743.7ms/step | 10.4 samples/s +Step 31910/50462 | Loss: 2.5150 | LR: 1.53e-04 | 5741.1ms/step | 10.5 samples/s +Step 31920/50462 | Loss: 2.4764 | LR: 1.53e-04 | 5742.5ms/step | 10.4 samples/s +Step 31930/50462 | Loss: 2.4965 | LR: 1.53e-04 | 5745.9ms/step | 10.4 samples/s +Step 31940/50462 | Loss: 2.5256 | LR: 1.53e-04 | 5745.8ms/step | 10.4 samples/s +Step 31950/50462 | Loss: 2.5308 | LR: 1.52e-04 | 5743.6ms/step | 10.4 samples/s +Step 31960/50462 | Loss: 2.5208 | LR: 1.52e-04 | 5742.4ms/step | 10.4 samples/s +Step 31970/50462 | Loss: 2.5123 | LR: 1.52e-04 | 5747.3ms/step | 10.4 samples/s +Step 31980/50462 | Loss: 2.5344 | LR: 1.52e-04 | 5736.7ms/step | 10.5 samples/s +Step 31990/50462 | Loss: 2.5339 | LR: 1.52e-04 | 5743.2ms/step | 10.4 samples/s +Step 32000/50462 | Loss: 2.5581 | LR: 1.52e-04 | 5740.7ms/step | 10.5 samples/s +Validation at step 32000: Loss = 2.5226, Perplexity = 12.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-32000 +Step 32010/50462 | Loss: 2.5398 | LR: 1.52e-04 | 9479.6ms/step | 6.3 samples/s +Step 32020/50462 | Loss: 2.4940 | LR: 1.51e-04 | 5738.9ms/step | 10.5 samples/s +Step 32030/50462 | Loss: 2.4963 | LR: 1.51e-04 | 5741.8ms/step | 10.4 samples/s +Step 32040/50462 | Loss: 2.5489 | LR: 1.51e-04 | 5742.2ms/step | 10.4 samples/s +Step 32050/50462 | Loss: 2.5127 | LR: 1.51e-04 | 5747.2ms/step | 10.4 samples/s +Step 32060/50462 | Loss: 2.4972 | LR: 1.51e-04 | 5743.8ms/step | 10.4 samples/s +Step 32070/50462 | Loss: 2.5153 | LR: 1.51e-04 | 5740.1ms/step | 10.5 samples/s +Step 32080/50462 | Loss: 2.5216 | LR: 1.51e-04 | 5748.3ms/step | 10.4 samples/s +Step 32090/50462 | Loss: 2.5077 | LR: 1.50e-04 | 5749.9ms/step | 10.4 samples/s +Step 32100/50462 | Loss: 2.5032 | LR: 1.50e-04 | 5749.8ms/step | 10.4 samples/s +Step 32110/50462 | Loss: 2.4894 | LR: 1.50e-04 | 5748.2ms/step | 10.4 samples/s +Step 32120/50462 | Loss: 2.5170 | LR: 1.50e-04 | 5747.2ms/step | 10.4 samples/s +Step 32130/50462 | Loss: 2.5053 | LR: 1.50e-04 | 5743.8ms/step | 10.4 samples/s +Step 32140/50462 | Loss: 2.5249 | LR: 1.50e-04 | 5748.2ms/step | 10.4 samples/s +Step 32150/50462 | Loss: 2.5445 | LR: 1.50e-04 | 5747.8ms/step | 10.4 samples/s +Step 32160/50462 | Loss: 2.5086 | LR: 1.49e-04 | 5750.1ms/step | 10.4 samples/s +Step 32170/50462 | Loss: 2.5087 | LR: 1.49e-04 | 5750.9ms/step | 10.4 samples/s +Step 32180/50462 | Loss: 2.4950 | LR: 1.49e-04 | 5750.7ms/step | 10.4 samples/s +Step 32190/50462 | Loss: 2.5228 | LR: 1.49e-04 | 5750.0ms/step | 10.4 samples/s +Step 32200/50462 | Loss: 2.5043 | LR: 1.49e-04 | 5748.4ms/step | 10.4 samples/s +Step 32210/50462 | Loss: 2.5245 | LR: 1.49e-04 | 5749.9ms/step | 10.4 samples/s +Step 32220/50462 | Loss: 2.5202 | LR: 1.49e-04 | 5750.5ms/step | 10.4 samples/s +Step 32230/50462 | Loss: 2.5105 | LR: 1.48e-04 | 5743.5ms/step | 10.4 samples/s +Step 32240/50462 | Loss: 2.4975 | LR: 1.48e-04 | 5745.4ms/step | 10.4 samples/s +Step 32250/50462 | Loss: 2.5125 | LR: 1.48e-04 | 5742.1ms/step | 10.4 samples/s +Validation at step 32250: Loss = 2.5215, Perplexity = 12.45 +Step 32260/50462 | Loss: 2.5041 | LR: 1.48e-04 | 8603.4ms/step | 7.0 samples/s +Step 32270/50462 | Loss: 2.5214 | LR: 1.48e-04 | 5748.2ms/step | 10.4 samples/s +Step 32280/50462 | Loss: 2.5089 | LR: 1.48e-04 | 5747.6ms/step | 10.4 samples/s +Step 32290/50462 | Loss: 2.5230 | LR: 1.48e-04 | 5744.2ms/step | 10.4 samples/s +Step 32300/50462 | Loss: 2.5290 | LR: 1.47e-04 | 5737.4ms/step | 10.5 samples/s +Step 32310/50462 | Loss: 2.5504 | LR: 1.47e-04 | 5742.1ms/step | 10.4 samples/s +Step 32320/50462 | Loss: 2.4871 | LR: 1.47e-04 | 5740.0ms/step | 10.5 samples/s +Step 32330/50462 | Loss: 2.5318 | LR: 1.47e-04 | 5740.3ms/step | 10.5 samples/s +Step 32340/50462 | Loss: 2.5606 | LR: 1.47e-04 | 5741.2ms/step | 10.5 samples/s +Step 32350/50462 | Loss: 2.4966 | LR: 1.47e-04 | 6175.0ms/step | 9.7 samples/s +Step 32360/50462 | Loss: 2.5104 | LR: 1.47e-04 | 5742.0ms/step | 10.4 samples/s +Step 32370/50462 | Loss: 2.5205 | LR: 1.46e-04 | 5739.9ms/step | 10.5 samples/s +Step 32380/50462 | Loss: 2.4710 | LR: 1.46e-04 | 5739.9ms/step | 10.5 samples/s +Step 32390/50462 | Loss: 2.4990 | LR: 1.46e-04 | 5740.2ms/step | 10.5 samples/s +Step 32400/50462 | Loss: 2.5299 | LR: 1.46e-04 | 5736.8ms/step | 10.5 samples/s +Step 32410/50462 | Loss: 2.5252 | LR: 1.46e-04 | 5738.7ms/step | 10.5 samples/s +Step 32420/50462 | Loss: 2.5024 | LR: 1.46e-04 | 5737.9ms/step | 10.5 samples/s +Step 32430/50462 | Loss: 2.4982 | LR: 1.45e-04 | 5730.6ms/step | 10.5 samples/s +Step 32440/50462 | Loss: 2.5033 | LR: 1.45e-04 | 5737.4ms/step | 10.5 samples/s +Step 32450/50462 | Loss: 2.5205 | LR: 1.45e-04 | 5735.8ms/step | 10.5 samples/s +Step 32460/50462 | Loss: 2.5003 | LR: 1.45e-04 | 5735.2ms/step | 10.5 samples/s +Step 32470/50462 | Loss: 2.5248 | LR: 1.45e-04 | 5733.3ms/step | 10.5 samples/s +Step 32480/50462 | Loss: 2.5288 | LR: 1.45e-04 | 5734.0ms/step | 10.5 samples/s +Step 32490/50462 | Loss: 2.5048 | LR: 1.45e-04 | 5727.6ms/step | 10.5 samples/s +Step 32500/50462 | Loss: 2.5528 | LR: 1.44e-04 | 5729.1ms/step | 10.5 samples/s +Validation at step 32500: Loss = 2.5204, Perplexity = 12.43 +Step 32510/50462 | Loss: 2.5067 | LR: 1.44e-04 | 8576.8ms/step | 7.0 samples/s +Step 32520/50462 | Loss: 2.5068 | LR: 1.44e-04 | 5737.2ms/step | 10.5 samples/s +Step 32530/50462 | Loss: 2.5117 | LR: 1.44e-04 | 5731.7ms/step | 10.5 samples/s +Step 32540/50462 | Loss: 2.5065 | LR: 1.44e-04 | 5734.9ms/step | 10.5 samples/s +Step 32550/50462 | Loss: 2.5100 | LR: 1.44e-04 | 5729.1ms/step | 10.5 samples/s +Step 32560/50462 | Loss: 2.4724 | LR: 1.44e-04 | 5730.2ms/step | 10.5 samples/s +Step 32570/50462 | Loss: 2.4974 | LR: 1.43e-04 | 5732.0ms/step | 10.5 samples/s +Step 32580/50462 | Loss: 2.4936 | LR: 1.43e-04 | 5728.2ms/step | 10.5 samples/s +Step 32590/50462 | Loss: 2.5056 | LR: 1.43e-04 | 5729.7ms/step | 10.5 samples/s +Step 32600/50462 | Loss: 2.5080 | LR: 1.43e-04 | 5729.6ms/step | 10.5 samples/s +Step 32610/50462 | Loss: 2.5001 | LR: 1.43e-04 | 5726.4ms/step | 10.5 samples/s +Step 32620/50462 | Loss: 2.5149 | LR: 1.43e-04 | 5724.5ms/step | 10.5 samples/s +Step 32630/50462 | Loss: 2.4995 | LR: 1.43e-04 | 5725.8ms/step | 10.5 samples/s +Step 32640/50462 | Loss: 2.5062 | LR: 1.42e-04 | 6154.0ms/step | 9.7 samples/s +Step 32650/50462 | Loss: 2.5057 | LR: 1.42e-04 | 6177.8ms/step | 9.7 samples/s +Step 32660/50462 | Loss: 2.5225 | LR: 1.42e-04 | 5698.6ms/step | 10.5 samples/s +Step 32670/50462 | Loss: 2.5248 | LR: 1.42e-04 | 5689.6ms/step | 10.5 samples/s +Step 32680/50462 | Loss: 2.5152 | LR: 1.42e-04 | 5684.9ms/step | 10.6 samples/s +Step 32690/50462 | Loss: 2.4833 | LR: 1.42e-04 | 5682.6ms/step | 10.6 samples/s +Step 32700/50462 | Loss: 2.4821 | LR: 1.42e-04 | 5525.1ms/step | 10.9 samples/s +Step 32710/50462 | Loss: 2.5301 | LR: 1.41e-04 | 5324.7ms/step | 11.3 samples/s +Step 32720/50462 | Loss: 2.5191 | LR: 1.41e-04 | 5660.5ms/step | 10.6 samples/s +Step 32730/50462 | Loss: 2.4943 | LR: 1.41e-04 | 5663.1ms/step | 10.6 samples/s +Step 32740/50462 | Loss: 2.5288 | LR: 1.41e-04 | 5656.4ms/step | 10.6 samples/s +Step 32750/50462 | Loss: 2.5177 | LR: 1.41e-04 | 5651.1ms/step | 10.6 samples/s +Validation at step 32750: Loss = 2.5196, Perplexity = 12.42 +Step 32760/50462 | Loss: 2.5259 | LR: 1.41e-04 | 8452.0ms/step | 7.1 samples/s +Step 32770/50462 | Loss: 2.5064 | LR: 1.41e-04 | 5651.5ms/step | 10.6 samples/s +Step 32780/50462 | Loss: 2.4916 | LR: 1.41e-04 | 5643.7ms/step | 10.6 samples/s +Step 32790/50462 | Loss: 2.5182 | LR: 1.40e-04 | 5643.8ms/step | 10.6 samples/s +Step 32800/50462 | Loss: 2.4765 | LR: 1.40e-04 | 5646.2ms/step | 10.6 samples/s +Step 32810/50462 | Loss: 2.5052 | LR: 1.40e-04 | 5645.9ms/step | 10.6 samples/s +Step 32820/50462 | Loss: 2.5223 | LR: 1.40e-04 | 5647.9ms/step | 10.6 samples/s +Step 32830/50462 | Loss: 2.5218 | LR: 1.40e-04 | 5642.5ms/step | 10.6 samples/s +Step 32840/50462 | Loss: 2.5357 | LR: 1.40e-04 | 5655.2ms/step | 10.6 samples/s +Step 32850/50462 | Loss: 2.5503 | LR: 1.40e-04 | 5649.7ms/step | 10.6 samples/s +Step 32860/50462 | Loss: 2.5165 | LR: 1.39e-04 | 5643.6ms/step | 10.6 samples/s +Step 32870/50462 | Loss: 2.4958 | LR: 1.39e-04 | 5641.5ms/step | 10.6 samples/s +Step 32880/50462 | Loss: 2.5009 | LR: 1.39e-04 | 5647.6ms/step | 10.6 samples/s +Step 32890/50462 | Loss: 2.5281 | LR: 1.39e-04 | 5645.5ms/step | 10.6 samples/s +Step 32900/50462 | Loss: 2.5103 | LR: 1.39e-04 | 5641.7ms/step | 10.6 samples/s +Step 32910/50462 | Loss: 2.4989 | LR: 1.39e-04 | 5301.9ms/step | 11.3 samples/s +Step 32920/50462 | Loss: 2.5148 | LR: 1.39e-04 | 5129.0ms/step | 11.7 samples/s +Step 32930/50462 | Loss: 2.5075 | LR: 1.38e-04 | 5456.3ms/step | 11.0 samples/s +Step 32940/50462 | Loss: 2.5024 | LR: 1.38e-04 | 5697.8ms/step | 10.5 samples/s +Step 32950/50462 | Loss: 2.4923 | LR: 1.38e-04 | 5702.9ms/step | 10.5 samples/s +Step 32960/50462 | Loss: 2.4788 | LR: 1.38e-04 | 5700.0ms/step | 10.5 samples/s +Step 32970/50462 | Loss: 2.5266 | LR: 1.38e-04 | 5706.8ms/step | 10.5 samples/s +Step 32980/50462 | Loss: 2.4943 | LR: 1.38e-04 | 5714.4ms/step | 10.5 samples/s +Step 32990/50462 | Loss: 2.4887 | LR: 1.38e-04 | 5710.2ms/step | 10.5 samples/s +Step 33000/50462 | Loss: 2.5033 | LR: 1.37e-04 | 5717.1ms/step | 10.5 samples/s +Validation at step 33000: Loss = 2.5187, Perplexity = 12.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-33000 +Step 33010/50462 | Loss: 2.5205 | LR: 1.37e-04 | 9409.1ms/step | 6.4 samples/s +Step 33020/50462 | Loss: 2.4933 | LR: 1.37e-04 | 5704.7ms/step | 10.5 samples/s +Step 33030/50462 | Loss: 2.5234 | LR: 1.37e-04 | 5705.2ms/step | 10.5 samples/s +Step 33040/50462 | Loss: 2.5130 | LR: 1.37e-04 | 5703.3ms/step | 10.5 samples/s +Step 33050/50462 | Loss: 2.4782 | LR: 1.37e-04 | 5697.2ms/step | 10.5 samples/s +Step 33060/50462 | Loss: 2.5086 | LR: 1.37e-04 | 5697.4ms/step | 10.5 samples/s +Step 33070/50462 | Loss: 2.5025 | LR: 1.36e-04 | 5697.8ms/step | 10.5 samples/s +Step 33080/50462 | Loss: 2.5069 | LR: 1.36e-04 | 5694.9ms/step | 10.5 samples/s +Step 33090/50462 | Loss: 2.5050 | LR: 1.36e-04 | 5695.5ms/step | 10.5 samples/s +Step 33100/50462 | Loss: 2.4916 | LR: 1.36e-04 | 5700.1ms/step | 10.5 samples/s +Step 33110/50462 | Loss: 2.4907 | LR: 1.36e-04 | 5700.7ms/step | 10.5 samples/s +Step 33120/50462 | Loss: 2.4999 | LR: 1.36e-04 | 5677.8ms/step | 10.6 samples/s +Step 33130/50462 | Loss: 2.4958 | LR: 1.36e-04 | 5425.0ms/step | 11.1 samples/s +Step 33140/50462 | Loss: 2.5070 | LR: 1.35e-04 | 5409.0ms/step | 11.1 samples/s +Step 33150/50462 | Loss: 2.5142 | LR: 1.35e-04 | 5639.4ms/step | 10.6 samples/s +Step 33160/50462 | Loss: 2.5356 | LR: 1.35e-04 | 5637.0ms/step | 10.6 samples/s +Step 33170/50462 | Loss: 2.5136 | LR: 1.35e-04 | 5640.9ms/step | 10.6 samples/s +Step 33180/50462 | Loss: 2.4989 | LR: 1.35e-04 | 5643.6ms/step | 10.6 samples/s +Step 33190/50462 | Loss: 2.4882 | LR: 1.35e-04 | 5640.9ms/step | 10.6 samples/s +Step 33200/50462 | Loss: 2.5106 | LR: 1.35e-04 | 5650.6ms/step | 10.6 samples/s +Step 33210/50462 | Loss: 2.5388 | LR: 1.34e-04 | 5649.5ms/step | 10.6 samples/s +Step 33220/50462 | Loss: 2.5130 | LR: 1.34e-04 | 5641.2ms/step | 10.6 samples/s +Step 33230/50462 | Loss: 2.5088 | LR: 1.34e-04 | 5639.1ms/step | 10.6 samples/s +Step 33240/50462 | Loss: 2.4595 | LR: 1.34e-04 | 5635.5ms/step | 10.6 samples/s +Step 33250/50462 | Loss: 2.5245 | LR: 1.34e-04 | 5235.6ms/step | 11.5 samples/s +Validation at step 33250: Loss = 2.5177, Perplexity = 12.40 +Step 33260/50462 | Loss: 2.4736 | LR: 1.34e-04 | 8230.9ms/step | 7.3 samples/s +Step 33270/50462 | Loss: 2.5203 | LR: 1.34e-04 | 5707.6ms/step | 10.5 samples/s +Step 33280/50462 | Loss: 2.5186 | LR: 1.33e-04 | 5723.5ms/step | 10.5 samples/s +Step 33290/50462 | Loss: 2.5022 | LR: 1.33e-04 | 5709.4ms/step | 10.5 samples/s +Step 33300/50462 | Loss: 2.5134 | LR: 1.33e-04 | 5708.5ms/step | 10.5 samples/s +Step 33310/50462 | Loss: 2.5313 | LR: 1.33e-04 | 5700.5ms/step | 10.5 samples/s +Step 33320/50462 | Loss: 2.5156 | LR: 1.33e-04 | 5698.9ms/step | 10.5 samples/s +Step 33330/50462 | Loss: 2.5370 | LR: 1.33e-04 | 5700.3ms/step | 10.5 samples/s +Step 33340/50462 | Loss: 2.5592 | LR: 1.33e-04 | 5699.8ms/step | 10.5 samples/s +Step 33350/50462 | Loss: 2.5105 | LR: 1.32e-04 | 5717.2ms/step | 10.5 samples/s +Step 33360/50462 | Loss: 2.4874 | LR: 1.32e-04 | 5700.8ms/step | 10.5 samples/s +Step 33370/50462 | Loss: 2.4814 | LR: 1.32e-04 | 5337.8ms/step | 11.2 samples/s +Step 33380/50462 | Loss: 2.5154 | LR: 1.32e-04 | 5644.1ms/step | 10.6 samples/s +Step 33390/50462 | Loss: 2.4884 | LR: 1.32e-04 | 5661.0ms/step | 10.6 samples/s +Step 33400/50462 | Loss: 2.4943 | LR: 1.32e-04 | 5660.8ms/step | 10.6 samples/s +Step 33410/50462 | Loss: 2.4922 | LR: 1.32e-04 | 5644.4ms/step | 10.6 samples/s +Step 33420/50462 | Loss: 2.4987 | LR: 1.32e-04 | 5648.8ms/step | 10.6 samples/s +Step 33430/50462 | Loss: 2.5116 | LR: 1.31e-04 | 5650.6ms/step | 10.6 samples/s +Step 33440/50462 | Loss: 2.5515 | LR: 1.31e-04 | 5642.8ms/step | 10.6 samples/s +Step 33450/50462 | Loss: 2.5191 | LR: 1.31e-04 | 5647.4ms/step | 10.6 samples/s +Step 33460/50462 | Loss: 2.4966 | LR: 1.31e-04 | 5560.4ms/step | 10.8 samples/s +Step 33470/50462 | Loss: 2.4973 | LR: 1.31e-04 | 5128.5ms/step | 11.7 samples/s +Step 33480/50462 | Loss: 2.4814 | LR: 1.31e-04 | 5167.9ms/step | 11.6 samples/s +Step 33490/50462 | Loss: 2.5209 | LR: 1.31e-04 | 5446.7ms/step | 11.0 samples/s +Step 33500/50462 | Loss: 2.4975 | LR: 1.30e-04 | 5612.7ms/step | 10.7 samples/s +Validation at step 33500: Loss = 2.5170, Perplexity = 12.39 +Step 33510/50462 | Loss: 2.5196 | LR: 1.30e-04 | 8400.4ms/step | 7.1 samples/s +Step 33520/50462 | Loss: 2.5017 | LR: 1.30e-04 | 5617.8ms/step | 10.7 samples/s +Step 33530/50462 | Loss: 2.4725 | LR: 1.30e-04 | 5611.5ms/step | 10.7 samples/s +Step 33540/50462 | Loss: 2.5075 | LR: 1.30e-04 | 5612.3ms/step | 10.7 samples/s +Step 33550/50462 | Loss: 2.5082 | LR: 1.30e-04 | 5612.1ms/step | 10.7 samples/s +Step 33560/50462 | Loss: 2.4945 | LR: 1.30e-04 | 5612.2ms/step | 10.7 samples/s +Step 33570/50462 | Loss: 2.4666 | LR: 1.29e-04 | 5612.9ms/step | 10.7 samples/s +Step 33580/50462 | Loss: 2.5063 | LR: 1.29e-04 | 5608.1ms/step | 10.7 samples/s +Step 33590/50462 | Loss: 2.5055 | LR: 1.29e-04 | 5610.3ms/step | 10.7 samples/s +Step 33600/50462 | Loss: 2.4990 | LR: 1.29e-04 | 5604.6ms/step | 10.7 samples/s +Step 33610/50462 | Loss: 2.5227 | LR: 1.29e-04 | 5607.2ms/step | 10.7 samples/s +Step 33620/50462 | Loss: 2.5071 | LR: 1.29e-04 | 5607.8ms/step | 10.7 samples/s +Step 33630/50462 | Loss: 2.4599 | LR: 1.29e-04 | 5611.4ms/step | 10.7 samples/s +Step 33640/50462 | Loss: 2.5084 | LR: 1.28e-04 | 5624.6ms/step | 10.7 samples/s +Step 33650/50462 | Loss: 2.5008 | LR: 1.28e-04 | 5623.1ms/step | 10.7 samples/s +Step 33660/50462 | Loss: 2.4999 | LR: 1.28e-04 | 5621.8ms/step | 10.7 samples/s +Step 33670/50462 | Loss: 2.4910 | LR: 1.28e-04 | 5626.1ms/step | 10.7 samples/s +Step 33680/50462 | Loss: 2.5162 | LR: 1.28e-04 | 5627.7ms/step | 10.7 samples/s +Step 33690/50462 | Loss: 2.5026 | LR: 1.28e-04 | 5622.6ms/step | 10.7 samples/s +Step 33700/50462 | Loss: 2.4861 | LR: 1.28e-04 | 5627.6ms/step | 10.7 samples/s +Step 33710/50462 | Loss: 2.5287 | LR: 1.27e-04 | 5624.1ms/step | 10.7 samples/s +Step 33720/50462 | Loss: 2.5120 | LR: 1.27e-04 | 5627.7ms/step | 10.7 samples/s +Step 33730/50462 | Loss: 2.4797 | LR: 1.27e-04 | 5629.8ms/step | 10.7 samples/s +Step 33740/50462 | Loss: 2.4999 | LR: 1.27e-04 | 5629.9ms/step | 10.7 samples/s +Step 33750/50462 | Loss: 2.5146 | LR: 1.27e-04 | 5628.4ms/step | 10.7 samples/s +Validation at step 33750: Loss = 2.5162, Perplexity = 12.38 +Step 33760/50462 | Loss: 2.4750 | LR: 1.27e-04 | 8417.9ms/step | 7.1 samples/s +Step 33770/50462 | Loss: 2.4971 | LR: 1.27e-04 | 5620.0ms/step | 10.7 samples/s +Step 33780/50462 | Loss: 2.5153 | LR: 1.27e-04 | 5619.7ms/step | 10.7 samples/s +Step 33790/50462 | Loss: 2.5165 | LR: 1.26e-04 | 5619.7ms/step | 10.7 samples/s +Step 33800/50462 | Loss: 2.5056 | LR: 1.26e-04 | 5624.5ms/step | 10.7 samples/s +Step 33810/50462 | Loss: 2.5118 | LR: 1.26e-04 | 5619.3ms/step | 10.7 samples/s +Step 33820/50462 | Loss: 2.4755 | LR: 1.26e-04 | 5620.6ms/step | 10.7 samples/s +Step 33830/50462 | Loss: 2.4711 | LR: 1.26e-04 | 5620.5ms/step | 10.7 samples/s +Step 33840/50462 | Loss: 2.5128 | LR: 1.26e-04 | 5621.5ms/step | 10.7 samples/s +Step 33850/50462 | Loss: 2.4983 | LR: 1.26e-04 | 5184.4ms/step | 11.6 samples/s +Step 33860/50462 | Loss: 2.4998 | LR: 1.25e-04 | 5146.9ms/step | 11.7 samples/s +Step 33870/50462 | Loss: 2.4847 | LR: 1.25e-04 | 5158.4ms/step | 11.6 samples/s +Step 33880/50462 | Loss: 2.5445 | LR: 1.25e-04 | 5153.1ms/step | 11.6 samples/s +Step 33890/50462 | Loss: 2.5097 | LR: 1.25e-04 | 5140.5ms/step | 11.7 samples/s +Step 33900/50462 | Loss: 2.5280 | LR: 1.25e-04 | 5518.8ms/step | 10.9 samples/s +Step 33910/50462 | Loss: 2.4968 | LR: 1.25e-04 | 5772.0ms/step | 10.4 samples/s +Step 33920/50462 | Loss: 2.5334 | LR: 1.25e-04 | 5765.7ms/step | 10.4 samples/s +Step 33930/50462 | Loss: 2.4828 | LR: 1.24e-04 | 5778.5ms/step | 10.4 samples/s +Step 33940/50462 | Loss: 2.4947 | LR: 1.24e-04 | 5778.5ms/step | 10.4 samples/s +Step 33950/50462 | Loss: 2.5051 | LR: 1.24e-04 | 5770.9ms/step | 10.4 samples/s +Step 33960/50462 | Loss: 2.5356 | LR: 1.24e-04 | 5731.3ms/step | 10.5 samples/s +Step 33970/50462 | Loss: 2.4699 | LR: 1.24e-04 | 5694.8ms/step | 10.5 samples/s +Step 33980/50462 | Loss: 2.5159 | LR: 1.24e-04 | 5689.3ms/step | 10.5 samples/s +Step 33990/50462 | Loss: 2.5079 | LR: 1.24e-04 | 5678.4ms/step | 10.6 samples/s +Step 34000/50462 | Loss: 2.4681 | LR: 1.24e-04 | 5674.3ms/step | 10.6 samples/s +Validation at step 34000: Loss = 2.5152, Perplexity = 12.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-34000 +Step 34010/50462 | Loss: 2.4963 | LR: 1.23e-04 | 9251.1ms/step | 6.5 samples/s +Step 34020/50462 | Loss: 2.5078 | LR: 1.23e-04 | 5683.7ms/step | 10.6 samples/s +Step 34030/50462 | Loss: 2.4779 | LR: 1.23e-04 | 5676.4ms/step | 10.6 samples/s +Step 34040/50462 | Loss: 2.5119 | LR: 1.23e-04 | 5672.6ms/step | 10.6 samples/s +Step 34050/50462 | Loss: 2.5226 | LR: 1.23e-04 | 5670.7ms/step | 10.6 samples/s +Step 34060/50462 | Loss: 2.5111 | LR: 1.23e-04 | 5676.8ms/step | 10.6 samples/s +Step 34070/50462 | Loss: 2.4988 | LR: 1.23e-04 | 5668.5ms/step | 10.6 samples/s +Step 34080/50462 | Loss: 2.5243 | LR: 1.22e-04 | 5668.4ms/step | 10.6 samples/s +Step 34090/50462 | Loss: 2.5213 | LR: 1.22e-04 | 5665.8ms/step | 10.6 samples/s +Step 34100/50462 | Loss: 2.5235 | LR: 1.22e-04 | 5668.3ms/step | 10.6 samples/s +Step 34110/50462 | Loss: 2.5073 | LR: 1.22e-04 | 5660.5ms/step | 10.6 samples/s +Step 34120/50462 | Loss: 2.5013 | LR: 1.22e-04 | 5663.1ms/step | 10.6 samples/s +Step 34130/50462 | Loss: 2.4841 | LR: 1.22e-04 | 5655.4ms/step | 10.6 samples/s +Step 34140/50462 | Loss: 2.5295 | LR: 1.22e-04 | 5658.6ms/step | 10.6 samples/s +Step 34150/50462 | Loss: 2.4824 | LR: 1.21e-04 | 5654.4ms/step | 10.6 samples/s +Step 34160/50462 | Loss: 2.5137 | LR: 1.21e-04 | 5657.4ms/step | 10.6 samples/s +Step 34170/50462 | Loss: 2.4890 | LR: 1.21e-04 | 5661.8ms/step | 10.6 samples/s +Step 34180/50462 | Loss: 2.5028 | LR: 1.21e-04 | 5651.6ms/step | 10.6 samples/s +Step 34190/50462 | Loss: 2.4943 | LR: 1.21e-04 | 5652.1ms/step | 10.6 samples/s +Step 34200/50462 | Loss: 2.4908 | LR: 1.21e-04 | 5655.3ms/step | 10.6 samples/s +Step 34210/50462 | Loss: 2.5026 | LR: 1.21e-04 | 5650.0ms/step | 10.6 samples/s +Step 34220/50462 | Loss: 2.5037 | LR: 1.21e-04 | 5649.5ms/step | 10.6 samples/s +Step 34230/50462 | Loss: 2.5139 | LR: 1.20e-04 | 5651.5ms/step | 10.6 samples/s +Step 34240/50462 | Loss: 2.4852 | LR: 1.20e-04 | 5649.1ms/step | 10.6 samples/s +Step 34250/50462 | Loss: 2.4955 | LR: 1.20e-04 | 5647.4ms/step | 10.6 samples/s +Validation at step 34250: Loss = 2.5146, Perplexity = 12.36 +Step 34260/50462 | Loss: 2.4543 | LR: 1.20e-04 | 8447.1ms/step | 7.1 samples/s +Step 34270/50462 | Loss: 2.5172 | LR: 1.20e-04 | 5643.2ms/step | 10.6 samples/s +Step 34280/50462 | Loss: 2.5001 | LR: 1.20e-04 | 5493.7ms/step | 10.9 samples/s +Step 34290/50462 | Loss: 2.5245 | LR: 1.20e-04 | 5301.4ms/step | 11.3 samples/s +Step 34300/50462 | Loss: 2.5204 | LR: 1.19e-04 | 5672.7ms/step | 10.6 samples/s +Step 34310/50462 | Loss: 2.5335 | LR: 1.19e-04 | 5673.1ms/step | 10.6 samples/s +Step 34320/50462 | Loss: 2.4828 | LR: 1.19e-04 | 5675.8ms/step | 10.6 samples/s +Step 34330/50462 | Loss: 2.4905 | LR: 1.19e-04 | 5674.0ms/step | 10.6 samples/s +Step 34340/50462 | Loss: 2.4819 | LR: 1.19e-04 | 5673.2ms/step | 10.6 samples/s +Step 34350/50462 | Loss: 2.4965 | LR: 1.19e-04 | 5676.9ms/step | 10.6 samples/s +Step 34360/50462 | Loss: 2.5073 | LR: 1.19e-04 | 5674.5ms/step | 10.6 samples/s +Step 34370/50462 | Loss: 2.4945 | LR: 1.19e-04 | 5672.2ms/step | 10.6 samples/s +Step 34380/50462 | Loss: 2.4753 | LR: 1.18e-04 | 5673.4ms/step | 10.6 samples/s +Step 34390/50462 | Loss: 2.5015 | LR: 1.18e-04 | 5670.8ms/step | 10.6 samples/s +Step 34400/50462 | Loss: 2.5100 | LR: 1.18e-04 | 5673.5ms/step | 10.6 samples/s +Step 34410/50462 | Loss: 2.4965 | LR: 1.18e-04 | 5673.7ms/step | 10.6 samples/s +Step 34420/50462 | Loss: 2.4871 | LR: 1.18e-04 | 6072.0ms/step | 9.9 samples/s +Step 34430/50462 | Loss: 2.5080 | LR: 1.18e-04 | 5666.0ms/step | 10.6 samples/s +Step 34440/50462 | Loss: 2.4910 | LR: 1.18e-04 | 5665.6ms/step | 10.6 samples/s +Step 34450/50462 | Loss: 2.5248 | LR: 1.17e-04 | 5662.1ms/step | 10.6 samples/s +Step 34460/50462 | Loss: 2.4846 | LR: 1.17e-04 | 5671.2ms/step | 10.6 samples/s +Step 34470/50462 | Loss: 2.5231 | LR: 1.17e-04 | 5668.9ms/step | 10.6 samples/s +Step 34480/50462 | Loss: 2.5266 | LR: 1.17e-04 | 5666.1ms/step | 10.6 samples/s +Step 34490/50462 | Loss: 2.4967 | LR: 1.17e-04 | 5665.9ms/step | 10.6 samples/s +Step 34500/50462 | Loss: 2.5088 | LR: 1.17e-04 | 5669.2ms/step | 10.6 samples/s +Validation at step 34500: Loss = 2.5138, Perplexity = 12.35 +Step 34510/50462 | Loss: 2.5045 | LR: 1.17e-04 | 8495.3ms/step | 7.1 samples/s +Step 34520/50462 | Loss: 2.4877 | LR: 1.16e-04 | 5672.2ms/step | 10.6 samples/s +Step 34530/50462 | Loss: 2.4929 | LR: 1.16e-04 | 5671.2ms/step | 10.6 samples/s +Step 34540/50462 | Loss: 2.4960 | LR: 1.16e-04 | 5669.1ms/step | 10.6 samples/s +Step 34550/50462 | Loss: 2.4780 | LR: 1.16e-04 | 5667.2ms/step | 10.6 samples/s +Step 34560/50462 | Loss: 2.5165 | LR: 1.16e-04 | 5666.8ms/step | 10.6 samples/s +Step 34570/50462 | Loss: 2.4857 | LR: 1.16e-04 | 5669.1ms/step | 10.6 samples/s +Step 34580/50462 | Loss: 2.5004 | LR: 1.16e-04 | 5667.5ms/step | 10.6 samples/s +Step 34590/50462 | Loss: 2.5045 | LR: 1.16e-04 | 5669.6ms/step | 10.6 samples/s +Step 34600/50462 | Loss: 2.4924 | LR: 1.15e-04 | 5667.1ms/step | 10.6 samples/s +Step 34610/50462 | Loss: 2.5026 | LR: 1.15e-04 | 5668.0ms/step | 10.6 samples/s +Step 34620/50462 | Loss: 2.5093 | LR: 1.15e-04 | 5668.7ms/step | 10.6 samples/s +Step 34630/50462 | Loss: 2.4964 | LR: 1.15e-04 | 5669.9ms/step | 10.6 samples/s +Step 34640/50462 | Loss: 2.5272 | LR: 1.15e-04 | 5666.3ms/step | 10.6 samples/s +Step 34650/50462 | Loss: 2.5122 | LR: 1.15e-04 | 5665.6ms/step | 10.6 samples/s +Step 34660/50462 | Loss: 2.5115 | LR: 1.15e-04 | 5667.6ms/step | 10.6 samples/s +Step 34670/50462 | Loss: 2.5258 | LR: 1.15e-04 | 5669.3ms/step | 10.6 samples/s +Step 34680/50462 | Loss: 2.5151 | LR: 1.14e-04 | 5669.2ms/step | 10.6 samples/s +Step 34690/50462 | Loss: 2.5131 | LR: 1.14e-04 | 5673.3ms/step | 10.6 samples/s +Step 34700/50462 | Loss: 2.5108 | LR: 1.14e-04 | 5669.2ms/step | 10.6 samples/s +Step 34710/50462 | Loss: 2.5032 | LR: 1.14e-04 | 5668.2ms/step | 10.6 samples/s +Step 34720/50462 | Loss: 2.4977 | LR: 1.14e-04 | 5668.9ms/step | 10.6 samples/s +Step 34730/50462 | Loss: 2.4977 | LR: 1.14e-04 | 5672.4ms/step | 10.6 samples/s +Step 34740/50462 | Loss: 2.5117 | LR: 1.14e-04 | 5666.4ms/step | 10.6 samples/s +Step 34750/50462 | Loss: 2.5123 | LR: 1.13e-04 | 5668.4ms/step | 10.6 samples/s +Validation at step 34750: Loss = 2.5131, Perplexity = 12.34 +Step 34760/50462 | Loss: 2.5012 | LR: 1.13e-04 | 8917.0ms/step | 6.7 samples/s +Step 34770/50462 | Loss: 2.4821 | LR: 1.13e-04 | 6134.5ms/step | 9.8 samples/s +Step 34780/50462 | Loss: 2.4981 | LR: 1.13e-04 | 5668.1ms/step | 10.6 samples/s +Step 34790/50462 | Loss: 2.4898 | LR: 1.13e-04 | 5671.6ms/step | 10.6 samples/s +Step 34800/50462 | Loss: 2.5031 | LR: 1.13e-04 | 5669.7ms/step | 10.6 samples/s +Step 34810/50462 | Loss: 2.5182 | LR: 1.13e-04 | 5671.7ms/step | 10.6 samples/s +Step 34820/50462 | Loss: 2.5108 | LR: 1.13e-04 | 5674.9ms/step | 10.6 samples/s +Step 34830/50462 | Loss: 2.5031 | LR: 1.12e-04 | 5671.8ms/step | 10.6 samples/s +Step 34840/50462 | Loss: 2.5157 | LR: 1.12e-04 | 5674.9ms/step | 10.6 samples/s +Step 34850/50462 | Loss: 2.5359 | LR: 1.12e-04 | 5673.0ms/step | 10.6 samples/s +Step 34860/50462 | Loss: 2.5186 | LR: 1.12e-04 | 5671.5ms/step | 10.6 samples/s +Step 34870/50462 | Loss: 2.4813 | LR: 1.12e-04 | 5672.4ms/step | 10.6 samples/s +Step 34880/50462 | Loss: 2.5052 | LR: 1.12e-04 | 5672.4ms/step | 10.6 samples/s +Step 34890/50462 | Loss: 2.4940 | LR: 1.12e-04 | 5671.6ms/step | 10.6 samples/s +Step 34900/50462 | Loss: 2.5031 | LR: 1.11e-04 | 5672.5ms/step | 10.6 samples/s +Step 34910/50462 | Loss: 2.5111 | LR: 1.11e-04 | 5668.3ms/step | 10.6 samples/s +Step 34920/50462 | Loss: 2.4481 | LR: 1.11e-04 | 5672.9ms/step | 10.6 samples/s +Step 34930/50462 | Loss: 2.4869 | LR: 1.11e-04 | 5666.0ms/step | 10.6 samples/s +Step 34940/50462 | Loss: 2.5023 | LR: 1.11e-04 | 5667.1ms/step | 10.6 samples/s +Step 34950/50462 | Loss: 2.5052 | LR: 1.11e-04 | 5671.5ms/step | 10.6 samples/s +Step 34960/50462 | Loss: 2.5059 | LR: 1.11e-04 | 5670.4ms/step | 10.6 samples/s +Step 34970/50462 | Loss: 2.4847 | LR: 1.11e-04 | 5667.0ms/step | 10.6 samples/s +Step 34980/50462 | Loss: 2.4892 | LR: 1.10e-04 | 5670.8ms/step | 10.6 samples/s +Step 34990/50462 | Loss: 2.5204 | LR: 1.10e-04 | 5675.6ms/step | 10.6 samples/s +Step 35000/50462 | Loss: 2.5307 | LR: 1.10e-04 | 5669.3ms/step | 10.6 samples/s +Validation at step 35000: Loss = 2.5124, Perplexity = 12.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-35000 +Step 35010/50462 | Loss: 2.5102 | LR: 1.10e-04 | 9326.6ms/step | 6.4 samples/s +Step 35020/50462 | Loss: 2.4942 | LR: 1.10e-04 | 5671.4ms/step | 10.6 samples/s +Step 35030/50462 | Loss: 2.5055 | LR: 1.10e-04 | 5672.1ms/step | 10.6 samples/s +Step 35040/50462 | Loss: 2.4903 | LR: 1.10e-04 | 5668.2ms/step | 10.6 samples/s +Step 35050/50462 | Loss: 2.4924 | LR: 1.09e-04 | 5669.7ms/step | 10.6 samples/s +Step 35060/50462 | Loss: 2.4855 | LR: 1.09e-04 | 5670.2ms/step | 10.6 samples/s +Step 35070/50462 | Loss: 2.4812 | LR: 1.09e-04 | 5675.4ms/step | 10.6 samples/s +Step 35080/50462 | Loss: 2.5189 | LR: 1.09e-04 | 5667.2ms/step | 10.6 samples/s +Step 35090/50462 | Loss: 2.4744 | LR: 1.09e-04 | 5663.7ms/step | 10.6 samples/s +Step 35100/50462 | Loss: 2.5064 | LR: 1.09e-04 | 5670.0ms/step | 10.6 samples/s +Step 35110/50462 | Loss: 2.5083 | LR: 1.09e-04 | 5669.8ms/step | 10.6 samples/s +Step 35120/50462 | Loss: 2.4914 | LR: 1.09e-04 | 5670.3ms/step | 10.6 samples/s +Step 35130/50462 | Loss: 2.4701 | LR: 1.08e-04 | 5671.2ms/step | 10.6 samples/s +Step 35140/50462 | Loss: 2.4807 | LR: 1.08e-04 | 5669.4ms/step | 10.6 samples/s +Step 35150/50462 | Loss: 2.4998 | LR: 1.08e-04 | 5669.3ms/step | 10.6 samples/s +Step 35160/50462 | Loss: 2.5113 | LR: 1.08e-04 | 5669.9ms/step | 10.6 samples/s +Step 35170/50462 | Loss: 2.5049 | LR: 1.08e-04 | 5667.0ms/step | 10.6 samples/s +Step 35180/50462 | Loss: 2.4971 | LR: 1.08e-04 | 5665.4ms/step | 10.6 samples/s +Step 35190/50462 | Loss: 2.5081 | LR: 1.08e-04 | 5667.7ms/step | 10.6 samples/s +Step 35200/50462 | Loss: 2.4997 | LR: 1.08e-04 | 5666.9ms/step | 10.6 samples/s +Step 35210/50462 | Loss: 2.4977 | LR: 1.07e-04 | 5670.0ms/step | 10.6 samples/s +Step 35220/50462 | Loss: 2.5026 | LR: 1.07e-04 | 5670.4ms/step | 10.6 samples/s +Step 35230/50462 | Loss: 2.5128 | LR: 1.07e-04 | 5671.0ms/step | 10.6 samples/s +Step 35240/50462 | Loss: 2.5050 | LR: 1.07e-04 | 5668.6ms/step | 10.6 samples/s +Step 35250/50462 | Loss: 2.4798 | LR: 1.07e-04 | 5505.8ms/step | 10.9 samples/s +Validation at step 35250: Loss = 2.5117, Perplexity = 12.33 +Step 35260/50462 | Loss: 2.4598 | LR: 1.07e-04 | 7712.7ms/step | 7.8 samples/s +Step 35270/50462 | Loss: 2.4878 | LR: 1.07e-04 | 5149.8ms/step | 11.7 samples/s +Step 35280/50462 | Loss: 2.4895 | LR: 1.07e-04 | 5222.8ms/step | 11.5 samples/s +Step 35290/50462 | Loss: 2.4720 | LR: 1.06e-04 | 5730.5ms/step | 10.5 samples/s +Step 35300/50462 | Loss: 2.4910 | LR: 1.06e-04 | 5743.0ms/step | 10.4 samples/s +Step 35310/50462 | Loss: 2.5150 | LR: 1.06e-04 | 5740.2ms/step | 10.5 samples/s +Step 35320/50462 | Loss: 2.5066 | LR: 1.06e-04 | 5741.1ms/step | 10.5 samples/s +Step 35330/50462 | Loss: 2.5192 | LR: 1.06e-04 | 5739.8ms/step | 10.5 samples/s +Step 35340/50462 | Loss: 2.4984 | LR: 1.06e-04 | 5742.0ms/step | 10.4 samples/s +Step 35350/50462 | Loss: 2.5098 | LR: 1.06e-04 | 5749.1ms/step | 10.4 samples/s +Step 35360/50462 | Loss: 2.5001 | LR: 1.05e-04 | 5752.7ms/step | 10.4 samples/s +Step 35370/50462 | Loss: 2.5126 | LR: 1.05e-04 | 5745.7ms/step | 10.4 samples/s +Step 35380/50462 | Loss: 2.5019 | LR: 1.05e-04 | 5744.8ms/step | 10.4 samples/s +Step 35390/50462 | Loss: 2.5048 | LR: 1.05e-04 | 5747.2ms/step | 10.4 samples/s +Step 35400/50462 | Loss: 2.5257 | LR: 1.05e-04 | 5748.2ms/step | 10.4 samples/s +Step 35410/50462 | Loss: 2.5048 | LR: 1.05e-04 | 5748.3ms/step | 10.4 samples/s +Step 35420/50462 | Loss: 2.5008 | LR: 1.05e-04 | 5742.8ms/step | 10.4 samples/s +Step 35430/50462 | Loss: 2.5000 | LR: 1.05e-04 | 5747.8ms/step | 10.4 samples/s +Step 35440/50462 | Loss: 2.5009 | LR: 1.04e-04 | 5744.2ms/step | 10.4 samples/s +Step 35450/50462 | Loss: 2.4968 | LR: 1.04e-04 | 5743.9ms/step | 10.4 samples/s +Step 35460/50462 | Loss: 2.5017 | LR: 1.04e-04 | 5741.8ms/step | 10.4 samples/s +Step 35470/50462 | Loss: 2.4958 | LR: 1.04e-04 | 5741.9ms/step | 10.4 samples/s +Step 35480/50462 | Loss: 2.4876 | LR: 1.04e-04 | 5741.5ms/step | 10.5 samples/s +Step 35490/50462 | Loss: 2.5002 | LR: 1.04e-04 | 5738.0ms/step | 10.5 samples/s +Step 35500/50462 | Loss: 2.5205 | LR: 1.04e-04 | 5743.4ms/step | 10.4 samples/s +Validation at step 35500: Loss = 2.5114, Perplexity = 12.32 +Step 35510/50462 | Loss: 2.5044 | LR: 1.04e-04 | 8598.3ms/step | 7.0 samples/s +Step 35520/50462 | Loss: 2.4851 | LR: 1.03e-04 | 5743.0ms/step | 10.4 samples/s +Step 35530/50462 | Loss: 2.5027 | LR: 1.03e-04 | 5745.8ms/step | 10.4 samples/s +Step 35540/50462 | Loss: 2.4603 | LR: 1.03e-04 | 5743.8ms/step | 10.4 samples/s +Step 35550/50462 | Loss: 2.5044 | LR: 1.03e-04 | 5744.1ms/step | 10.4 samples/s +Step 35560/50462 | Loss: 2.5083 | LR: 1.03e-04 | 5743.8ms/step | 10.4 samples/s +Step 35570/50462 | Loss: 2.5145 | LR: 1.03e-04 | 5745.5ms/step | 10.4 samples/s +Step 35580/50462 | Loss: 2.4911 | LR: 1.03e-04 | 5741.9ms/step | 10.4 samples/s +Step 35590/50462 | Loss: 2.5154 | LR: 1.03e-04 | 5746.3ms/step | 10.4 samples/s +Step 35600/50462 | Loss: 2.4676 | LR: 1.02e-04 | 5743.9ms/step | 10.4 samples/s +Step 35610/50462 | Loss: 2.4851 | LR: 1.02e-04 | 5745.0ms/step | 10.4 samples/s +Step 35620/50462 | Loss: 2.4909 | LR: 1.02e-04 | 5746.8ms/step | 10.4 samples/s +Step 35630/50462 | Loss: 2.5058 | LR: 1.02e-04 | 5749.4ms/step | 10.4 samples/s +Step 35640/50462 | Loss: 2.4772 | LR: 1.02e-04 | 5748.9ms/step | 10.4 samples/s +Step 35650/50462 | Loss: 2.4996 | LR: 1.02e-04 | 5751.6ms/step | 10.4 samples/s +Step 35660/50462 | Loss: 2.4954 | LR: 1.02e-04 | 5749.5ms/step | 10.4 samples/s +Step 35670/50462 | Loss: 2.4786 | LR: 1.02e-04 | 5752.7ms/step | 10.4 samples/s +Step 35680/50462 | Loss: 2.5161 | LR: 1.01e-04 | 5746.7ms/step | 10.4 samples/s +Step 35690/50462 | Loss: 2.4830 | LR: 1.01e-04 | 5752.4ms/step | 10.4 samples/s +Step 35700/50462 | Loss: 2.4939 | LR: 1.01e-04 | 5750.9ms/step | 10.4 samples/s +Step 35710/50462 | Loss: 2.5276 | LR: 1.01e-04 | 5745.2ms/step | 10.4 samples/s +Step 35720/50462 | Loss: 2.4652 | LR: 1.01e-04 | 5747.4ms/step | 10.4 samples/s +Step 35730/50462 | Loss: 2.5265 | LR: 1.01e-04 | 5746.1ms/step | 10.4 samples/s +Step 35740/50462 | Loss: 2.4791 | LR: 1.01e-04 | 5748.5ms/step | 10.4 samples/s +Step 35750/50462 | Loss: 2.5003 | LR: 1.00e-04 | 5752.7ms/step | 10.4 samples/s +Validation at step 35750: Loss = 2.5106, Perplexity = 12.31 +Step 35760/50462 | Loss: 2.4949 | LR: 1.00e-04 | 8611.0ms/step | 7.0 samples/s +Step 35770/50462 | Loss: 2.5000 | LR: 1.00e-04 | 5751.0ms/step | 10.4 samples/s +Step 35780/50462 | Loss: 2.4785 | LR: 1.00e-04 | 5747.6ms/step | 10.4 samples/s +Step 35790/50462 | Loss: 2.4992 | LR: 1.00e-04 | 5743.5ms/step | 10.4 samples/s +Step 35800/50462 | Loss: 2.4867 | LR: 9.99e-05 | 5742.8ms/step | 10.4 samples/s +Step 35810/50462 | Loss: 2.5099 | LR: 9.97e-05 | 5750.2ms/step | 10.4 samples/s +Step 35820/50462 | Loss: 2.5110 | LR: 9.96e-05 | 5745.7ms/step | 10.4 samples/s +Step 35830/50462 | Loss: 2.4913 | LR: 9.95e-05 | 5741.9ms/step | 10.4 samples/s +Step 35840/50462 | Loss: 2.4973 | LR: 9.94e-05 | 5739.9ms/step | 10.5 samples/s +Step 35850/50462 | Loss: 2.4899 | LR: 9.92e-05 | 5739.9ms/step | 10.5 samples/s +Step 35860/50462 | Loss: 2.4899 | LR: 9.91e-05 | 5742.3ms/step | 10.4 samples/s +Step 35870/50462 | Loss: 2.5026 | LR: 9.90e-05 | 5742.3ms/step | 10.4 samples/s +Step 35880/50462 | Loss: 2.5112 | LR: 9.88e-05 | 5743.1ms/step | 10.4 samples/s +Step 35890/50462 | Loss: 2.5041 | LR: 9.87e-05 | 5737.6ms/step | 10.5 samples/s +Step 35900/50462 | Loss: 2.4833 | LR: 9.86e-05 | 5742.9ms/step | 10.4 samples/s +Step 35910/50462 | Loss: 2.5188 | LR: 9.85e-05 | 5743.0ms/step | 10.4 samples/s +Step 35920/50462 | Loss: 2.5136 | LR: 9.83e-05 | 5740.6ms/step | 10.5 samples/s +Step 35930/50462 | Loss: 2.4899 | LR: 9.82e-05 | 5738.3ms/step | 10.5 samples/s +Step 35940/50462 | Loss: 2.4897 | LR: 9.81e-05 | 5741.4ms/step | 10.5 samples/s +Step 35950/50462 | Loss: 2.4715 | LR: 9.80e-05 | 5732.5ms/step | 10.5 samples/s +Step 35960/50462 | Loss: 2.4964 | LR: 9.78e-05 | 5733.5ms/step | 10.5 samples/s +Step 35970/50462 | Loss: 2.4891 | LR: 9.77e-05 | 5731.2ms/step | 10.5 samples/s +Step 35980/50462 | Loss: 2.4858 | LR: 9.76e-05 | 5738.6ms/step | 10.5 samples/s +Step 35990/50462 | Loss: 2.4781 | LR: 9.75e-05 | 5738.2ms/step | 10.5 samples/s +Step 36000/50462 | Loss: 2.4772 | LR: 9.73e-05 | 5735.1ms/step | 10.5 samples/s +Validation at step 36000: Loss = 2.5099, Perplexity = 12.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-36000 +Step 36010/50462 | Loss: 2.4733 | LR: 9.72e-05 | 9380.7ms/step | 6.4 samples/s +Step 36020/50462 | Loss: 2.4992 | LR: 9.71e-05 | 5700.5ms/step | 10.5 samples/s +Step 36030/50462 | Loss: 2.4760 | LR: 9.70e-05 | 5702.8ms/step | 10.5 samples/s +Step 36040/50462 | Loss: 2.5312 | LR: 9.68e-05 | 5706.4ms/step | 10.5 samples/s +Step 36050/50462 | Loss: 2.5003 | LR: 9.67e-05 | 5700.9ms/step | 10.5 samples/s +Step 36060/50462 | Loss: 2.5094 | LR: 9.66e-05 | 5695.7ms/step | 10.5 samples/s +Step 36070/50462 | Loss: 2.4870 | LR: 9.65e-05 | 5698.9ms/step | 10.5 samples/s +Step 36080/50462 | Loss: 2.5041 | LR: 9.63e-05 | 5702.2ms/step | 10.5 samples/s +Step 36090/50462 | Loss: 2.4826 | LR: 9.62e-05 | 5703.1ms/step | 10.5 samples/s +Step 36100/50462 | Loss: 2.5156 | LR: 9.61e-05 | 5697.8ms/step | 10.5 samples/s +Step 36110/50462 | Loss: 2.4757 | LR: 9.60e-05 | 5696.7ms/step | 10.5 samples/s +Step 36120/50462 | Loss: 2.5055 | LR: 9.58e-05 | 5701.1ms/step | 10.5 samples/s +Step 36130/50462 | Loss: 2.5070 | LR: 9.57e-05 | 5704.1ms/step | 10.5 samples/s +Step 36140/50462 | Loss: 2.5073 | LR: 9.56e-05 | 5701.8ms/step | 10.5 samples/s +Step 36150/50462 | Loss: 2.4801 | LR: 9.55e-05 | 5698.2ms/step | 10.5 samples/s +Step 36160/50462 | Loss: 2.4803 | LR: 9.53e-05 | 5696.6ms/step | 10.5 samples/s +Step 36170/50462 | Loss: 2.4983 | LR: 9.52e-05 | 5700.6ms/step | 10.5 samples/s +Step 36180/50462 | Loss: 2.4957 | LR: 9.51e-05 | 5695.5ms/step | 10.5 samples/s +Step 36190/50462 | Loss: 2.5063 | LR: 9.50e-05 | 5696.2ms/step | 10.5 samples/s +Step 36200/50462 | Loss: 2.5230 | LR: 9.49e-05 | 5692.7ms/step | 10.5 samples/s +Step 36210/50462 | Loss: 2.5143 | LR: 9.47e-05 | 5693.2ms/step | 10.5 samples/s +Step 36220/50462 | Loss: 2.4873 | LR: 9.46e-05 | 5692.9ms/step | 10.5 samples/s +Step 36230/50462 | Loss: 2.4993 | LR: 9.45e-05 | 5689.9ms/step | 10.5 samples/s +Step 36240/50462 | Loss: 2.5189 | LR: 9.44e-05 | 5691.1ms/step | 10.5 samples/s +Step 36250/50462 | Loss: 2.4856 | LR: 9.42e-05 | 5705.0ms/step | 10.5 samples/s +Validation at step 36250: Loss = 2.5095, Perplexity = 12.30 +Step 36260/50462 | Loss: 2.5048 | LR: 9.41e-05 | 8485.9ms/step | 7.1 samples/s +Step 36270/50462 | Loss: 2.4800 | LR: 9.40e-05 | 5322.1ms/step | 11.3 samples/s +Step 36280/50462 | Loss: 2.5006 | LR: 9.39e-05 | 5501.3ms/step | 10.9 samples/s +Step 36290/50462 | Loss: 2.4675 | LR: 9.37e-05 | 5658.4ms/step | 10.6 samples/s +Step 36300/50462 | Loss: 2.4572 | LR: 9.36e-05 | 5664.0ms/step | 10.6 samples/s +Step 36310/50462 | Loss: 2.4841 | LR: 9.35e-05 | 5664.1ms/step | 10.6 samples/s +Step 36320/50462 | Loss: 2.4917 | LR: 9.34e-05 | 5660.6ms/step | 10.6 samples/s +Step 36330/50462 | Loss: 2.5045 | LR: 9.32e-05 | 5662.2ms/step | 10.6 samples/s +Step 36340/50462 | Loss: 2.5047 | LR: 9.31e-05 | 5665.0ms/step | 10.6 samples/s +Step 36350/50462 | Loss: 2.5003 | LR: 9.30e-05 | 5661.3ms/step | 10.6 samples/s +Step 36360/50462 | Loss: 2.5072 | LR: 9.29e-05 | 5663.1ms/step | 10.6 samples/s +Step 36370/50462 | Loss: 2.5392 | LR: 9.28e-05 | 5662.3ms/step | 10.6 samples/s +Step 36380/50462 | Loss: 2.4761 | LR: 9.26e-05 | 5660.1ms/step | 10.6 samples/s +Step 36390/50462 | Loss: 2.4786 | LR: 9.25e-05 | 5663.7ms/step | 10.6 samples/s +Step 36400/50462 | Loss: 2.5113 | LR: 9.24e-05 | 5660.5ms/step | 10.6 samples/s +Step 36410/50462 | Loss: 2.4830 | LR: 9.23e-05 | 5665.8ms/step | 10.6 samples/s +Step 36420/50462 | Loss: 2.5157 | LR: 9.21e-05 | 5660.4ms/step | 10.6 samples/s +Step 36430/50462 | Loss: 2.5253 | LR: 9.20e-05 | 5662.0ms/step | 10.6 samples/s +Step 36440/50462 | Loss: 2.4784 | LR: 9.19e-05 | 5664.2ms/step | 10.6 samples/s +Step 36450/50462 | Loss: 2.4785 | LR: 9.18e-05 | 5658.4ms/step | 10.6 samples/s +Step 36460/50462 | Loss: 2.5040 | LR: 9.17e-05 | 5662.4ms/step | 10.6 samples/s +Step 36470/50462 | Loss: 2.5013 | LR: 9.15e-05 | 5662.7ms/step | 10.6 samples/s +Step 36480/50462 | Loss: 2.4987 | LR: 9.14e-05 | 5776.2ms/step | 10.4 samples/s +Step 36490/50462 | Loss: 2.5209 | LR: 9.13e-05 | 5154.3ms/step | 11.6 samples/s +Step 36500/50462 | Loss: 2.5203 | LR: 9.12e-05 | 5475.1ms/step | 11.0 samples/s +Validation at step 36500: Loss = 2.5089, Perplexity = 12.29 +Step 36510/50462 | Loss: 2.4856 | LR: 9.10e-05 | 8540.0ms/step | 7.0 samples/s +Step 36520/50462 | Loss: 2.4821 | LR: 9.09e-05 | 5709.3ms/step | 10.5 samples/s +Step 36530/50462 | Loss: 2.5325 | LR: 9.08e-05 | 5704.1ms/step | 10.5 samples/s +Step 36540/50462 | Loss: 2.4952 | LR: 9.07e-05 | 5731.7ms/step | 10.5 samples/s +Step 36550/50462 | Loss: 2.4966 | LR: 9.06e-05 | 5739.4ms/step | 10.5 samples/s +Step 36560/50462 | Loss: 2.4892 | LR: 9.04e-05 | 5738.3ms/step | 10.5 samples/s +Step 36570/50462 | Loss: 2.4831 | LR: 9.03e-05 | 5732.7ms/step | 10.5 samples/s +Step 36580/50462 | Loss: 2.4934 | LR: 9.02e-05 | 5736.3ms/step | 10.5 samples/s +Step 36590/50462 | Loss: 2.5119 | LR: 9.01e-05 | 5725.9ms/step | 10.5 samples/s +Step 36600/50462 | Loss: 2.4733 | LR: 8.99e-05 | 5725.1ms/step | 10.5 samples/s +Step 36610/50462 | Loss: 2.4842 | LR: 8.98e-05 | 5713.0ms/step | 10.5 samples/s +Step 36620/50462 | Loss: 2.4871 | LR: 8.97e-05 | 5706.5ms/step | 10.5 samples/s +Step 36630/50462 | Loss: 2.4765 | LR: 8.96e-05 | 5696.8ms/step | 10.5 samples/s +Step 36640/50462 | Loss: 2.5163 | LR: 8.95e-05 | 5710.4ms/step | 10.5 samples/s +Step 36650/50462 | Loss: 2.4764 | LR: 8.93e-05 | 5717.1ms/step | 10.5 samples/s +Step 36660/50462 | Loss: 2.5119 | LR: 8.92e-05 | 5713.2ms/step | 10.5 samples/s +Step 36670/50462 | Loss: 2.4789 | LR: 8.91e-05 | 5713.2ms/step | 10.5 samples/s +Step 36680/50462 | Loss: 2.4843 | LR: 8.90e-05 | 5703.8ms/step | 10.5 samples/s +Step 36690/50462 | Loss: 2.5276 | LR: 8.89e-05 | 5694.4ms/step | 10.5 samples/s +Step 36700/50462 | Loss: 2.4935 | LR: 8.87e-05 | 5462.6ms/step | 11.0 samples/s +Step 36710/50462 | Loss: 2.4761 | LR: 8.86e-05 | 5439.8ms/step | 11.0 samples/s +Step 36720/50462 | Loss: 2.4833 | LR: 8.85e-05 | 5656.2ms/step | 10.6 samples/s +Step 36730/50462 | Loss: 2.5246 | LR: 8.84e-05 | 5664.0ms/step | 10.6 samples/s +Step 36740/50462 | Loss: 2.4992 | LR: 8.83e-05 | 5641.8ms/step | 10.6 samples/s +Step 36750/50462 | Loss: 2.4992 | LR: 8.81e-05 | 5662.4ms/step | 10.6 samples/s +Validation at step 36750: Loss = 2.5083, Perplexity = 12.28 +Step 36760/50462 | Loss: 2.4767 | LR: 8.80e-05 | 8469.0ms/step | 7.1 samples/s +Step 36770/50462 | Loss: 2.4954 | LR: 8.79e-05 | 5646.0ms/step | 10.6 samples/s +Step 36780/50462 | Loss: 2.5241 | LR: 8.78e-05 | 5640.7ms/step | 10.6 samples/s +Step 36790/50462 | Loss: 2.4798 | LR: 8.77e-05 | 5653.8ms/step | 10.6 samples/s +Step 36800/50462 | Loss: 2.4879 | LR: 8.75e-05 | 5656.0ms/step | 10.6 samples/s +Step 36810/50462 | Loss: 2.5096 | LR: 8.74e-05 | 5439.7ms/step | 11.0 samples/s +Step 36820/50462 | Loss: 2.5114 | LR: 8.73e-05 | 5240.1ms/step | 11.5 samples/s +Step 36830/50462 | Loss: 2.5137 | LR: 8.72e-05 | 5719.7ms/step | 10.5 samples/s +Step 36840/50462 | Loss: 2.5098 | LR: 8.71e-05 | 5718.2ms/step | 10.5 samples/s +Step 36850/50462 | Loss: 2.5077 | LR: 8.69e-05 | 5732.1ms/step | 10.5 samples/s +Step 36860/50462 | Loss: 2.5156 | LR: 8.68e-05 | 5726.4ms/step | 10.5 samples/s +Step 36870/50462 | Loss: 2.4832 | LR: 8.67e-05 | 6189.2ms/step | 9.7 samples/s +Step 36880/50462 | Loss: 2.4807 | LR: 8.66e-05 | 5720.0ms/step | 10.5 samples/s +Step 36890/50462 | Loss: 2.4667 | LR: 8.65e-05 | 6183.9ms/step | 9.7 samples/s +Step 36900/50462 | Loss: 2.4760 | LR: 8.63e-05 | 5718.4ms/step | 10.5 samples/s +Step 36910/50462 | Loss: 2.5179 | LR: 8.62e-05 | 5709.2ms/step | 10.5 samples/s +Step 36920/50462 | Loss: 2.4977 | LR: 8.61e-05 | 5705.0ms/step | 10.5 samples/s +Step 36930/50462 | Loss: 2.4946 | LR: 8.60e-05 | 5531.6ms/step | 10.8 samples/s +Step 36940/50462 | Loss: 2.4986 | LR: 8.59e-05 | 5427.1ms/step | 11.1 samples/s +Step 36950/50462 | Loss: 2.5001 | LR: 8.57e-05 | 5635.7ms/step | 10.6 samples/s +Step 36960/50462 | Loss: 2.4794 | LR: 8.56e-05 | 5643.9ms/step | 10.6 samples/s +Step 36970/50462 | Loss: 2.4833 | LR: 8.55e-05 | 5643.6ms/step | 10.6 samples/s +Step 36980/50462 | Loss: 2.4612 | LR: 8.54e-05 | 5645.4ms/step | 10.6 samples/s +Step 36990/50462 | Loss: 2.4629 | LR: 8.53e-05 | 5649.0ms/step | 10.6 samples/s +Step 37000/50462 | Loss: 2.4846 | LR: 8.51e-05 | 5652.7ms/step | 10.6 samples/s +Validation at step 37000: Loss = 2.5079, Perplexity = 12.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-37000 +Step 37010/50462 | Loss: 2.5239 | LR: 8.50e-05 | 9201.3ms/step | 6.5 samples/s +Step 37020/50462 | Loss: 2.4946 | LR: 8.49e-05 | 5558.5ms/step | 10.8 samples/s +Step 37030/50462 | Loss: 2.4831 | LR: 8.48e-05 | 5161.6ms/step | 11.6 samples/s +Step 37040/50462 | Loss: 2.4530 | LR: 8.47e-05 | 5144.1ms/step | 11.7 samples/s +Step 37050/50462 | Loss: 2.5184 | LR: 8.46e-05 | 5445.6ms/step | 11.0 samples/s +Step 37060/50462 | Loss: 2.4834 | LR: 8.44e-05 | 5625.2ms/step | 10.7 samples/s +Step 37070/50462 | Loss: 2.4817 | LR: 8.43e-05 | 5634.6ms/step | 10.6 samples/s +Step 37080/50462 | Loss: 2.5253 | LR: 8.42e-05 | 5641.7ms/step | 10.6 samples/s +Step 37090/50462 | Loss: 2.4939 | LR: 8.41e-05 | 5637.7ms/step | 10.6 samples/s +Step 37100/50462 | Loss: 2.4949 | LR: 8.40e-05 | 5635.5ms/step | 10.6 samples/s +Step 37110/50462 | Loss: 2.4929 | LR: 8.38e-05 | 5637.5ms/step | 10.6 samples/s +Step 37120/50462 | Loss: 2.4781 | LR: 8.37e-05 | 5635.1ms/step | 10.6 samples/s +Step 37130/50462 | Loss: 2.5055 | LR: 8.36e-05 | 5635.4ms/step | 10.6 samples/s +Step 37140/50462 | Loss: 2.4905 | LR: 8.35e-05 | 5632.6ms/step | 10.7 samples/s +Step 37150/50462 | Loss: 2.5238 | LR: 8.34e-05 | 5630.2ms/step | 10.7 samples/s +Step 37160/50462 | Loss: 2.5018 | LR: 8.33e-05 | 5630.7ms/step | 10.7 samples/s +Step 37170/50462 | Loss: 2.5087 | LR: 8.31e-05 | 5628.2ms/step | 10.7 samples/s +Step 37180/50462 | Loss: 2.4830 | LR: 8.30e-05 | 5629.6ms/step | 10.7 samples/s +Step 37190/50462 | Loss: 2.4973 | LR: 8.29e-05 | 5626.5ms/step | 10.7 samples/s +Step 37200/50462 | Loss: 2.4983 | LR: 8.28e-05 | 5625.7ms/step | 10.7 samples/s +Step 37210/50462 | Loss: 2.4503 | LR: 8.27e-05 | 5632.3ms/step | 10.7 samples/s +Step 37220/50462 | Loss: 2.4946 | LR: 8.25e-05 | 5621.6ms/step | 10.7 samples/s +Step 37230/50462 | Loss: 2.4828 | LR: 8.24e-05 | 5622.0ms/step | 10.7 samples/s +Step 37240/50462 | Loss: 2.4836 | LR: 8.23e-05 | 5621.8ms/step | 10.7 samples/s +Step 37250/50462 | Loss: 2.4901 | LR: 8.22e-05 | 5625.0ms/step | 10.7 samples/s +Validation at step 37250: Loss = 2.5072, Perplexity = 12.27 +Step 37260/50462 | Loss: 2.4964 | LR: 8.21e-05 | 8416.9ms/step | 7.1 samples/s +Step 37270/50462 | Loss: 2.5114 | LR: 8.20e-05 | 5624.3ms/step | 10.7 samples/s +Step 37280/50462 | Loss: 2.5079 | LR: 8.18e-05 | 5622.2ms/step | 10.7 samples/s +Step 37290/50462 | Loss: 2.4833 | LR: 8.17e-05 | 5617.4ms/step | 10.7 samples/s +Step 37300/50462 | Loss: 2.4980 | LR: 8.16e-05 | 5618.2ms/step | 10.7 samples/s +Step 37310/50462 | Loss: 2.4779 | LR: 8.15e-05 | 5618.7ms/step | 10.7 samples/s +Step 37320/50462 | Loss: 2.5002 | LR: 8.14e-05 | 5620.1ms/step | 10.7 samples/s +Step 37330/50462 | Loss: 2.5056 | LR: 8.13e-05 | 5623.6ms/step | 10.7 samples/s +Step 37340/50462 | Loss: 2.4627 | LR: 8.11e-05 | 5622.0ms/step | 10.7 samples/s +Step 37350/50462 | Loss: 2.5065 | LR: 8.10e-05 | 5619.3ms/step | 10.7 samples/s +Step 37360/50462 | Loss: 2.5123 | LR: 8.09e-05 | 5619.9ms/step | 10.7 samples/s +Step 37370/50462 | Loss: 2.4819 | LR: 8.08e-05 | 5614.9ms/step | 10.7 samples/s +Step 37380/50462 | Loss: 2.4978 | LR: 8.07e-05 | 5620.4ms/step | 10.7 samples/s +Step 37390/50462 | Loss: 2.5077 | LR: 8.06e-05 | 5619.2ms/step | 10.7 samples/s +Step 37400/50462 | Loss: 2.4989 | LR: 8.04e-05 | 5619.8ms/step | 10.7 samples/s +Step 37410/50462 | Loss: 2.4947 | LR: 8.03e-05 | 5345.1ms/step | 11.2 samples/s +Step 37420/50462 | Loss: 2.5009 | LR: 8.02e-05 | 5157.3ms/step | 11.6 samples/s +Step 37430/50462 | Loss: 2.5040 | LR: 8.01e-05 | 5177.6ms/step | 11.6 samples/s +Step 37440/50462 | Loss: 2.4891 | LR: 8.00e-05 | 5155.3ms/step | 11.6 samples/s +Step 37450/50462 | Loss: 2.5244 | LR: 7.99e-05 | 5160.5ms/step | 11.6 samples/s +Step 37460/50462 | Loss: 2.4897 | LR: 7.98e-05 | 5162.4ms/step | 11.6 samples/s +Step 37470/50462 | Loss: 2.5120 | LR: 7.96e-05 | 5323.8ms/step | 11.3 samples/s +Step 37480/50462 | Loss: 2.4884 | LR: 7.95e-05 | 5762.4ms/step | 10.4 samples/s +Step 37490/50462 | Loss: 2.5039 | LR: 7.94e-05 | 5725.0ms/step | 10.5 samples/s +Step 37500/50462 | Loss: 2.4860 | LR: 7.93e-05 | 5768.7ms/step | 10.4 samples/s +Validation at step 37500: Loss = 2.5069, Perplexity = 12.27 +Step 37510/50462 | Loss: 2.4983 | LR: 7.92e-05 | 8614.5ms/step | 7.0 samples/s +Step 37520/50462 | Loss: 2.4755 | LR: 7.91e-05 | 5761.0ms/step | 10.4 samples/s +Step 37530/50462 | Loss: 2.4963 | LR: 7.89e-05 | 5708.6ms/step | 10.5 samples/s +Step 37540/50462 | Loss: 2.4934 | LR: 7.88e-05 | 5676.0ms/step | 10.6 samples/s +Step 37550/50462 | Loss: 2.5126 | LR: 7.87e-05 | 5683.4ms/step | 10.6 samples/s +Step 37560/50462 | Loss: 2.4880 | LR: 7.86e-05 | 5660.3ms/step | 10.6 samples/s +Step 37570/50462 | Loss: 2.4999 | LR: 7.85e-05 | 5662.4ms/step | 10.6 samples/s +Step 37580/50462 | Loss: 2.4896 | LR: 7.84e-05 | 5670.7ms/step | 10.6 samples/s +Step 37590/50462 | Loss: 2.4924 | LR: 7.83e-05 | 5686.7ms/step | 10.6 samples/s +Step 37600/50462 | Loss: 2.4950 | LR: 7.81e-05 | 5686.3ms/step | 10.6 samples/s +Step 37610/50462 | Loss: 2.5028 | LR: 7.80e-05 | 5669.3ms/step | 10.6 samples/s +Step 37620/50462 | Loss: 2.5064 | LR: 7.79e-05 | 5671.6ms/step | 10.6 samples/s +Step 37630/50462 | Loss: 2.4809 | LR: 7.78e-05 | 5669.0ms/step | 10.6 samples/s +Step 37640/50462 | Loss: 2.4921 | LR: 7.77e-05 | 5673.3ms/step | 10.6 samples/s +Step 37650/50462 | Loss: 2.4927 | LR: 7.76e-05 | 5668.6ms/step | 10.6 samples/s +Step 37660/50462 | Loss: 2.4979 | LR: 7.75e-05 | 5667.0ms/step | 10.6 samples/s +Step 37670/50462 | Loss: 2.4661 | LR: 7.73e-05 | 5664.9ms/step | 10.6 samples/s +Step 37680/50462 | Loss: 2.4958 | LR: 7.72e-05 | 5666.6ms/step | 10.6 samples/s +Step 37690/50462 | Loss: 2.5127 | LR: 7.71e-05 | 5664.1ms/step | 10.6 samples/s +Step 37700/50462 | Loss: 2.4833 | LR: 7.70e-05 | 5659.1ms/step | 10.6 samples/s +Step 37710/50462 | Loss: 2.5201 | LR: 7.69e-05 | 5660.8ms/step | 10.6 samples/s +Step 37720/50462 | Loss: 2.4565 | LR: 7.68e-05 | 5658.7ms/step | 10.6 samples/s +Step 37730/50462 | Loss: 2.5054 | LR: 7.67e-05 | 5654.0ms/step | 10.6 samples/s +Step 37740/50462 | Loss: 2.5162 | LR: 7.65e-05 | 5657.1ms/step | 10.6 samples/s +Step 37750/50462 | Loss: 2.4714 | LR: 7.64e-05 | 5651.8ms/step | 10.6 samples/s +Validation at step 37750: Loss = 2.5066, Perplexity = 12.26 +Step 37760/50462 | Loss: 2.4976 | LR: 7.63e-05 | 8451.1ms/step | 7.1 samples/s +Step 37770/50462 | Loss: 2.4999 | LR: 7.62e-05 | 5650.7ms/step | 10.6 samples/s +Step 37780/50462 | Loss: 2.4747 | LR: 7.61e-05 | 5649.5ms/step | 10.6 samples/s +Step 37790/50462 | Loss: 2.4570 | LR: 7.60e-05 | 5643.2ms/step | 10.6 samples/s +Step 37800/50462 | Loss: 2.4755 | LR: 7.59e-05 | 5647.9ms/step | 10.6 samples/s +Step 37810/50462 | Loss: 2.4879 | LR: 7.57e-05 | 5646.1ms/step | 10.6 samples/s +Step 37820/50462 | Loss: 2.5040 | LR: 7.56e-05 | 5647.0ms/step | 10.6 samples/s +Step 37830/50462 | Loss: 2.5061 | LR: 7.55e-05 | 5644.0ms/step | 10.6 samples/s +Step 37840/50462 | Loss: 2.5121 | LR: 7.54e-05 | 5642.5ms/step | 10.6 samples/s +Step 37850/50462 | Loss: 2.5112 | LR: 7.53e-05 | 5642.7ms/step | 10.6 samples/s +Step 37860/50462 | Loss: 2.5138 | LR: 7.52e-05 | 5380.8ms/step | 11.2 samples/s +Step 37870/50462 | Loss: 2.4906 | LR: 7.51e-05 | 5275.3ms/step | 11.4 samples/s +Step 37880/50462 | Loss: 2.5215 | LR: 7.50e-05 | 5671.6ms/step | 10.6 samples/s +Step 37890/50462 | Loss: 2.4999 | LR: 7.48e-05 | 5673.1ms/step | 10.6 samples/s +Step 37900/50462 | Loss: 2.4702 | LR: 7.47e-05 | 5669.7ms/step | 10.6 samples/s +Step 37910/50462 | Loss: 2.4978 | LR: 7.46e-05 | 5671.3ms/step | 10.6 samples/s +Step 37920/50462 | Loss: 2.4696 | LR: 7.45e-05 | 5669.5ms/step | 10.6 samples/s +Step 37930/50462 | Loss: 2.4777 | LR: 7.44e-05 | 5673.8ms/step | 10.6 samples/s +Step 37940/50462 | Loss: 2.4924 | LR: 7.43e-05 | 5673.3ms/step | 10.6 samples/s +Step 37950/50462 | Loss: 2.5059 | LR: 7.42e-05 | 5672.5ms/step | 10.6 samples/s +Step 37960/50462 | Loss: 2.5076 | LR: 7.41e-05 | 5670.8ms/step | 10.6 samples/s +Step 37970/50462 | Loss: 2.5107 | LR: 7.39e-05 | 5666.2ms/step | 10.6 samples/s +Step 37980/50462 | Loss: 2.4783 | LR: 7.38e-05 | 5675.0ms/step | 10.6 samples/s +Step 37990/50462 | Loss: 2.4671 | LR: 7.37e-05 | 5672.2ms/step | 10.6 samples/s +Step 38000/50462 | Loss: 2.4910 | LR: 7.36e-05 | 5674.4ms/step | 10.6 samples/s +Validation at step 38000: Loss = 2.5061, Perplexity = 12.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-38000 +Step 38010/50462 | Loss: 2.4958 | LR: 7.35e-05 | 9372.5ms/step | 6.4 samples/s +Step 38020/50462 | Loss: 2.4853 | LR: 7.34e-05 | 5672.3ms/step | 10.6 samples/s +Step 38030/50462 | Loss: 2.5112 | LR: 7.33e-05 | 5674.4ms/step | 10.6 samples/s +Step 38040/50462 | Loss: 2.4787 | LR: 7.32e-05 | 5676.2ms/step | 10.6 samples/s +Step 38050/50462 | Loss: 2.4979 | LR: 7.30e-05 | 5674.9ms/step | 10.6 samples/s +Step 38060/50462 | Loss: 2.4944 | LR: 7.29e-05 | 5675.0ms/step | 10.6 samples/s +Step 38070/50462 | Loss: 2.5348 | LR: 7.28e-05 | 5677.4ms/step | 10.6 samples/s +Step 38080/50462 | Loss: 2.4939 | LR: 7.27e-05 | 5675.4ms/step | 10.6 samples/s +Step 38090/50462 | Loss: 2.4933 | LR: 7.26e-05 | 5674.1ms/step | 10.6 samples/s +Step 38100/50462 | Loss: 2.5165 | LR: 7.25e-05 | 5671.9ms/step | 10.6 samples/s +Step 38110/50462 | Loss: 2.4499 | LR: 7.24e-05 | 5670.5ms/step | 10.6 samples/s +Step 38120/50462 | Loss: 2.5012 | LR: 7.23e-05 | 5672.8ms/step | 10.6 samples/s +Step 38130/50462 | Loss: 2.4986 | LR: 7.22e-05 | 5673.2ms/step | 10.6 samples/s +Step 38140/50462 | Loss: 2.5012 | LR: 7.20e-05 | 5675.3ms/step | 10.6 samples/s +Step 38150/50462 | Loss: 2.5220 | LR: 7.19e-05 | 5674.8ms/step | 10.6 samples/s +Step 38160/50462 | Loss: 2.4779 | LR: 7.18e-05 | 5671.5ms/step | 10.6 samples/s +Step 38170/50462 | Loss: 2.5082 | LR: 7.17e-05 | 5671.4ms/step | 10.6 samples/s +Step 38180/50462 | Loss: 2.5034 | LR: 7.16e-05 | 5671.1ms/step | 10.6 samples/s +Step 38190/50462 | Loss: 2.4905 | LR: 7.15e-05 | 5668.8ms/step | 10.6 samples/s +Step 38200/50462 | Loss: 2.5257 | LR: 7.14e-05 | 5674.3ms/step | 10.6 samples/s +Step 38210/50462 | Loss: 2.5155 | LR: 7.13e-05 | 5673.4ms/step | 10.6 samples/s +Step 38220/50462 | Loss: 2.4847 | LR: 7.12e-05 | 5674.9ms/step | 10.6 samples/s +Step 38230/50462 | Loss: 2.4996 | LR: 7.10e-05 | 5674.9ms/step | 10.6 samples/s +Step 38240/50462 | Loss: 2.4981 | LR: 7.09e-05 | 5674.5ms/step | 10.6 samples/s +Step 38250/50462 | Loss: 2.5031 | LR: 7.08e-05 | 5671.2ms/step | 10.6 samples/s +Validation at step 38250: Loss = 2.5057, Perplexity = 12.25 +Step 38260/50462 | Loss: 2.4857 | LR: 7.07e-05 | 8496.0ms/step | 7.1 samples/s +Step 38270/50462 | Loss: 2.5233 | LR: 7.06e-05 | 5671.1ms/step | 10.6 samples/s +Step 38280/50462 | Loss: 2.4656 | LR: 7.05e-05 | 5674.4ms/step | 10.6 samples/s +Step 38290/50462 | Loss: 2.4875 | LR: 7.04e-05 | 5674.6ms/step | 10.6 samples/s +Step 38300/50462 | Loss: 2.4947 | LR: 7.03e-05 | 5667.3ms/step | 10.6 samples/s +Step 38310/50462 | Loss: 2.5216 | LR: 7.02e-05 | 5668.1ms/step | 10.6 samples/s +Step 38320/50462 | Loss: 2.5282 | LR: 7.01e-05 | 5668.5ms/step | 10.6 samples/s +Step 38330/50462 | Loss: 2.4866 | LR: 6.99e-05 | 5672.4ms/step | 10.6 samples/s +Step 38340/50462 | Loss: 2.4882 | LR: 6.98e-05 | 5676.2ms/step | 10.6 samples/s +Step 38350/50462 | Loss: 2.5083 | LR: 6.97e-05 | 5670.9ms/step | 10.6 samples/s +Step 38360/50462 | Loss: 2.5047 | LR: 6.96e-05 | 5670.1ms/step | 10.6 samples/s +Step 38370/50462 | Loss: 2.5032 | LR: 6.95e-05 | 5674.0ms/step | 10.6 samples/s +Step 38380/50462 | Loss: 2.4823 | LR: 6.94e-05 | 5673.5ms/step | 10.6 samples/s +Step 38390/50462 | Loss: 2.4997 | LR: 6.93e-05 | 5674.1ms/step | 10.6 samples/s +Step 38400/50462 | Loss: 2.4920 | LR: 6.92e-05 | 5670.6ms/step | 10.6 samples/s +Step 38410/50462 | Loss: 2.4958 | LR: 6.91e-05 | 5671.8ms/step | 10.6 samples/s +Step 38420/50462 | Loss: 2.5097 | LR: 6.90e-05 | 5669.5ms/step | 10.6 samples/s +Step 38430/50462 | Loss: 2.4952 | LR: 6.89e-05 | 5672.1ms/step | 10.6 samples/s +Step 38440/50462 | Loss: 2.4835 | LR: 6.87e-05 | 5668.5ms/step | 10.6 samples/s +Step 38450/50462 | Loss: 2.5058 | LR: 6.86e-05 | 5670.7ms/step | 10.6 samples/s +Step 38460/50462 | Loss: 2.4937 | LR: 6.85e-05 | 5668.5ms/step | 10.6 samples/s +Step 38470/50462 | Loss: 2.4923 | LR: 6.84e-05 | 5674.3ms/step | 10.6 samples/s +Step 38480/50462 | Loss: 2.4816 | LR: 6.83e-05 | 5669.6ms/step | 10.6 samples/s +Step 38490/50462 | Loss: 2.5195 | LR: 6.82e-05 | 5673.0ms/step | 10.6 samples/s +Step 38500/50462 | Loss: 2.4969 | LR: 6.81e-05 | 5671.1ms/step | 10.6 samples/s +Validation at step 38500: Loss = 2.5054, Perplexity = 12.25 +Step 38510/50462 | Loss: 2.4937 | LR: 6.80e-05 | 8500.8ms/step | 7.1 samples/s +Step 38520/50462 | Loss: 2.4822 | LR: 6.79e-05 | 5670.1ms/step | 10.6 samples/s +Step 38530/50462 | Loss: 2.4876 | LR: 6.78e-05 | 5667.4ms/step | 10.6 samples/s +Step 38540/50462 | Loss: 2.4660 | LR: 6.77e-05 | 5668.4ms/step | 10.6 samples/s +Step 38550/50462 | Loss: 2.5037 | LR: 6.76e-05 | 6127.0ms/step | 9.8 samples/s +Step 38560/50462 | Loss: 2.4764 | LR: 6.74e-05 | 5666.8ms/step | 10.6 samples/s +Step 38570/50462 | Loss: 2.4799 | LR: 6.73e-05 | 5663.3ms/step | 10.6 samples/s +Step 38580/50462 | Loss: 2.4916 | LR: 6.72e-05 | 5669.0ms/step | 10.6 samples/s +Step 38590/50462 | Loss: 2.4911 | LR: 6.71e-05 | 5665.0ms/step | 10.6 samples/s +Step 38600/50462 | Loss: 2.4972 | LR: 6.70e-05 | 5668.7ms/step | 10.6 samples/s +Step 38610/50462 | Loss: 2.4730 | LR: 6.69e-05 | 5665.4ms/step | 10.6 samples/s +Step 38620/50462 | Loss: 2.4803 | LR: 6.68e-05 | 5661.6ms/step | 10.6 samples/s +Step 38630/50462 | Loss: 2.5047 | LR: 6.67e-05 | 5656.1ms/step | 10.6 samples/s +Step 38640/50462 | Loss: 2.4814 | LR: 6.66e-05 | 5663.6ms/step | 10.6 samples/s +Step 38650/50462 | Loss: 2.5100 | LR: 6.65e-05 | 5667.1ms/step | 10.6 samples/s +Step 38660/50462 | Loss: 2.4783 | LR: 6.64e-05 | 5665.9ms/step | 10.6 samples/s +Step 38670/50462 | Loss: 2.4866 | LR: 6.63e-05 | 5666.3ms/step | 10.6 samples/s +Step 38680/50462 | Loss: 2.4774 | LR: 6.62e-05 | 5658.2ms/step | 10.6 samples/s +Step 38690/50462 | Loss: 2.4550 | LR: 6.60e-05 | 5665.4ms/step | 10.6 samples/s +Step 38700/50462 | Loss: 2.5065 | LR: 6.59e-05 | 5667.2ms/step | 10.6 samples/s +Step 38710/50462 | Loss: 2.4809 | LR: 6.58e-05 | 5656.3ms/step | 10.6 samples/s +Step 38720/50462 | Loss: 2.4821 | LR: 6.57e-05 | 5664.1ms/step | 10.6 samples/s +Step 38730/50462 | Loss: 2.5241 | LR: 6.56e-05 | 5655.4ms/step | 10.6 samples/s +Step 38740/50462 | Loss: 2.4919 | LR: 6.55e-05 | 5668.9ms/step | 10.6 samples/s +Step 38750/50462 | Loss: 2.4965 | LR: 6.54e-05 | 5670.3ms/step | 10.6 samples/s +Validation at step 38750: Loss = 2.5050, Perplexity = 12.24 +Step 38760/50462 | Loss: 2.4987 | LR: 6.53e-05 | 8482.0ms/step | 7.1 samples/s +Step 38770/50462 | Loss: 2.5122 | LR: 6.52e-05 | 5665.9ms/step | 10.6 samples/s +Step 38780/50462 | Loss: 2.4845 | LR: 6.51e-05 | 5665.8ms/step | 10.6 samples/s +Step 38790/50462 | Loss: 2.5365 | LR: 6.50e-05 | 5665.7ms/step | 10.6 samples/s +Step 38800/50462 | Loss: 2.5005 | LR: 6.49e-05 | 5665.5ms/step | 10.6 samples/s +Step 38810/50462 | Loss: 2.4839 | LR: 6.48e-05 | 5664.5ms/step | 10.6 samples/s +Step 38820/50462 | Loss: 2.4889 | LR: 6.47e-05 | 5664.5ms/step | 10.6 samples/s +Step 38830/50462 | Loss: 2.4974 | LR: 6.46e-05 | 5393.6ms/step | 11.1 samples/s +Step 38840/50462 | Loss: 2.5081 | LR: 6.45e-05 | 5159.8ms/step | 11.6 samples/s +Step 38850/50462 | Loss: 2.5174 | LR: 6.43e-05 | 5149.8ms/step | 11.7 samples/s +Step 38860/50462 | Loss: 2.4793 | LR: 6.42e-05 | 5165.6ms/step | 11.6 samples/s +Step 38870/50462 | Loss: 2.4959 | LR: 6.41e-05 | 5329.4ms/step | 11.3 samples/s +Step 38880/50462 | Loss: 2.4911 | LR: 6.40e-05 | 5737.4ms/step | 10.5 samples/s +Step 38890/50462 | Loss: 2.5022 | LR: 6.39e-05 | 5745.9ms/step | 10.4 samples/s +Step 38900/50462 | Loss: 2.4986 | LR: 6.38e-05 | 5745.1ms/step | 10.4 samples/s +Step 38910/50462 | Loss: 2.5002 | LR: 6.37e-05 | 5742.6ms/step | 10.4 samples/s +Step 38920/50462 | Loss: 2.4915 | LR: 6.36e-05 | 5749.9ms/step | 10.4 samples/s +Step 38930/50462 | Loss: 2.5072 | LR: 6.35e-05 | 5746.4ms/step | 10.4 samples/s +Step 38940/50462 | Loss: 2.4988 | LR: 6.34e-05 | 5750.2ms/step | 10.4 samples/s +Step 38950/50462 | Loss: 2.5063 | LR: 6.33e-05 | 5743.7ms/step | 10.4 samples/s +Step 38960/50462 | Loss: 2.4895 | LR: 6.32e-05 | 5747.5ms/step | 10.4 samples/s +Step 38970/50462 | Loss: 2.4847 | LR: 6.31e-05 | 5746.7ms/step | 10.4 samples/s +Step 38980/50462 | Loss: 2.4916 | LR: 6.30e-05 | 5750.1ms/step | 10.4 samples/s +Step 38990/50462 | Loss: 2.4684 | LR: 6.29e-05 | 6207.6ms/step | 9.7 samples/s +Step 39000/50462 | Loss: 2.4998 | LR: 6.28e-05 | 6261.5ms/step | 9.6 samples/s +Validation at step 39000: Loss = 2.5047, Perplexity = 12.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-39000 +Step 39010/50462 | Loss: 2.5150 | LR: 6.27e-05 | 9372.3ms/step | 6.4 samples/s +Step 39020/50462 | Loss: 2.4688 | LR: 6.26e-05 | 5734.9ms/step | 10.5 samples/s +Step 39030/50462 | Loss: 2.4995 | LR: 6.25e-05 | 5744.1ms/step | 10.4 samples/s +Step 39040/50462 | Loss: 2.4886 | LR: 6.23e-05 | 5734.2ms/step | 10.5 samples/s +Step 39050/50462 | Loss: 2.4661 | LR: 6.22e-05 | 5744.0ms/step | 10.4 samples/s +Step 39060/50462 | Loss: 2.4789 | LR: 6.21e-05 | 5744.8ms/step | 10.4 samples/s +Step 39070/50462 | Loss: 2.5006 | LR: 6.20e-05 | 5746.0ms/step | 10.4 samples/s +Step 39080/50462 | Loss: 2.5161 | LR: 6.19e-05 | 5745.2ms/step | 10.4 samples/s +Step 39090/50462 | Loss: 2.4745 | LR: 6.18e-05 | 5745.5ms/step | 10.4 samples/s +Step 39100/50462 | Loss: 2.4974 | LR: 6.17e-05 | 5741.7ms/step | 10.4 samples/s +Step 39110/50462 | Loss: 2.4706 | LR: 6.16e-05 | 5741.4ms/step | 10.5 samples/s +Step 39120/50462 | Loss: 2.4760 | LR: 6.15e-05 | 5740.4ms/step | 10.5 samples/s +Step 39130/50462 | Loss: 2.5149 | LR: 6.14e-05 | 5744.6ms/step | 10.4 samples/s +Step 39140/50462 | Loss: 2.4870 | LR: 6.13e-05 | 5742.0ms/step | 10.4 samples/s +Step 39150/50462 | Loss: 2.4911 | LR: 6.12e-05 | 5740.8ms/step | 10.5 samples/s +Step 39160/50462 | Loss: 2.4782 | LR: 6.11e-05 | 5747.0ms/step | 10.4 samples/s +Step 39170/50462 | Loss: 2.5067 | LR: 6.10e-05 | 5743.9ms/step | 10.4 samples/s +Step 39180/50462 | Loss: 2.5185 | LR: 6.09e-05 | 5744.7ms/step | 10.4 samples/s +Step 39190/50462 | Loss: 2.5002 | LR: 6.08e-05 | 5745.2ms/step | 10.4 samples/s +Step 39200/50462 | Loss: 2.4696 | LR: 6.07e-05 | 5739.9ms/step | 10.5 samples/s +Step 39210/50462 | Loss: 2.4926 | LR: 6.06e-05 | 5742.2ms/step | 10.4 samples/s +Step 39220/50462 | Loss: 2.4674 | LR: 6.05e-05 | 5747.4ms/step | 10.4 samples/s +Step 39230/50462 | Loss: 2.5111 | LR: 6.04e-05 | 5749.0ms/step | 10.4 samples/s +Step 39240/50462 | Loss: 2.4893 | LR: 6.03e-05 | 5743.6ms/step | 10.4 samples/s +Step 39250/50462 | Loss: 2.5100 | LR: 6.02e-05 | 5751.1ms/step | 10.4 samples/s +Validation at step 39250: Loss = 2.5045, Perplexity = 12.24 +Step 39260/50462 | Loss: 2.4937 | LR: 6.01e-05 | 8605.4ms/step | 7.0 samples/s +Step 39270/50462 | Loss: 2.4952 | LR: 6.00e-05 | 5744.9ms/step | 10.4 samples/s +Step 39280/50462 | Loss: 2.5088 | LR: 5.99e-05 | 5734.3ms/step | 10.5 samples/s +Step 39290/50462 | Loss: 2.4883 | LR: 5.98e-05 | 5746.3ms/step | 10.4 samples/s +Step 39300/50462 | Loss: 2.5067 | LR: 5.97e-05 | 5750.4ms/step | 10.4 samples/s +Step 39310/50462 | Loss: 2.4942 | LR: 5.96e-05 | 5751.0ms/step | 10.4 samples/s +Step 39320/50462 | Loss: 2.4492 | LR: 5.95e-05 | 5748.8ms/step | 10.4 samples/s +Step 39330/50462 | Loss: 2.4940 | LR: 5.94e-05 | 5750.5ms/step | 10.4 samples/s +Step 39340/50462 | Loss: 2.4994 | LR: 5.93e-05 | 5748.0ms/step | 10.4 samples/s +Step 39350/50462 | Loss: 2.4935 | LR: 5.91e-05 | 5750.3ms/step | 10.4 samples/s +Step 39360/50462 | Loss: 2.4670 | LR: 5.90e-05 | 5747.6ms/step | 10.4 samples/s +Step 39370/50462 | Loss: 2.4957 | LR: 5.89e-05 | 5748.2ms/step | 10.4 samples/s +Step 39380/50462 | Loss: 2.4845 | LR: 5.88e-05 | 5750.9ms/step | 10.4 samples/s +Step 39390/50462 | Loss: 2.4994 | LR: 5.87e-05 | 5749.7ms/step | 10.4 samples/s +Step 39400/50462 | Loss: 2.5054 | LR: 5.86e-05 | 5746.6ms/step | 10.4 samples/s +Step 39410/50462 | Loss: 2.4551 | LR: 5.85e-05 | 5747.1ms/step | 10.4 samples/s +Step 39420/50462 | Loss: 2.4737 | LR: 5.84e-05 | 5745.1ms/step | 10.4 samples/s +Step 39430/50462 | Loss: 2.4717 | LR: 5.83e-05 | 5744.1ms/step | 10.4 samples/s +Step 39440/50462 | Loss: 2.4926 | LR: 5.82e-05 | 5737.4ms/step | 10.5 samples/s +Step 39450/50462 | Loss: 2.4814 | LR: 5.81e-05 | 5740.6ms/step | 10.5 samples/s +Step 39460/50462 | Loss: 2.4939 | LR: 5.80e-05 | 5729.4ms/step | 10.5 samples/s +Step 39470/50462 | Loss: 2.5295 | LR: 5.79e-05 | 5736.6ms/step | 10.5 samples/s +Step 39480/50462 | Loss: 2.5143 | LR: 5.78e-05 | 5741.7ms/step | 10.4 samples/s +Step 39490/50462 | Loss: 2.4850 | LR: 5.77e-05 | 5739.8ms/step | 10.5 samples/s +Step 39500/50462 | Loss: 2.4997 | LR: 5.76e-05 | 5736.2ms/step | 10.5 samples/s +Validation at step 39500: Loss = 2.5042, Perplexity = 12.23 +Step 39510/50462 | Loss: 2.4952 | LR: 5.75e-05 | 8565.3ms/step | 7.0 samples/s +Step 39520/50462 | Loss: 2.5071 | LR: 5.74e-05 | 5716.9ms/step | 10.5 samples/s +Step 39530/50462 | Loss: 2.4969 | LR: 5.73e-05 | 5721.8ms/step | 10.5 samples/s +Step 39540/50462 | Loss: 2.4959 | LR: 5.72e-05 | 5713.1ms/step | 10.5 samples/s +Step 39550/50462 | Loss: 2.4979 | LR: 5.71e-05 | 5708.9ms/step | 10.5 samples/s +Step 39560/50462 | Loss: 2.4869 | LR: 5.70e-05 | 5723.2ms/step | 10.5 samples/s +Step 39570/50462 | Loss: 2.4977 | LR: 5.69e-05 | 5725.4ms/step | 10.5 samples/s +Step 39580/50462 | Loss: 2.4797 | LR: 5.68e-05 | 5715.2ms/step | 10.5 samples/s +Step 39590/50462 | Loss: 2.5051 | LR: 5.67e-05 | 5711.0ms/step | 10.5 samples/s +Step 39600/50462 | Loss: 2.4881 | LR: 5.66e-05 | 5708.2ms/step | 10.5 samples/s +Step 39610/50462 | Loss: 2.4805 | LR: 5.65e-05 | 5705.2ms/step | 10.5 samples/s +Step 39620/50462 | Loss: 2.4752 | LR: 5.64e-05 | 5706.9ms/step | 10.5 samples/s +Step 39630/50462 | Loss: 2.4670 | LR: 5.63e-05 | 5707.3ms/step | 10.5 samples/s +Step 39640/50462 | Loss: 2.5207 | LR: 5.62e-05 | 5708.4ms/step | 10.5 samples/s +Step 39650/50462 | Loss: 2.4949 | LR: 5.61e-05 | 5709.1ms/step | 10.5 samples/s +Step 39660/50462 | Loss: 2.4961 | LR: 5.60e-05 | 5706.6ms/step | 10.5 samples/s +Step 39670/50462 | Loss: 2.4946 | LR: 5.59e-05 | 5707.1ms/step | 10.5 samples/s +Step 39680/50462 | Loss: 2.5026 | LR: 5.58e-05 | 5706.9ms/step | 10.5 samples/s +Step 39690/50462 | Loss: 2.4895 | LR: 5.57e-05 | 5702.8ms/step | 10.5 samples/s +Step 39700/50462 | Loss: 2.4850 | LR: 5.56e-05 | 5704.8ms/step | 10.5 samples/s +Step 39710/50462 | Loss: 2.4761 | LR: 5.55e-05 | 5704.5ms/step | 10.5 samples/s +Step 39720/50462 | Loss: 2.4657 | LR: 5.54e-05 | 5707.3ms/step | 10.5 samples/s +Step 39730/50462 | Loss: 2.4698 | LR: 5.53e-05 | 5699.0ms/step | 10.5 samples/s +Step 39740/50462 | Loss: 2.4902 | LR: 5.52e-05 | 5702.3ms/step | 10.5 samples/s +Step 39750/50462 | Loss: 2.5007 | LR: 5.51e-05 | 5702.7ms/step | 10.5 samples/s +Validation at step 39750: Loss = 2.5039, Perplexity = 12.23 +Step 39760/50462 | Loss: 2.4875 | LR: 5.50e-05 | 8545.9ms/step | 7.0 samples/s +Step 39770/50462 | Loss: 2.5117 | LR: 5.49e-05 | 5697.4ms/step | 10.5 samples/s +Step 39780/50462 | Loss: 2.5097 | LR: 5.48e-05 | 5708.5ms/step | 10.5 samples/s +Step 39790/50462 | Loss: 2.4873 | LR: 5.47e-05 | 5703.1ms/step | 10.5 samples/s +Step 39800/50462 | Loss: 2.4822 | LR: 5.46e-05 | 5693.6ms/step | 10.5 samples/s +Step 39810/50462 | Loss: 2.4949 | LR: 5.45e-05 | 5690.7ms/step | 10.5 samples/s +Step 39820/50462 | Loss: 2.5155 | LR: 5.44e-05 | 5688.5ms/step | 10.5 samples/s +Step 39830/50462 | Loss: 2.4883 | LR: 5.43e-05 | 5688.8ms/step | 10.5 samples/s +Step 39840/50462 | Loss: 2.4985 | LR: 5.42e-05 | 5666.3ms/step | 10.6 samples/s +Step 39850/50462 | Loss: 2.5038 | LR: 5.41e-05 | 5628.1ms/step | 10.7 samples/s +Step 39860/50462 | Loss: 2.4665 | LR: 5.40e-05 | 5120.1ms/step | 11.7 samples/s +Step 39870/50462 | Loss: 2.4924 | LR: 5.39e-05 | 5542.5ms/step | 10.8 samples/s +Step 39880/50462 | Loss: 2.4932 | LR: 5.38e-05 | 5645.6ms/step | 10.6 samples/s +Step 39890/50462 | Loss: 2.5041 | LR: 5.38e-05 | 5649.5ms/step | 10.6 samples/s +Step 39900/50462 | Loss: 2.4842 | LR: 5.37e-05 | 5643.2ms/step | 10.6 samples/s +Step 39910/50462 | Loss: 2.4898 | LR: 5.36e-05 | 5642.9ms/step | 10.6 samples/s +Step 39920/50462 | Loss: 2.4921 | LR: 5.35e-05 | 5647.1ms/step | 10.6 samples/s +Step 39930/50462 | Loss: 2.4953 | LR: 5.34e-05 | 5649.6ms/step | 10.6 samples/s +Step 39940/50462 | Loss: 2.4855 | LR: 5.33e-05 | 5642.5ms/step | 10.6 samples/s +Step 39950/50462 | Loss: 2.4770 | LR: 5.32e-05 | 5641.2ms/step | 10.6 samples/s +Step 39960/50462 | Loss: 2.4935 | LR: 5.31e-05 | 5644.9ms/step | 10.6 samples/s +Step 39970/50462 | Loss: 2.5166 | LR: 5.30e-05 | 5640.4ms/step | 10.6 samples/s +Step 39980/50462 | Loss: 2.4933 | LR: 5.29e-05 | 5649.0ms/step | 10.6 samples/s +Step 39990/50462 | Loss: 2.5273 | LR: 5.28e-05 | 5640.3ms/step | 10.6 samples/s +Step 40000/50462 | Loss: 2.4656 | LR: 5.27e-05 | 5641.4ms/step | 10.6 samples/s +Validation at step 40000: Loss = 2.5037, Perplexity = 12.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-40000 +Step 40010/50462 | Loss: 2.5094 | LR: 5.26e-05 | 9208.2ms/step | 6.5 samples/s +Step 40020/50462 | Loss: 2.5301 | LR: 5.25e-05 | 5639.9ms/step | 10.6 samples/s +Step 40030/50462 | Loss: 2.4757 | LR: 5.24e-05 | 5639.3ms/step | 10.6 samples/s +Step 40040/50462 | Loss: 2.5085 | LR: 5.23e-05 | 5638.4ms/step | 10.6 samples/s +Step 40050/50462 | Loss: 2.4954 | LR: 5.22e-05 | 5642.6ms/step | 10.6 samples/s +Step 40060/50462 | Loss: 2.4900 | LR: 5.21e-05 | 5469.3ms/step | 11.0 samples/s +Step 40070/50462 | Loss: 2.4995 | LR: 5.20e-05 | 5122.4ms/step | 11.7 samples/s +Step 40080/50462 | Loss: 2.4734 | LR: 5.19e-05 | 5181.7ms/step | 11.6 samples/s +Step 40090/50462 | Loss: 2.5082 | LR: 5.18e-05 | 5687.9ms/step | 10.5 samples/s +Step 40100/50462 | Loss: 2.5029 | LR: 5.17e-05 | 5701.1ms/step | 10.5 samples/s +Step 40110/50462 | Loss: 2.4924 | LR: 5.16e-05 | 5699.7ms/step | 10.5 samples/s +Step 40120/50462 | Loss: 2.5057 | LR: 5.15e-05 | 5699.6ms/step | 10.5 samples/s +Step 40130/50462 | Loss: 2.4868 | LR: 5.14e-05 | 5706.6ms/step | 10.5 samples/s +Step 40140/50462 | Loss: 2.4847 | LR: 5.13e-05 | 5708.8ms/step | 10.5 samples/s +Step 40150/50462 | Loss: 2.4625 | LR: 5.12e-05 | 5713.4ms/step | 10.5 samples/s +Step 40160/50462 | Loss: 2.4607 | LR: 5.11e-05 | 5716.5ms/step | 10.5 samples/s +Step 40170/50462 | Loss: 2.4986 | LR: 5.10e-05 | 5713.0ms/step | 10.5 samples/s +Step 40180/50462 | Loss: 2.4989 | LR: 5.09e-05 | 5707.4ms/step | 10.5 samples/s +Step 40190/50462 | Loss: 2.4955 | LR: 5.09e-05 | 5706.9ms/step | 10.5 samples/s +Step 40200/50462 | Loss: 2.5050 | LR: 5.08e-05 | 5707.8ms/step | 10.5 samples/s +Step 40210/50462 | Loss: 2.5102 | LR: 5.07e-05 | 5700.1ms/step | 10.5 samples/s +Step 40220/50462 | Loss: 2.5111 | LR: 5.06e-05 | 5702.9ms/step | 10.5 samples/s +Step 40230/50462 | Loss: 2.5021 | LR: 5.05e-05 | 5699.0ms/step | 10.5 samples/s +Step 40240/50462 | Loss: 2.5316 | LR: 5.04e-05 | 5697.8ms/step | 10.5 samples/s +Step 40250/50462 | Loss: 2.4931 | LR: 5.03e-05 | 5696.2ms/step | 10.5 samples/s +Validation at step 40250: Loss = 2.5034, Perplexity = 12.22 +Step 40260/50462 | Loss: 2.4982 | LR: 5.02e-05 | 8515.2ms/step | 7.0 samples/s +Step 40270/50462 | Loss: 2.4741 | LR: 5.01e-05 | 5682.3ms/step | 10.6 samples/s +Step 40280/50462 | Loss: 2.4979 | LR: 5.00e-05 | 5669.5ms/step | 10.6 samples/s +Step 40290/50462 | Loss: 2.4958 | LR: 4.99e-05 | 5224.7ms/step | 11.5 samples/s +Step 40300/50462 | Loss: 2.4922 | LR: 4.98e-05 | 5623.9ms/step | 10.7 samples/s +Step 40310/50462 | Loss: 2.4766 | LR: 4.97e-05 | 5647.4ms/step | 10.6 samples/s +Step 40320/50462 | Loss: 2.4948 | LR: 4.96e-05 | 5644.2ms/step | 10.6 samples/s +Step 40330/50462 | Loss: 2.5181 | LR: 4.95e-05 | 5642.5ms/step | 10.6 samples/s +Step 40340/50462 | Loss: 2.4784 | LR: 4.94e-05 | 5644.3ms/step | 10.6 samples/s +Step 40350/50462 | Loss: 2.4688 | LR: 4.93e-05 | 5643.3ms/step | 10.6 samples/s +Step 40360/50462 | Loss: 2.4976 | LR: 4.92e-05 | 5645.8ms/step | 10.6 samples/s +Step 40370/50462 | Loss: 2.5105 | LR: 4.91e-05 | 5641.6ms/step | 10.6 samples/s +Step 40380/50462 | Loss: 2.4969 | LR: 4.90e-05 | 5642.6ms/step | 10.6 samples/s +Step 40390/50462 | Loss: 2.4928 | LR: 4.90e-05 | 5646.7ms/step | 10.6 samples/s +Step 40400/50462 | Loss: 2.5036 | LR: 4.89e-05 | 5495.0ms/step | 10.9 samples/s +Step 40410/50462 | Loss: 2.5184 | LR: 4.88e-05 | 5122.7ms/step | 11.7 samples/s +Step 40420/50462 | Loss: 2.5164 | LR: 4.87e-05 | 5613.7ms/step | 10.7 samples/s +Step 40430/50462 | Loss: 2.4839 | LR: 4.86e-05 | 5705.3ms/step | 10.5 samples/s +Step 40440/50462 | Loss: 2.5065 | LR: 4.85e-05 | 5706.4ms/step | 10.5 samples/s +Step 40450/50462 | Loss: 2.4925 | LR: 4.84e-05 | 5708.5ms/step | 10.5 samples/s +Step 40460/50462 | Loss: 2.5248 | LR: 4.83e-05 | 5712.1ms/step | 10.5 samples/s +Step 40470/50462 | Loss: 2.4792 | LR: 4.82e-05 | 5709.9ms/step | 10.5 samples/s +Step 40480/50462 | Loss: 2.4904 | LR: 4.81e-05 | 5710.7ms/step | 10.5 samples/s +Step 40490/50462 | Loss: 2.5066 | LR: 4.80e-05 | 5701.4ms/step | 10.5 samples/s +Step 40500/50462 | Loss: 2.4964 | LR: 4.79e-05 | 5706.2ms/step | 10.5 samples/s +Validation at step 40500: Loss = 2.5033, Perplexity = 12.22 +Step 40510/50462 | Loss: 2.4951 | LR: 4.78e-05 | 8536.8ms/step | 7.0 samples/s +Step 40520/50462 | Loss: 2.4813 | LR: 4.77e-05 | 5609.7ms/step | 10.7 samples/s +Step 40530/50462 | Loss: 2.4841 | LR: 4.76e-05 | 5170.3ms/step | 11.6 samples/s +Step 40540/50462 | Loss: 2.4575 | LR: 4.76e-05 | 5615.0ms/step | 10.7 samples/s +Step 40550/50462 | Loss: 2.5134 | LR: 4.75e-05 | 5643.5ms/step | 10.6 samples/s +Step 40560/50462 | Loss: 2.5043 | LR: 4.74e-05 | 5643.6ms/step | 10.6 samples/s +Step 40570/50462 | Loss: 2.4786 | LR: 4.73e-05 | 5640.5ms/step | 10.6 samples/s +Step 40580/50462 | Loss: 2.5061 | LR: 4.72e-05 | 5643.9ms/step | 10.6 samples/s +Step 40590/50462 | Loss: 2.4909 | LR: 4.71e-05 | 5640.4ms/step | 10.6 samples/s +Step 40600/50462 | Loss: 2.4921 | LR: 4.70e-05 | 5642.5ms/step | 10.6 samples/s +Step 40610/50462 | Loss: 2.5107 | LR: 4.69e-05 | 6075.0ms/step | 9.9 samples/s +Step 40620/50462 | Loss: 2.4666 | LR: 4.68e-05 | 5608.9ms/step | 10.7 samples/s +Step 40630/50462 | Loss: 2.4887 | LR: 4.67e-05 | 5114.1ms/step | 11.7 samples/s +Step 40640/50462 | Loss: 2.5010 | LR: 4.66e-05 | 5120.2ms/step | 11.7 samples/s +Step 40650/50462 | Loss: 2.4986 | LR: 4.65e-05 | 5233.4ms/step | 11.5 samples/s +Step 40660/50462 | Loss: 2.5099 | LR: 4.64e-05 | 5601.2ms/step | 10.7 samples/s +Step 40670/50462 | Loss: 2.4875 | LR: 4.64e-05 | 5613.0ms/step | 10.7 samples/s +Step 40680/50462 | Loss: 2.4922 | LR: 4.63e-05 | 5623.5ms/step | 10.7 samples/s +Step 40690/50462 | Loss: 2.4885 | LR: 4.62e-05 | 5617.2ms/step | 10.7 samples/s +Step 40700/50462 | Loss: 2.4603 | LR: 4.61e-05 | 5620.1ms/step | 10.7 samples/s +Step 40710/50462 | Loss: 2.5002 | LR: 4.60e-05 | 5617.7ms/step | 10.7 samples/s +Step 40720/50462 | Loss: 2.4676 | LR: 4.59e-05 | 5613.8ms/step | 10.7 samples/s +Step 40730/50462 | Loss: 2.4788 | LR: 4.58e-05 | 5615.2ms/step | 10.7 samples/s +Step 40740/50462 | Loss: 2.4745 | LR: 4.57e-05 | 5613.1ms/step | 10.7 samples/s +Step 40750/50462 | Loss: 2.4848 | LR: 4.56e-05 | 5610.3ms/step | 10.7 samples/s +Validation at step 40750: Loss = 2.5030, Perplexity = 12.22 +Step 40760/50462 | Loss: 2.5099 | LR: 4.55e-05 | 8389.8ms/step | 7.2 samples/s +Step 40770/50462 | Loss: 2.4904 | LR: 4.54e-05 | 5613.9ms/step | 10.7 samples/s +Step 40780/50462 | Loss: 2.5065 | LR: 4.54e-05 | 5612.1ms/step | 10.7 samples/s +Step 40790/50462 | Loss: 2.4914 | LR: 4.53e-05 | 5607.8ms/step | 10.7 samples/s +Step 40800/50462 | Loss: 2.4998 | LR: 4.52e-05 | 5607.8ms/step | 10.7 samples/s +Step 40810/50462 | Loss: 2.4577 | LR: 4.51e-05 | 5606.3ms/step | 10.7 samples/s +Step 40820/50462 | Loss: 2.5316 | LR: 4.50e-05 | 5607.2ms/step | 10.7 samples/s +Step 40830/50462 | Loss: 2.4448 | LR: 4.49e-05 | 5608.9ms/step | 10.7 samples/s +Step 40840/50462 | Loss: 2.4691 | LR: 4.48e-05 | 5609.0ms/step | 10.7 samples/s +Step 40850/50462 | Loss: 2.5123 | LR: 4.47e-05 | 5606.8ms/step | 10.7 samples/s +Step 40860/50462 | Loss: 2.4595 | LR: 4.46e-05 | 5609.9ms/step | 10.7 samples/s +Step 40870/50462 | Loss: 2.5131 | LR: 4.45e-05 | 5604.2ms/step | 10.7 samples/s +Step 40880/50462 | Loss: 2.5200 | LR: 4.45e-05 | 5609.7ms/step | 10.7 samples/s +Step 40890/50462 | Loss: 2.4547 | LR: 4.44e-05 | 5607.7ms/step | 10.7 samples/s +Step 40900/50462 | Loss: 2.4792 | LR: 4.43e-05 | 5609.1ms/step | 10.7 samples/s +Step 40910/50462 | Loss: 2.4981 | LR: 4.42e-05 | 5607.9ms/step | 10.7 samples/s +Step 40920/50462 | Loss: 2.5088 | LR: 4.41e-05 | 5607.2ms/step | 10.7 samples/s +Step 40930/50462 | Loss: 2.4992 | LR: 4.40e-05 | 5604.2ms/step | 10.7 samples/s +Step 40940/50462 | Loss: 2.5163 | LR: 4.39e-05 | 5607.0ms/step | 10.7 samples/s +Step 40950/50462 | Loss: 2.4700 | LR: 4.38e-05 | 5601.0ms/step | 10.7 samples/s +Step 40960/50462 | Loss: 2.5006 | LR: 4.37e-05 | 5601.6ms/step | 10.7 samples/s +Step 40970/50462 | Loss: 2.4895 | LR: 4.36e-05 | 5607.9ms/step | 10.7 samples/s +Step 40980/50462 | Loss: 2.4907 | LR: 4.36e-05 | 5603.1ms/step | 10.7 samples/s +Step 40990/50462 | Loss: 2.4594 | LR: 4.35e-05 | 5600.9ms/step | 10.7 samples/s +Step 41000/50462 | Loss: 2.5042 | LR: 4.34e-05 | 5599.3ms/step | 10.7 samples/s +Validation at step 41000: Loss = 2.5029, Perplexity = 12.22 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-41000 +Step 41010/50462 | Loss: 2.5108 | LR: 4.33e-05 | 9037.6ms/step | 6.6 samples/s +Step 41020/50462 | Loss: 2.5051 | LR: 4.32e-05 | 5171.8ms/step | 11.6 samples/s +Step 41030/50462 | Loss: 2.5085 | LR: 4.31e-05 | 5165.6ms/step | 11.6 samples/s +Step 41040/50462 | Loss: 2.4721 | LR: 4.30e-05 | 5143.9ms/step | 11.7 samples/s +Step 41050/50462 | Loss: 2.5025 | LR: 4.29e-05 | 5158.2ms/step | 11.6 samples/s +Step 41060/50462 | Loss: 2.5078 | LR: 4.28e-05 | 5170.2ms/step | 11.6 samples/s +Step 41070/50462 | Loss: 2.4860 | LR: 4.28e-05 | 5411.9ms/step | 11.1 samples/s +Step 41080/50462 | Loss: 2.4785 | LR: 4.27e-05 | 5759.9ms/step | 10.4 samples/s +Step 41090/50462 | Loss: 2.4566 | LR: 4.26e-05 | 5740.9ms/step | 10.5 samples/s +Step 41100/50462 | Loss: 2.4925 | LR: 4.25e-05 | 5765.3ms/step | 10.4 samples/s +Step 41110/50462 | Loss: 2.4965 | LR: 4.24e-05 | 6229.3ms/step | 9.6 samples/s +Step 41120/50462 | Loss: 2.4853 | LR: 4.23e-05 | 6266.6ms/step | 9.6 samples/s +Step 41130/50462 | Loss: 2.5103 | LR: 4.22e-05 | 5713.9ms/step | 10.5 samples/s +Step 41140/50462 | Loss: 2.4830 | LR: 4.21e-05 | 5667.0ms/step | 10.6 samples/s +Step 41150/50462 | Loss: 2.4970 | LR: 4.21e-05 | 5686.2ms/step | 10.6 samples/s +Step 41160/50462 | Loss: 2.4939 | LR: 4.20e-05 | 5664.7ms/step | 10.6 samples/s +Step 41170/50462 | Loss: 2.4988 | LR: 4.19e-05 | 5666.0ms/step | 10.6 samples/s +Step 41180/50462 | Loss: 2.4841 | LR: 4.18e-05 | 5670.2ms/step | 10.6 samples/s +Step 41190/50462 | Loss: 2.4436 | LR: 4.17e-05 | 5685.8ms/step | 10.6 samples/s +Step 41200/50462 | Loss: 2.4798 | LR: 4.16e-05 | 5697.3ms/step | 10.5 samples/s +Step 41210/50462 | Loss: 2.4570 | LR: 4.15e-05 | 5671.6ms/step | 10.6 samples/s +Step 41220/50462 | Loss: 2.4738 | LR: 4.14e-05 | 5672.0ms/step | 10.6 samples/s +Step 41230/50462 | Loss: 2.4950 | LR: 4.14e-05 | 5672.1ms/step | 10.6 samples/s +Step 41240/50462 | Loss: 2.4852 | LR: 4.13e-05 | 5674.5ms/step | 10.6 samples/s +Step 41250/50462 | Loss: 2.4999 | LR: 4.12e-05 | 5669.8ms/step | 10.6 samples/s +Validation at step 41250: Loss = 2.5027, Perplexity = 12.22 +Step 41260/50462 | Loss: 2.4963 | LR: 4.11e-05 | 8483.5ms/step | 7.1 samples/s +Step 41270/50462 | Loss: 2.4705 | LR: 4.10e-05 | 5668.6ms/step | 10.6 samples/s +Step 41280/50462 | Loss: 2.4475 | LR: 4.09e-05 | 5657.5ms/step | 10.6 samples/s +Step 41290/50462 | Loss: 2.5105 | LR: 4.08e-05 | 5661.2ms/step | 10.6 samples/s +Step 41300/50462 | Loss: 2.4995 | LR: 4.07e-05 | 5662.6ms/step | 10.6 samples/s +Step 41310/50462 | Loss: 2.4608 | LR: 4.07e-05 | 5662.0ms/step | 10.6 samples/s +Step 41320/50462 | Loss: 2.5441 | LR: 4.06e-05 | 5659.5ms/step | 10.6 samples/s +Step 41330/50462 | Loss: 2.4810 | LR: 4.05e-05 | 5659.1ms/step | 10.6 samples/s +Step 41340/50462 | Loss: 2.4735 | LR: 4.04e-05 | 5657.7ms/step | 10.6 samples/s +Step 41350/50462 | Loss: 2.4935 | LR: 4.03e-05 | 5652.7ms/step | 10.6 samples/s +Step 41360/50462 | Loss: 2.5204 | LR: 4.02e-05 | 5654.2ms/step | 10.6 samples/s +Step 41370/50462 | Loss: 2.4923 | LR: 4.01e-05 | 5656.2ms/step | 10.6 samples/s +Step 41380/50462 | Loss: 2.4837 | LR: 4.01e-05 | 5652.5ms/step | 10.6 samples/s +Step 41390/50462 | Loss: 2.4611 | LR: 4.00e-05 | 5656.1ms/step | 10.6 samples/s +Step 41400/50462 | Loss: 2.5001 | LR: 3.99e-05 | 5652.4ms/step | 10.6 samples/s +Step 41410/50462 | Loss: 2.5073 | LR: 3.98e-05 | 5647.9ms/step | 10.6 samples/s +Step 41420/50462 | Loss: 2.4713 | LR: 3.97e-05 | 5646.7ms/step | 10.6 samples/s +Step 41430/50462 | Loss: 2.5022 | LR: 3.96e-05 | 5645.9ms/step | 10.6 samples/s +Step 41440/50462 | Loss: 2.4693 | LR: 3.95e-05 | 5647.8ms/step | 10.6 samples/s +Step 41450/50462 | Loss: 2.4908 | LR: 3.95e-05 | 5645.1ms/step | 10.6 samples/s +Step 41460/50462 | Loss: 2.4718 | LR: 3.94e-05 | 5429.3ms/step | 11.1 samples/s +Step 41470/50462 | Loss: 2.4523 | LR: 3.93e-05 | 5254.9ms/step | 11.4 samples/s +Step 41480/50462 | Loss: 2.5105 | LR: 3.92e-05 | 5670.1ms/step | 10.6 samples/s +Step 41490/50462 | Loss: 2.4644 | LR: 3.91e-05 | 5671.5ms/step | 10.6 samples/s +Step 41500/50462 | Loss: 2.5047 | LR: 3.90e-05 | 5676.3ms/step | 10.6 samples/s +Validation at step 41500: Loss = 2.5026, Perplexity = 12.21 +Step 41510/50462 | Loss: 2.4757 | LR: 3.90e-05 | 8501.6ms/step | 7.1 samples/s +Step 41520/50462 | Loss: 2.4907 | LR: 3.89e-05 | 5671.9ms/step | 10.6 samples/s +Step 41530/50462 | Loss: 2.5017 | LR: 3.88e-05 | 5668.8ms/step | 10.6 samples/s +Step 41540/50462 | Loss: 2.4710 | LR: 3.87e-05 | 5677.3ms/step | 10.6 samples/s +Step 41550/50462 | Loss: 2.4978 | LR: 3.86e-05 | 5673.3ms/step | 10.6 samples/s +Step 41560/50462 | Loss: 2.5081 | LR: 3.85e-05 | 5673.5ms/step | 10.6 samples/s +Step 41570/50462 | Loss: 2.4953 | LR: 3.84e-05 | 5672.9ms/step | 10.6 samples/s +Step 41580/50462 | Loss: 2.5016 | LR: 3.84e-05 | 5673.1ms/step | 10.6 samples/s +Step 41590/50462 | Loss: 2.4840 | LR: 3.83e-05 | 5673.3ms/step | 10.6 samples/s +Step 41600/50462 | Loss: 2.4977 | LR: 3.82e-05 | 5672.8ms/step | 10.6 samples/s +Step 41610/50462 | Loss: 2.4799 | LR: 3.81e-05 | 5675.4ms/step | 10.6 samples/s +Step 41620/50462 | Loss: 2.4858 | LR: 3.80e-05 | 5672.1ms/step | 10.6 samples/s +Step 41630/50462 | Loss: 2.4676 | LR: 3.79e-05 | 5673.3ms/step | 10.6 samples/s +Step 41640/50462 | Loss: 2.4838 | LR: 3.79e-05 | 5676.1ms/step | 10.6 samples/s +Step 41650/50462 | Loss: 2.4687 | LR: 3.78e-05 | 5677.1ms/step | 10.6 samples/s +Step 41660/50462 | Loss: 2.4986 | LR: 3.77e-05 | 5676.4ms/step | 10.6 samples/s +Step 41670/50462 | Loss: 2.4903 | LR: 3.76e-05 | 5676.8ms/step | 10.6 samples/s +Step 41680/50462 | Loss: 2.4769 | LR: 3.75e-05 | 5677.0ms/step | 10.6 samples/s +Step 41690/50462 | Loss: 2.4724 | LR: 3.74e-05 | 5676.1ms/step | 10.6 samples/s +Step 41700/50462 | Loss: 2.4840 | LR: 3.74e-05 | 5673.4ms/step | 10.6 samples/s +Step 41710/50462 | Loss: 2.4768 | LR: 3.73e-05 | 5670.9ms/step | 10.6 samples/s +Step 41720/50462 | Loss: 2.4728 | LR: 3.72e-05 | 5671.9ms/step | 10.6 samples/s +Step 41730/50462 | Loss: 2.4794 | LR: 3.71e-05 | 5672.0ms/step | 10.6 samples/s +Step 41740/50462 | Loss: 2.4724 | LR: 3.70e-05 | 5674.1ms/step | 10.6 samples/s +Step 41750/50462 | Loss: 2.4782 | LR: 3.69e-05 | 5669.2ms/step | 10.6 samples/s +Validation at step 41750: Loss = 2.5025, Perplexity = 12.21 +Step 41760/50462 | Loss: 2.4913 | LR: 3.69e-05 | 8496.6ms/step | 7.1 samples/s +Step 41770/50462 | Loss: 2.4978 | LR: 3.68e-05 | 5679.1ms/step | 10.6 samples/s +Step 41780/50462 | Loss: 2.4845 | LR: 3.67e-05 | 5667.6ms/step | 10.6 samples/s +Step 41790/50462 | Loss: 2.4703 | LR: 3.66e-05 | 5673.0ms/step | 10.6 samples/s +Step 41800/50462 | Loss: 2.4743 | LR: 3.65e-05 | 5674.3ms/step | 10.6 samples/s +Step 41810/50462 | Loss: 2.5094 | LR: 3.64e-05 | 5676.5ms/step | 10.6 samples/s +Step 41820/50462 | Loss: 2.4351 | LR: 3.64e-05 | 5673.9ms/step | 10.6 samples/s +Step 41830/50462 | Loss: 2.4828 | LR: 3.63e-05 | 5675.2ms/step | 10.6 samples/s +Step 41840/50462 | Loss: 2.4565 | LR: 3.62e-05 | 5666.8ms/step | 10.6 samples/s +Step 41850/50462 | Loss: 2.4995 | LR: 3.61e-05 | 5670.1ms/step | 10.6 samples/s +Step 41860/50462 | Loss: 2.4841 | LR: 3.60e-05 | 5672.5ms/step | 10.6 samples/s +Step 41870/50462 | Loss: 2.4832 | LR: 3.60e-05 | 5668.9ms/step | 10.6 samples/s +Step 41880/50462 | Loss: 2.4943 | LR: 3.59e-05 | 5671.8ms/step | 10.6 samples/s +Step 41890/50462 | Loss: 2.4861 | LR: 3.58e-05 | 5672.4ms/step | 10.6 samples/s +Step 41900/50462 | Loss: 2.5109 | LR: 3.57e-05 | 5672.0ms/step | 10.6 samples/s +Step 41910/50462 | Loss: 2.5026 | LR: 3.56e-05 | 5673.6ms/step | 10.6 samples/s +Step 41920/50462 | Loss: 2.4933 | LR: 3.55e-05 | 5672.2ms/step | 10.6 samples/s +Step 41930/50462 | Loss: 2.4728 | LR: 3.55e-05 | 5676.9ms/step | 10.6 samples/s +Step 41940/50462 | Loss: 2.5004 | LR: 3.54e-05 | 5675.2ms/step | 10.6 samples/s +Step 41950/50462 | Loss: 2.4724 | LR: 3.53e-05 | 5673.0ms/step | 10.6 samples/s +Step 41960/50462 | Loss: 2.4997 | LR: 3.52e-05 | 5671.6ms/step | 10.6 samples/s +Step 41970/50462 | Loss: 2.4872 | LR: 3.51e-05 | 5669.3ms/step | 10.6 samples/s +Step 41980/50462 | Loss: 2.4854 | LR: 3.51e-05 | 5667.5ms/step | 10.6 samples/s +Step 41990/50462 | Loss: 2.5174 | LR: 3.50e-05 | 5669.3ms/step | 10.6 samples/s +Step 42000/50462 | Loss: 2.4957 | LR: 3.49e-05 | 5673.7ms/step | 10.6 samples/s +Validation at step 42000: Loss = 2.5024, Perplexity = 12.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-42000 +Step 42010/50462 | Loss: 2.4647 | LR: 3.48e-05 | 9374.5ms/step | 6.4 samples/s +Step 42020/50462 | Loss: 2.5057 | LR: 3.47e-05 | 5668.4ms/step | 10.6 samples/s +Step 42030/50462 | Loss: 2.5248 | LR: 3.47e-05 | 5671.2ms/step | 10.6 samples/s +Step 42040/50462 | Loss: 2.4904 | LR: 3.46e-05 | 5671.8ms/step | 10.6 samples/s +Step 42050/50462 | Loss: 2.4926 | LR: 3.45e-05 | 5675.3ms/step | 10.6 samples/s +Step 42060/50462 | Loss: 2.5255 | LR: 3.44e-05 | 5669.8ms/step | 10.6 samples/s +Step 42070/50462 | Loss: 2.5066 | LR: 3.43e-05 | 5671.0ms/step | 10.6 samples/s +Step 42080/50462 | Loss: 2.4779 | LR: 3.43e-05 | 5673.3ms/step | 10.6 samples/s +Step 42090/50462 | Loss: 2.5093 | LR: 3.42e-05 | 5670.9ms/step | 10.6 samples/s +Step 42100/50462 | Loss: 2.4894 | LR: 3.41e-05 | 5672.8ms/step | 10.6 samples/s +Step 42110/50462 | Loss: 2.4575 | LR: 3.40e-05 | 5672.9ms/step | 10.6 samples/s +Step 42120/50462 | Loss: 2.4926 | LR: 3.39e-05 | 5670.1ms/step | 10.6 samples/s +Step 42130/50462 | Loss: 2.5057 | LR: 3.39e-05 | 5670.6ms/step | 10.6 samples/s +Step 42140/50462 | Loss: 2.5012 | LR: 3.38e-05 | 5672.7ms/step | 10.6 samples/s +Step 42150/50462 | Loss: 2.4973 | LR: 3.37e-05 | 5669.2ms/step | 10.6 samples/s +Step 42160/50462 | Loss: 2.4568 | LR: 3.36e-05 | 5668.0ms/step | 10.6 samples/s +Step 42170/50462 | Loss: 2.4825 | LR: 3.35e-05 | 5671.4ms/step | 10.6 samples/s +Step 42180/50462 | Loss: 2.4889 | LR: 3.35e-05 | 5670.7ms/step | 10.6 samples/s +Step 42190/50462 | Loss: 2.5192 | LR: 3.34e-05 | 5666.7ms/step | 10.6 samples/s +Step 42200/50462 | Loss: 2.4744 | LR: 3.33e-05 | 5665.7ms/step | 10.6 samples/s +Step 42210/50462 | Loss: 2.4798 | LR: 3.32e-05 | 5666.3ms/step | 10.6 samples/s +Step 42220/50462 | Loss: 2.5088 | LR: 3.32e-05 | 5668.8ms/step | 10.6 samples/s +Step 42230/50462 | Loss: 2.4947 | LR: 3.31e-05 | 5670.3ms/step | 10.6 samples/s +Step 42240/50462 | Loss: 2.4502 | LR: 3.30e-05 | 5672.1ms/step | 10.6 samples/s +Step 42250/50462 | Loss: 2.4998 | LR: 3.29e-05 | 5669.9ms/step | 10.6 samples/s +Validation at step 42250: Loss = 2.5022, Perplexity = 12.21 +Step 42260/50462 | Loss: 2.5069 | LR: 3.28e-05 | 8499.9ms/step | 7.1 samples/s +Step 42270/50462 | Loss: 2.5179 | LR: 3.28e-05 | 5673.4ms/step | 10.6 samples/s +Step 42280/50462 | Loss: 2.4876 | LR: 3.27e-05 | 5667.8ms/step | 10.6 samples/s +Step 42290/50462 | Loss: 2.5011 | LR: 3.26e-05 | 5668.7ms/step | 10.6 samples/s +Step 42300/50462 | Loss: 2.4880 | LR: 3.25e-05 | 5669.8ms/step | 10.6 samples/s +Step 42310/50462 | Loss: 2.4805 | LR: 3.24e-05 | 5668.1ms/step | 10.6 samples/s +Step 42320/50462 | Loss: 2.4619 | LR: 3.24e-05 | 5667.3ms/step | 10.6 samples/s +Step 42330/50462 | Loss: 2.5105 | LR: 3.23e-05 | 5666.3ms/step | 10.6 samples/s +Step 42340/50462 | Loss: 2.5179 | LR: 3.22e-05 | 5666.8ms/step | 10.6 samples/s +Step 42350/50462 | Loss: 2.4637 | LR: 3.21e-05 | 5667.1ms/step | 10.6 samples/s +Step 42360/50462 | Loss: 2.4747 | LR: 3.21e-05 | 5664.3ms/step | 10.6 samples/s +Step 42370/50462 | Loss: 2.4890 | LR: 3.20e-05 | 5665.8ms/step | 10.6 samples/s +Step 42380/50462 | Loss: 2.4737 | LR: 3.19e-05 | 5668.2ms/step | 10.6 samples/s +Step 42390/50462 | Loss: 2.5018 | LR: 3.18e-05 | 5662.0ms/step | 10.6 samples/s +Step 42400/50462 | Loss: 2.5107 | LR: 3.18e-05 | 5665.8ms/step | 10.6 samples/s +Step 42410/50462 | Loss: 2.5021 | LR: 3.17e-05 | 5669.0ms/step | 10.6 samples/s +Step 42420/50462 | Loss: 2.4962 | LR: 3.16e-05 | 5667.0ms/step | 10.6 samples/s +Step 42430/50462 | Loss: 2.4837 | LR: 3.15e-05 | 5483.6ms/step | 10.9 samples/s +Step 42440/50462 | Loss: 2.4822 | LR: 3.14e-05 | 5178.8ms/step | 11.6 samples/s +Step 42450/50462 | Loss: 2.5018 | LR: 3.14e-05 | 5170.6ms/step | 11.6 samples/s +Step 42460/50462 | Loss: 2.4823 | LR: 3.13e-05 | 5159.8ms/step | 11.6 samples/s +Step 42470/50462 | Loss: 2.4934 | LR: 3.12e-05 | 5172.0ms/step | 11.6 samples/s +Step 42480/50462 | Loss: 2.4495 | LR: 3.11e-05 | 5663.6ms/step | 10.6 samples/s +Step 42490/50462 | Loss: 2.4965 | LR: 3.11e-05 | 5741.2ms/step | 10.5 samples/s +Step 42500/50462 | Loss: 2.5064 | LR: 3.10e-05 | 5744.5ms/step | 10.4 samples/s +Validation at step 42500: Loss = 2.5021, Perplexity = 12.21 +Step 42510/50462 | Loss: 2.5014 | LR: 3.09e-05 | 8602.1ms/step | 7.0 samples/s +Step 42520/50462 | Loss: 2.4883 | LR: 3.08e-05 | 5745.7ms/step | 10.4 samples/s +Step 42530/50462 | Loss: 2.4801 | LR: 3.08e-05 | 5749.4ms/step | 10.4 samples/s +Step 42540/50462 | Loss: 2.4863 | LR: 3.07e-05 | 5745.2ms/step | 10.4 samples/s +Step 42550/50462 | Loss: 2.5104 | LR: 3.06e-05 | 5749.9ms/step | 10.4 samples/s +Step 42560/50462 | Loss: 2.5684 | LR: 3.05e-05 | 5739.4ms/step | 10.5 samples/s +Step 42570/50462 | Loss: 2.4986 | LR: 3.05e-05 | 5748.6ms/step | 10.4 samples/s +Step 42580/50462 | Loss: 2.4926 | LR: 3.04e-05 | 5746.4ms/step | 10.4 samples/s +Step 42590/50462 | Loss: 2.4865 | LR: 3.03e-05 | 5743.3ms/step | 10.4 samples/s +Step 42600/50462 | Loss: 2.5347 | LR: 3.02e-05 | 5742.6ms/step | 10.4 samples/s +Step 42610/50462 | Loss: 2.4836 | LR: 3.02e-05 | 5748.4ms/step | 10.4 samples/s +Step 42620/50462 | Loss: 2.4755 | LR: 3.01e-05 | 5747.6ms/step | 10.4 samples/s +Step 42630/50462 | Loss: 2.5088 | LR: 3.00e-05 | 5747.0ms/step | 10.4 samples/s +Step 42640/50462 | Loss: 2.5337 | LR: 2.99e-05 | 5750.3ms/step | 10.4 samples/s +Step 42650/50462 | Loss: 2.5034 | LR: 2.99e-05 | 5744.4ms/step | 10.4 samples/s +Step 42660/50462 | Loss: 2.5050 | LR: 2.98e-05 | 5745.9ms/step | 10.4 samples/s +Step 42670/50462 | Loss: 2.4742 | LR: 2.97e-05 | 5747.3ms/step | 10.4 samples/s +Step 42680/50462 | Loss: 2.4744 | LR: 2.96e-05 | 6163.0ms/step | 9.7 samples/s +Step 42690/50462 | Loss: 2.4958 | LR: 2.96e-05 | 5734.0ms/step | 10.5 samples/s +Step 42700/50462 | Loss: 2.5024 | LR: 2.95e-05 | 5732.0ms/step | 10.5 samples/s +Step 42710/50462 | Loss: 2.4855 | LR: 2.94e-05 | 5734.9ms/step | 10.5 samples/s +Step 42720/50462 | Loss: 2.4861 | LR: 2.93e-05 | 5746.6ms/step | 10.4 samples/s +Step 42730/50462 | Loss: 2.4869 | LR: 2.93e-05 | 5745.3ms/step | 10.4 samples/s +Step 42740/50462 | Loss: 2.4917 | LR: 2.92e-05 | 5743.2ms/step | 10.4 samples/s +Step 42750/50462 | Loss: 2.4885 | LR: 2.91e-05 | 5742.7ms/step | 10.4 samples/s +Validation at step 42750: Loss = 2.5020, Perplexity = 12.21 +Step 42760/50462 | Loss: 2.5006 | LR: 2.90e-05 | 8602.3ms/step | 7.0 samples/s +Step 42770/50462 | Loss: 2.4918 | LR: 2.90e-05 | 5744.5ms/step | 10.4 samples/s +Step 42780/50462 | Loss: 2.4902 | LR: 2.89e-05 | 5747.0ms/step | 10.4 samples/s +Step 42790/50462 | Loss: 2.5071 | LR: 2.88e-05 | 5749.3ms/step | 10.4 samples/s +Step 42800/50462 | Loss: 2.5074 | LR: 2.87e-05 | 5744.9ms/step | 10.4 samples/s +Step 42810/50462 | Loss: 2.5117 | LR: 2.87e-05 | 5742.6ms/step | 10.4 samples/s +Step 42820/50462 | Loss: 2.4835 | LR: 2.86e-05 | 5742.3ms/step | 10.4 samples/s +Step 42830/50462 | Loss: 2.4770 | LR: 2.85e-05 | 5749.6ms/step | 10.4 samples/s +Step 42840/50462 | Loss: 2.5048 | LR: 2.84e-05 | 5750.9ms/step | 10.4 samples/s +Step 42850/50462 | Loss: 2.4930 | LR: 2.84e-05 | 5745.9ms/step | 10.4 samples/s +Step 42860/50462 | Loss: 2.5215 | LR: 2.83e-05 | 5746.1ms/step | 10.4 samples/s +Step 42870/50462 | Loss: 2.5032 | LR: 2.82e-05 | 5742.8ms/step | 10.4 samples/s +Step 42880/50462 | Loss: 2.4987 | LR: 2.82e-05 | 5746.7ms/step | 10.4 samples/s +Step 42890/50462 | Loss: 2.4829 | LR: 2.81e-05 | 5742.2ms/step | 10.4 samples/s +Step 42900/50462 | Loss: 2.4913 | LR: 2.80e-05 | 5747.4ms/step | 10.4 samples/s +Step 42910/50462 | Loss: 2.5222 | LR: 2.79e-05 | 5749.2ms/step | 10.4 samples/s +Step 42920/50462 | Loss: 2.4813 | LR: 2.79e-05 | 5749.4ms/step | 10.4 samples/s +Step 42930/50462 | Loss: 2.4800 | LR: 2.78e-05 | 5749.4ms/step | 10.4 samples/s +Step 42940/50462 | Loss: 2.4699 | LR: 2.77e-05 | 5747.3ms/step | 10.4 samples/s +Step 42950/50462 | Loss: 2.5196 | LR: 2.76e-05 | 5745.2ms/step | 10.4 samples/s +Step 42960/50462 | Loss: 2.5041 | LR: 2.76e-05 | 5746.4ms/step | 10.4 samples/s +Step 42970/50462 | Loss: 2.4746 | LR: 2.75e-05 | 5745.5ms/step | 10.4 samples/s +Step 42980/50462 | Loss: 2.4889 | LR: 2.74e-05 | 5748.9ms/step | 10.4 samples/s +Step 42990/50462 | Loss: 2.4966 | LR: 2.74e-05 | 5744.0ms/step | 10.4 samples/s +Step 43000/50462 | Loss: 2.4733 | LR: 2.73e-05 | 5750.1ms/step | 10.4 samples/s +Validation at step 43000: Loss = 2.5019, Perplexity = 12.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-43000 +Step 43010/50462 | Loss: 2.5149 | LR: 2.72e-05 | 9415.9ms/step | 6.4 samples/s +Step 43020/50462 | Loss: 2.4874 | LR: 2.71e-05 | 5742.3ms/step | 10.4 samples/s +Step 43030/50462 | Loss: 2.4937 | LR: 2.71e-05 | 5738.8ms/step | 10.5 samples/s +Step 43040/50462 | Loss: 2.4810 | LR: 2.70e-05 | 5741.4ms/step | 10.5 samples/s +Step 43050/50462 | Loss: 2.5021 | LR: 2.69e-05 | 5741.5ms/step | 10.5 samples/s +Step 43060/50462 | Loss: 2.4760 | LR: 2.69e-05 | 5740.6ms/step | 10.5 samples/s +Step 43070/50462 | Loss: 2.4936 | LR: 2.68e-05 | 5746.0ms/step | 10.4 samples/s +Step 43080/50462 | Loss: 2.4736 | LR: 2.67e-05 | 5740.9ms/step | 10.5 samples/s +Step 43090/50462 | Loss: 2.4686 | LR: 2.66e-05 | 5737.7ms/step | 10.5 samples/s +Step 43100/50462 | Loss: 2.4684 | LR: 2.66e-05 | 5738.4ms/step | 10.5 samples/s +Step 43110/50462 | Loss: 2.4756 | LR: 2.65e-05 | 5737.0ms/step | 10.5 samples/s +Step 43120/50462 | Loss: 2.4905 | LR: 2.64e-05 | 5735.6ms/step | 10.5 samples/s +Step 43130/50462 | Loss: 2.5014 | LR: 2.64e-05 | 5738.8ms/step | 10.5 samples/s +Step 43140/50462 | Loss: 2.4709 | LR: 2.63e-05 | 5735.5ms/step | 10.5 samples/s +Step 43150/50462 | Loss: 2.4736 | LR: 2.62e-05 | 5734.9ms/step | 10.5 samples/s +Step 43160/50462 | Loss: 2.5124 | LR: 2.61e-05 | 5735.0ms/step | 10.5 samples/s +Step 43170/50462 | Loss: 2.5039 | LR: 2.61e-05 | 5733.4ms/step | 10.5 samples/s +Step 43180/50462 | Loss: 2.4579 | LR: 2.60e-05 | 5735.0ms/step | 10.5 samples/s +Step 43190/50462 | Loss: 2.5136 | LR: 2.59e-05 | 5732.1ms/step | 10.5 samples/s +Step 43200/50462 | Loss: 2.4904 | LR: 2.59e-05 | 5738.7ms/step | 10.5 samples/s +Step 43210/50462 | Loss: 2.4967 | LR: 2.58e-05 | 5739.1ms/step | 10.5 samples/s +Step 43220/50462 | Loss: 2.5043 | LR: 2.57e-05 | 6228.2ms/step | 9.6 samples/s +Step 43230/50462 | Loss: 2.4896 | LR: 2.57e-05 | 6257.9ms/step | 9.6 samples/s +Step 43240/50462 | Loss: 2.5106 | LR: 2.56e-05 | 5726.7ms/step | 10.5 samples/s +Step 43250/50462 | Loss: 2.4604 | LR: 2.55e-05 | 5731.6ms/step | 10.5 samples/s +Validation at step 43250: Loss = 2.5019, Perplexity = 12.21 +Step 43260/50462 | Loss: 2.4632 | LR: 2.54e-05 | 8571.5ms/step | 7.0 samples/s +Step 43270/50462 | Loss: 2.4740 | LR: 2.54e-05 | 5733.7ms/step | 10.5 samples/s +Step 43280/50462 | Loss: 2.4950 | LR: 2.53e-05 | 5732.7ms/step | 10.5 samples/s +Step 43290/50462 | Loss: 2.4750 | LR: 2.52e-05 | 5733.8ms/step | 10.5 samples/s +Step 43300/50462 | Loss: 2.5183 | LR: 2.52e-05 | 5732.9ms/step | 10.5 samples/s +Step 43310/50462 | Loss: 2.4830 | LR: 2.51e-05 | 5733.4ms/step | 10.5 samples/s +Step 43320/50462 | Loss: 2.4871 | LR: 2.50e-05 | 5727.2ms/step | 10.5 samples/s +Step 43330/50462 | Loss: 2.4724 | LR: 2.50e-05 | 5733.6ms/step | 10.5 samples/s +Step 43340/50462 | Loss: 2.4841 | LR: 2.49e-05 | 5726.6ms/step | 10.5 samples/s +Step 43350/50462 | Loss: 2.4703 | LR: 2.48e-05 | 5729.6ms/step | 10.5 samples/s +Step 43360/50462 | Loss: 2.5038 | LR: 2.48e-05 | 5719.8ms/step | 10.5 samples/s +Step 43370/50462 | Loss: 2.4957 | LR: 2.47e-05 | 5700.2ms/step | 10.5 samples/s +Step 43380/50462 | Loss: 2.4737 | LR: 2.46e-05 | 5711.5ms/step | 10.5 samples/s +Step 43390/50462 | Loss: 2.5023 | LR: 2.46e-05 | 5717.2ms/step | 10.5 samples/s +Step 43400/50462 | Loss: 2.4852 | LR: 2.45e-05 | 5707.4ms/step | 10.5 samples/s +Step 43410/50462 | Loss: 2.5055 | LR: 2.44e-05 | 5714.5ms/step | 10.5 samples/s +Step 43420/50462 | Loss: 2.4857 | LR: 2.44e-05 | 5711.3ms/step | 10.5 samples/s +Step 43430/50462 | Loss: 2.4900 | LR: 2.43e-05 | 5717.4ms/step | 10.5 samples/s +Step 43440/50462 | Loss: 2.4870 | LR: 2.42e-05 | 5699.3ms/step | 10.5 samples/s +Step 43450/50462 | Loss: 2.5119 | LR: 2.41e-05 | 5679.3ms/step | 10.6 samples/s +Step 43460/50462 | Loss: 2.4980 | LR: 2.41e-05 | 5283.1ms/step | 11.4 samples/s +Step 43470/50462 | Loss: 2.5135 | LR: 2.40e-05 | 5474.5ms/step | 11.0 samples/s +Step 43480/50462 | Loss: 2.5126 | LR: 2.39e-05 | 5663.0ms/step | 10.6 samples/s +Step 43490/50462 | Loss: 2.4820 | LR: 2.39e-05 | 5664.2ms/step | 10.6 samples/s +Step 43500/50462 | Loss: 2.4999 | LR: 2.38e-05 | 5667.0ms/step | 10.6 samples/s +Validation at step 43500: Loss = 2.5018, Perplexity = 12.20 +Step 43510/50462 | Loss: 2.5011 | LR: 2.37e-05 | 8489.9ms/step | 7.1 samples/s +Step 43520/50462 | Loss: 2.4650 | LR: 2.37e-05 | 5669.1ms/step | 10.6 samples/s +Step 43530/50462 | Loss: 2.4862 | LR: 2.36e-05 | 5664.3ms/step | 10.6 samples/s +Step 43540/50462 | Loss: 2.4832 | LR: 2.35e-05 | 5663.0ms/step | 10.6 samples/s +Step 43550/50462 | Loss: 2.4919 | LR: 2.35e-05 | 5662.3ms/step | 10.6 samples/s +Step 43560/50462 | Loss: 2.4710 | LR: 2.34e-05 | 5664.1ms/step | 10.6 samples/s +Step 43570/50462 | Loss: 2.4917 | LR: 2.33e-05 | 5667.4ms/step | 10.6 samples/s +Step 43580/50462 | Loss: 2.5189 | LR: 2.33e-05 | 5669.7ms/step | 10.6 samples/s +Step 43590/50462 | Loss: 2.4760 | LR: 2.32e-05 | 5665.4ms/step | 10.6 samples/s +Step 43600/50462 | Loss: 2.5106 | LR: 2.31e-05 | 5669.5ms/step | 10.6 samples/s +Step 43610/50462 | Loss: 2.4975 | LR: 2.31e-05 | 5667.0ms/step | 10.6 samples/s +Step 43620/50462 | Loss: 2.4802 | LR: 2.30e-05 | 5663.7ms/step | 10.6 samples/s +Step 43630/50462 | Loss: 2.5180 | LR: 2.29e-05 | 5654.8ms/step | 10.6 samples/s +Step 43640/50462 | Loss: 2.4795 | LR: 2.29e-05 | 5656.4ms/step | 10.6 samples/s +Step 43650/50462 | Loss: 2.4998 | LR: 2.28e-05 | 5656.1ms/step | 10.6 samples/s +Step 43660/50462 | Loss: 2.5129 | LR: 2.27e-05 | 5649.3ms/step | 10.6 samples/s +Step 43670/50462 | Loss: 2.4992 | LR: 2.27e-05 | 5119.1ms/step | 11.7 samples/s +Step 43680/50462 | Loss: 2.5032 | LR: 2.26e-05 | 5126.3ms/step | 11.7 samples/s +Step 43690/50462 | Loss: 2.4960 | LR: 2.25e-05 | 5545.7ms/step | 10.8 samples/s +Step 43700/50462 | Loss: 2.5136 | LR: 2.25e-05 | 5703.6ms/step | 10.5 samples/s +Step 43710/50462 | Loss: 2.5187 | LR: 2.24e-05 | 5704.2ms/step | 10.5 samples/s +Step 43720/50462 | Loss: 2.4739 | LR: 2.24e-05 | 5703.2ms/step | 10.5 samples/s +Step 43730/50462 | Loss: 2.4641 | LR: 2.23e-05 | 5713.1ms/step | 10.5 samples/s +Step 43740/50462 | Loss: 2.4883 | LR: 2.22e-05 | 5718.7ms/step | 10.5 samples/s +Step 43750/50462 | Loss: 2.4887 | LR: 2.22e-05 | 5719.7ms/step | 10.5 samples/s +Validation at step 43750: Loss = 2.5018, Perplexity = 12.20 +Step 43760/50462 | Loss: 2.4941 | LR: 2.21e-05 | 8600.4ms/step | 7.0 samples/s +Step 43770/50462 | Loss: 2.4795 | LR: 2.20e-05 | 5736.9ms/step | 10.5 samples/s +Step 43780/50462 | Loss: 2.4872 | LR: 2.20e-05 | 5732.1ms/step | 10.5 samples/s +Step 43790/50462 | Loss: 2.4967 | LR: 2.19e-05 | 5733.5ms/step | 10.5 samples/s +Step 43800/50462 | Loss: 2.5079 | LR: 2.18e-05 | 5732.8ms/step | 10.5 samples/s +Step 43810/50462 | Loss: 2.4523 | LR: 2.18e-05 | 5724.8ms/step | 10.5 samples/s +Step 43820/50462 | Loss: 2.4559 | LR: 2.17e-05 | 5725.0ms/step | 10.5 samples/s +Step 43830/50462 | Loss: 2.5006 | LR: 2.16e-05 | 5725.3ms/step | 10.5 samples/s +Step 43840/50462 | Loss: 2.5153 | LR: 2.16e-05 | 5727.3ms/step | 10.5 samples/s +Step 43850/50462 | Loss: 2.5111 | LR: 2.15e-05 | 5720.4ms/step | 10.5 samples/s +Step 43860/50462 | Loss: 2.4934 | LR: 2.14e-05 | 5712.7ms/step | 10.5 samples/s +Step 43870/50462 | Loss: 2.4839 | LR: 2.14e-05 | 5707.8ms/step | 10.5 samples/s +Step 43880/50462 | Loss: 2.5033 | LR: 2.13e-05 | 5696.1ms/step | 10.5 samples/s +Step 43890/50462 | Loss: 2.4957 | LR: 2.13e-05 | 5457.9ms/step | 11.0 samples/s +Step 43900/50462 | Loss: 2.5469 | LR: 2.12e-05 | 5434.2ms/step | 11.0 samples/s +Step 43910/50462 | Loss: 2.5136 | LR: 2.11e-05 | 5664.9ms/step | 10.6 samples/s +Step 43920/50462 | Loss: 2.4893 | LR: 2.11e-05 | 5667.4ms/step | 10.6 samples/s +Step 43930/50462 | Loss: 2.5146 | LR: 2.10e-05 | 5663.0ms/step | 10.6 samples/s +Step 43940/50462 | Loss: 2.4897 | LR: 2.09e-05 | 5666.7ms/step | 10.6 samples/s +Step 43950/50462 | Loss: 2.5018 | LR: 2.09e-05 | 5667.7ms/step | 10.6 samples/s +Step 43960/50462 | Loss: 2.4854 | LR: 2.08e-05 | 5666.3ms/step | 10.6 samples/s +Step 43970/50462 | Loss: 2.4801 | LR: 2.07e-05 | 5664.7ms/step | 10.6 samples/s +Step 43980/50462 | Loss: 2.4710 | LR: 2.07e-05 | 5668.4ms/step | 10.6 samples/s +Step 43990/50462 | Loss: 2.5085 | LR: 2.06e-05 | 5663.7ms/step | 10.6 samples/s +Step 44000/50462 | Loss: 2.5138 | LR: 2.06e-05 | 5657.9ms/step | 10.6 samples/s +Validation at step 44000: Loss = 2.5017, Perplexity = 12.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-44000 +Step 44010/50462 | Loss: 2.4985 | LR: 2.05e-05 | 8846.6ms/step | 6.8 samples/s +Step 44020/50462 | Loss: 2.4919 | LR: 2.04e-05 | 5715.0ms/step | 10.5 samples/s +Step 44030/50462 | Loss: 2.4874 | LR: 2.04e-05 | 5727.1ms/step | 10.5 samples/s +Step 44040/50462 | Loss: 2.4821 | LR: 2.03e-05 | 5735.9ms/step | 10.5 samples/s +Step 44050/50462 | Loss: 2.5150 | LR: 2.02e-05 | 5732.7ms/step | 10.5 samples/s +Step 44060/50462 | Loss: 2.5143 | LR: 2.02e-05 | 5732.4ms/step | 10.5 samples/s +Step 44070/50462 | Loss: 2.5137 | LR: 2.01e-05 | 5732.9ms/step | 10.5 samples/s +Step 44080/50462 | Loss: 2.4953 | LR: 2.01e-05 | 5731.0ms/step | 10.5 samples/s +Step 44090/50462 | Loss: 2.4650 | LR: 2.00e-05 | 5724.7ms/step | 10.5 samples/s +Step 44100/50462 | Loss: 2.4838 | LR: 1.99e-05 | 5730.7ms/step | 10.5 samples/s +Step 44110/50462 | Loss: 2.4862 | LR: 1.99e-05 | 5728.4ms/step | 10.5 samples/s +Step 44120/50462 | Loss: 2.5020 | LR: 1.98e-05 | 5695.5ms/step | 10.5 samples/s +Step 44130/50462 | Loss: 2.5080 | LR: 1.97e-05 | 5311.7ms/step | 11.3 samples/s +Step 44140/50462 | Loss: 2.4767 | LR: 1.97e-05 | 5649.5ms/step | 10.6 samples/s +Step 44150/50462 | Loss: 2.4720 | LR: 1.96e-05 | 5665.2ms/step | 10.6 samples/s +Step 44160/50462 | Loss: 2.5019 | LR: 1.96e-05 | 5667.1ms/step | 10.6 samples/s +Step 44170/50462 | Loss: 2.5059 | LR: 1.95e-05 | 5666.2ms/step | 10.6 samples/s +Step 44180/50462 | Loss: 2.4725 | LR: 1.94e-05 | 5665.8ms/step | 10.6 samples/s +Step 44190/50462 | Loss: 2.4781 | LR: 1.94e-05 | 5666.4ms/step | 10.6 samples/s +Step 44200/50462 | Loss: 2.5002 | LR: 1.93e-05 | 5667.2ms/step | 10.6 samples/s +Step 44210/50462 | Loss: 2.4883 | LR: 1.93e-05 | 5667.2ms/step | 10.6 samples/s +Step 44220/50462 | Loss: 2.4945 | LR: 1.92e-05 | 5556.5ms/step | 10.8 samples/s +Step 44230/50462 | Loss: 2.5052 | LR: 1.91e-05 | 5164.6ms/step | 11.6 samples/s +Step 44240/50462 | Loss: 2.4932 | LR: 1.91e-05 | 5175.6ms/step | 11.6 samples/s +Step 44250/50462 | Loss: 2.5066 | LR: 1.90e-05 | 5169.8ms/step | 11.6 samples/s +Validation at step 44250: Loss = 2.5017, Perplexity = 12.20 +Step 44260/50462 | Loss: 2.5010 | LR: 1.90e-05 | 7732.8ms/step | 7.8 samples/s +Step 44270/50462 | Loss: 2.4715 | LR: 1.89e-05 | 5174.6ms/step | 11.6 samples/s +Step 44280/50462 | Loss: 2.4952 | LR: 1.88e-05 | 5164.6ms/step | 11.6 samples/s +Step 44290/50462 | Loss: 2.4911 | LR: 1.88e-05 | 5156.7ms/step | 11.6 samples/s +Step 44300/50462 | Loss: 2.4427 | LR: 1.87e-05 | 5152.8ms/step | 11.6 samples/s +Step 44310/50462 | Loss: 2.4726 | LR: 1.87e-05 | 5159.1ms/step | 11.6 samples/s +Step 44320/50462 | Loss: 2.5047 | LR: 1.86e-05 | 5158.8ms/step | 11.6 samples/s +Step 44330/50462 | Loss: 2.4955 | LR: 1.85e-05 | 5170.4ms/step | 11.6 samples/s +Step 44340/50462 | Loss: 2.4919 | LR: 1.85e-05 | 5151.0ms/step | 11.6 samples/s +Step 44350/50462 | Loss: 2.4965 | LR: 1.84e-05 | 5167.6ms/step | 11.6 samples/s +Step 44360/50462 | Loss: 2.4695 | LR: 1.84e-05 | 5170.7ms/step | 11.6 samples/s +Step 44370/50462 | Loss: 2.4952 | LR: 1.83e-05 | 5166.1ms/step | 11.6 samples/s +Step 44380/50462 | Loss: 2.4861 | LR: 1.82e-05 | 5169.5ms/step | 11.6 samples/s +Step 44390/50462 | Loss: 2.5061 | LR: 1.82e-05 | 5164.6ms/step | 11.6 samples/s +Step 44400/50462 | Loss: 2.4724 | LR: 1.81e-05 | 5173.1ms/step | 11.6 samples/s +Step 44410/50462 | Loss: 2.4819 | LR: 1.81e-05 | 5175.1ms/step | 11.6 samples/s +Step 44420/50462 | Loss: 2.4916 | LR: 1.80e-05 | 5177.4ms/step | 11.6 samples/s +Step 44430/50462 | Loss: 2.4989 | LR: 1.79e-05 | 5157.3ms/step | 11.6 samples/s +Step 44440/50462 | Loss: 2.4878 | LR: 1.79e-05 | 5168.3ms/step | 11.6 samples/s +Step 44450/50462 | Loss: 2.4766 | LR: 1.78e-05 | 5174.5ms/step | 11.6 samples/s +Step 44460/50462 | Loss: 2.4923 | LR: 1.78e-05 | 5168.5ms/step | 11.6 samples/s +Step 44470/50462 | Loss: 2.5051 | LR: 1.77e-05 | 5162.0ms/step | 11.6 samples/s +Step 44480/50462 | Loss: 2.5196 | LR: 1.77e-05 | 5169.9ms/step | 11.6 samples/s +Step 44490/50462 | Loss: 2.5149 | LR: 1.76e-05 | 5163.0ms/step | 11.6 samples/s +Step 44500/50462 | Loss: 2.4786 | LR: 1.75e-05 | 5157.3ms/step | 11.6 samples/s +Validation at step 44500: Loss = 2.5017, Perplexity = 12.20 +Step 44510/50462 | Loss: 2.4672 | LR: 1.75e-05 | 7717.4ms/step | 7.8 samples/s +Step 44520/50462 | Loss: 2.4660 | LR: 1.74e-05 | 5162.9ms/step | 11.6 samples/s +Step 44530/50462 | Loss: 2.4923 | LR: 1.74e-05 | 5170.2ms/step | 11.6 samples/s +Step 44540/50462 | Loss: 2.4812 | LR: 1.73e-05 | 5149.0ms/step | 11.7 samples/s +Step 44550/50462 | Loss: 2.4930 | LR: 1.72e-05 | 5165.4ms/step | 11.6 samples/s +Step 44560/50462 | Loss: 2.4985 | LR: 1.72e-05 | 5160.0ms/step | 11.6 samples/s +Step 44570/50462 | Loss: 2.4853 | LR: 1.71e-05 | 5148.3ms/step | 11.7 samples/s +Step 44580/50462 | Loss: 2.4639 | LR: 1.71e-05 | 5147.4ms/step | 11.7 samples/s +Step 44590/50462 | Loss: 2.4818 | LR: 1.70e-05 | 5160.1ms/step | 11.6 samples/s +Step 44600/50462 | Loss: 2.5061 | LR: 1.70e-05 | 5150.7ms/step | 11.6 samples/s +Step 44610/50462 | Loss: 2.4884 | LR: 1.69e-05 | 5164.1ms/step | 11.6 samples/s +Step 44620/50462 | Loss: 2.5049 | LR: 1.68e-05 | 5151.8ms/step | 11.6 samples/s +Step 44630/50462 | Loss: 2.4895 | LR: 1.68e-05 | 5154.2ms/step | 11.6 samples/s +Step 44640/50462 | Loss: 2.4814 | LR: 1.67e-05 | 5166.6ms/step | 11.6 samples/s +Step 44650/50462 | Loss: 2.4700 | LR: 1.67e-05 | 5158.8ms/step | 11.6 samples/s +Step 44660/50462 | Loss: 2.5078 | LR: 1.66e-05 | 5166.0ms/step | 11.6 samples/s +Step 44670/50462 | Loss: 2.5170 | LR: 1.66e-05 | 5161.6ms/step | 11.6 samples/s +Step 44680/50462 | Loss: 2.5035 | LR: 1.65e-05 | 5170.0ms/step | 11.6 samples/s +Step 44690/50462 | Loss: 2.4997 | LR: 1.64e-05 | 5165.6ms/step | 11.6 samples/s +Step 44700/50462 | Loss: 2.5047 | LR: 1.64e-05 | 5155.8ms/step | 11.6 samples/s +Step 44710/50462 | Loss: 2.4886 | LR: 1.63e-05 | 5157.1ms/step | 11.6 samples/s +Step 44720/50462 | Loss: 2.4904 | LR: 1.63e-05 | 5164.3ms/step | 11.6 samples/s +Step 44730/50462 | Loss: 2.4792 | LR: 1.62e-05 | 5161.3ms/step | 11.6 samples/s +Step 44740/50462 | Loss: 2.4737 | LR: 1.62e-05 | 5164.4ms/step | 11.6 samples/s +Step 44750/50462 | Loss: 2.4983 | LR: 1.61e-05 | 5579.5ms/step | 10.8 samples/s +Validation at step 44750: Loss = 2.5016, Perplexity = 12.20 +Step 44760/50462 | Loss: 2.5140 | LR: 1.61e-05 | 7732.0ms/step | 7.8 samples/s +Step 44770/50462 | Loss: 2.4629 | LR: 1.60e-05 | 5181.5ms/step | 11.6 samples/s +Step 44780/50462 | Loss: 2.4911 | LR: 1.59e-05 | 5187.3ms/step | 11.6 samples/s +Step 44790/50462 | Loss: 2.4814 | LR: 1.59e-05 | 5166.9ms/step | 11.6 samples/s +Step 44800/50462 | Loss: 2.4824 | LR: 1.58e-05 | 5175.9ms/step | 11.6 samples/s +Step 44810/50462 | Loss: 2.5080 | LR: 1.58e-05 | 5171.2ms/step | 11.6 samples/s +Step 44820/50462 | Loss: 2.4983 | LR: 1.57e-05 | 5175.5ms/step | 11.6 samples/s +Step 44830/50462 | Loss: 2.5195 | LR: 1.57e-05 | 5168.7ms/step | 11.6 samples/s +Step 44840/50462 | Loss: 2.4759 | LR: 1.56e-05 | 5185.0ms/step | 11.6 samples/s +Step 44850/50462 | Loss: 2.4791 | LR: 1.56e-05 | 5187.7ms/step | 11.6 samples/s +Step 44860/50462 | Loss: 2.4784 | LR: 1.55e-05 | 5174.4ms/step | 11.6 samples/s +Step 44870/50462 | Loss: 2.4685 | LR: 1.54e-05 | 5161.6ms/step | 11.6 samples/s +Step 44880/50462 | Loss: 2.4918 | LR: 1.54e-05 | 5178.2ms/step | 11.6 samples/s +Step 44890/50462 | Loss: 2.4543 | LR: 1.53e-05 | 5163.3ms/step | 11.6 samples/s +Step 44900/50462 | Loss: 2.4549 | LR: 1.53e-05 | 5174.7ms/step | 11.6 samples/s +Step 44910/50462 | Loss: 2.4780 | LR: 1.52e-05 | 5145.4ms/step | 11.7 samples/s +Step 44920/50462 | Loss: 2.4804 | LR: 1.52e-05 | 5155.3ms/step | 11.6 samples/s +Step 44930/50462 | Loss: 2.4837 | LR: 1.51e-05 | 5180.4ms/step | 11.6 samples/s +Step 44940/50462 | Loss: 2.4868 | LR: 1.51e-05 | 5160.8ms/step | 11.6 samples/s +Step 44950/50462 | Loss: 2.4437 | LR: 1.50e-05 | 5167.3ms/step | 11.6 samples/s +Step 44960/50462 | Loss: 2.5224 | LR: 1.50e-05 | 5162.4ms/step | 11.6 samples/s +Step 44970/50462 | Loss: 2.4727 | LR: 1.49e-05 | 5157.0ms/step | 11.6 samples/s +Step 44980/50462 | Loss: 2.4844 | LR: 1.49e-05 | 5150.3ms/step | 11.6 samples/s +Step 44990/50462 | Loss: 2.5130 | LR: 1.48e-05 | 5130.3ms/step | 11.7 samples/s +Step 45000/50462 | Loss: 2.5040 | LR: 1.47e-05 | 5142.6ms/step | 11.7 samples/s +Validation at step 45000: Loss = 2.5016, Perplexity = 12.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-45000 +Step 45010/50462 | Loss: 2.4961 | LR: 1.47e-05 | 8566.6ms/step | 7.0 samples/s +Step 45020/50462 | Loss: 2.4802 | LR: 1.46e-05 | 5113.0ms/step | 11.7 samples/s +Step 45030/50462 | Loss: 2.4866 | LR: 1.46e-05 | 5157.1ms/step | 11.6 samples/s +Step 45040/50462 | Loss: 2.4992 | LR: 1.45e-05 | 5130.6ms/step | 11.7 samples/s +Step 45050/50462 | Loss: 2.4751 | LR: 1.45e-05 | 5131.5ms/step | 11.7 samples/s +Step 45060/50462 | Loss: 2.4977 | LR: 1.44e-05 | 5146.8ms/step | 11.7 samples/s +Step 45070/50462 | Loss: 2.4918 | LR: 1.44e-05 | 5123.7ms/step | 11.7 samples/s +Step 45080/50462 | Loss: 2.4832 | LR: 1.43e-05 | 5145.3ms/step | 11.7 samples/s +Step 45090/50462 | Loss: 2.4830 | LR: 1.43e-05 | 5133.0ms/step | 11.7 samples/s +Step 45100/50462 | Loss: 2.5169 | LR: 1.42e-05 | 5125.3ms/step | 11.7 samples/s +Step 45110/50462 | Loss: 2.5165 | LR: 1.42e-05 | 5106.6ms/step | 11.7 samples/s +Step 45120/50462 | Loss: 2.4996 | LR: 1.41e-05 | 5177.5ms/step | 11.6 samples/s +Step 45130/50462 | Loss: 2.5075 | LR: 1.41e-05 | 5609.5ms/step | 10.7 samples/s +Step 45140/50462 | Loss: 2.4769 | LR: 1.40e-05 | 5621.8ms/step | 10.7 samples/s +Step 45150/50462 | Loss: 2.5012 | LR: 1.40e-05 | 5644.3ms/step | 10.6 samples/s +Step 45160/50462 | Loss: 2.5075 | LR: 1.39e-05 | 5640.0ms/step | 10.6 samples/s +Step 45170/50462 | Loss: 2.4689 | LR: 1.39e-05 | 5631.4ms/step | 10.7 samples/s +Step 45180/50462 | Loss: 2.4976 | LR: 1.38e-05 | 5634.2ms/step | 10.6 samples/s +Step 45190/50462 | Loss: 2.4842 | LR: 1.37e-05 | 5627.5ms/step | 10.7 samples/s +Step 45200/50462 | Loss: 2.5073 | LR: 1.37e-05 | 5628.4ms/step | 10.7 samples/s +Step 45210/50462 | Loss: 2.4934 | LR: 1.36e-05 | 5625.4ms/step | 10.7 samples/s +Step 45220/50462 | Loss: 2.4786 | LR: 1.36e-05 | 5625.5ms/step | 10.7 samples/s +Step 45230/50462 | Loss: 2.4726 | LR: 1.35e-05 | 5628.7ms/step | 10.7 samples/s +Step 45240/50462 | Loss: 2.5078 | LR: 1.35e-05 | 5627.2ms/step | 10.7 samples/s +Step 45250/50462 | Loss: 2.4846 | LR: 1.34e-05 | 5620.8ms/step | 10.7 samples/s +Validation at step 45250: Loss = 2.5016, Perplexity = 12.20 +Step 45260/50462 | Loss: 2.4962 | LR: 1.34e-05 | 8424.3ms/step | 7.1 samples/s +Step 45270/50462 | Loss: 2.5062 | LR: 1.33e-05 | 5623.8ms/step | 10.7 samples/s +Step 45280/50462 | Loss: 2.4631 | LR: 1.33e-05 | 5625.7ms/step | 10.7 samples/s +Step 45290/50462 | Loss: 2.4898 | LR: 1.32e-05 | 5624.6ms/step | 10.7 samples/s +Step 45300/50462 | Loss: 2.5077 | LR: 1.32e-05 | 5623.3ms/step | 10.7 samples/s +Step 45310/50462 | Loss: 2.4768 | LR: 1.31e-05 | 5620.1ms/step | 10.7 samples/s +Step 45320/50462 | Loss: 2.4849 | LR: 1.31e-05 | 5618.3ms/step | 10.7 samples/s +Step 45330/50462 | Loss: 2.5010 | LR: 1.30e-05 | 5620.6ms/step | 10.7 samples/s +Step 45340/50462 | Loss: 2.4922 | LR: 1.30e-05 | 6111.5ms/step | 9.8 samples/s +Step 45350/50462 | Loss: 2.4893 | LR: 1.29e-05 | 6089.6ms/step | 9.9 samples/s +Step 45360/50462 | Loss: 2.5086 | LR: 1.29e-05 | 5607.8ms/step | 10.7 samples/s +Step 45370/50462 | Loss: 2.4823 | LR: 1.28e-05 | 5606.4ms/step | 10.7 samples/s +Step 45380/50462 | Loss: 2.4778 | LR: 1.28e-05 | 5607.3ms/step | 10.7 samples/s +Step 45390/50462 | Loss: 2.4851 | LR: 1.27e-05 | 5598.3ms/step | 10.7 samples/s +Step 45400/50462 | Loss: 2.4680 | LR: 1.27e-05 | 5602.3ms/step | 10.7 samples/s +Step 45410/50462 | Loss: 2.5136 | LR: 1.26e-05 | 5600.8ms/step | 10.7 samples/s +Step 45420/50462 | Loss: 2.4841 | LR: 1.26e-05 | 5600.5ms/step | 10.7 samples/s +Step 45430/50462 | Loss: 2.4594 | LR: 1.25e-05 | 5600.9ms/step | 10.7 samples/s +Step 45440/50462 | Loss: 2.5009 | LR: 1.25e-05 | 5599.1ms/step | 10.7 samples/s +Step 45450/50462 | Loss: 2.4909 | LR: 1.24e-05 | 5600.4ms/step | 10.7 samples/s +Step 45460/50462 | Loss: 2.4814 | LR: 1.24e-05 | 5602.2ms/step | 10.7 samples/s +Step 45470/50462 | Loss: 2.4901 | LR: 1.23e-05 | 5602.2ms/step | 10.7 samples/s +Step 45480/50462 | Loss: 2.4802 | LR: 1.23e-05 | 5516.1ms/step | 10.9 samples/s +Step 45490/50462 | Loss: 2.5015 | LR: 1.22e-05 | 5117.6ms/step | 11.7 samples/s +Step 45500/50462 | Loss: 2.5012 | LR: 1.22e-05 | 5117.4ms/step | 11.7 samples/s +Validation at step 45500: Loss = 2.5015, Perplexity = 12.20 +Step 45510/50462 | Loss: 2.4754 | LR: 1.21e-05 | 7668.7ms/step | 7.8 samples/s +Step 45520/50462 | Loss: 2.5215 | LR: 1.21e-05 | 5128.6ms/step | 11.7 samples/s +Step 45530/50462 | Loss: 2.4675 | LR: 1.20e-05 | 5110.3ms/step | 11.7 samples/s +Step 45540/50462 | Loss: 2.5156 | LR: 1.20e-05 | 5522.6ms/step | 10.9 samples/s +Step 45550/50462 | Loss: 2.4758 | LR: 1.19e-05 | 5720.5ms/step | 10.5 samples/s +Step 45560/50462 | Loss: 2.5005 | LR: 1.19e-05 | 5717.3ms/step | 10.5 samples/s +Step 45570/50462 | Loss: 2.5050 | LR: 1.19e-05 | 5749.1ms/step | 10.4 samples/s +Step 45580/50462 | Loss: 2.4824 | LR: 1.18e-05 | 5745.4ms/step | 10.4 samples/s +Step 45590/50462 | Loss: 2.4712 | LR: 1.18e-05 | 5744.2ms/step | 10.4 samples/s +Step 45600/50462 | Loss: 2.4795 | LR: 1.17e-05 | 5701.6ms/step | 10.5 samples/s +Step 45610/50462 | Loss: 2.4576 | LR: 1.17e-05 | 5651.1ms/step | 10.6 samples/s +Step 45620/50462 | Loss: 2.4892 | LR: 1.16e-05 | 5667.6ms/step | 10.6 samples/s +Step 45630/50462 | Loss: 2.5066 | LR: 1.16e-05 | 5640.5ms/step | 10.6 samples/s +Step 45640/50462 | Loss: 2.4848 | LR: 1.15e-05 | 5644.2ms/step | 10.6 samples/s +Step 45650/50462 | Loss: 2.5098 | LR: 1.15e-05 | 5647.8ms/step | 10.6 samples/s +Step 45660/50462 | Loss: 2.4946 | LR: 1.14e-05 | 5666.4ms/step | 10.6 samples/s +Step 45670/50462 | Loss: 2.4609 | LR: 1.14e-05 | 5662.4ms/step | 10.6 samples/s +Step 45680/50462 | Loss: 2.4804 | LR: 1.13e-05 | 5648.2ms/step | 10.6 samples/s +Step 45690/50462 | Loss: 2.5017 | LR: 1.13e-05 | 5646.9ms/step | 10.6 samples/s +Step 45700/50462 | Loss: 2.4931 | LR: 1.12e-05 | 5644.6ms/step | 10.6 samples/s +Step 45710/50462 | Loss: 2.4965 | LR: 1.12e-05 | 5652.6ms/step | 10.6 samples/s +Step 45720/50462 | Loss: 2.4878 | LR: 1.11e-05 | 5645.5ms/step | 10.6 samples/s +Step 45730/50462 | Loss: 2.4688 | LR: 1.11e-05 | 5646.9ms/step | 10.6 samples/s +Step 45740/50462 | Loss: 2.4830 | LR: 1.10e-05 | 5644.0ms/step | 10.6 samples/s +Step 45750/50462 | Loss: 2.5013 | LR: 1.10e-05 | 5643.8ms/step | 10.6 samples/s +Validation at step 45750: Loss = 2.5015, Perplexity = 12.20 +Step 45760/50462 | Loss: 2.4554 | LR: 1.10e-05 | 8434.1ms/step | 7.1 samples/s +Step 45770/50462 | Loss: 2.4871 | LR: 1.09e-05 | 5638.5ms/step | 10.6 samples/s +Step 45780/50462 | Loss: 2.4802 | LR: 1.09e-05 | 5641.8ms/step | 10.6 samples/s +Step 45790/50462 | Loss: 2.5183 | LR: 1.08e-05 | 5639.7ms/step | 10.6 samples/s +Step 45800/50462 | Loss: 2.4319 | LR: 1.08e-05 | 5637.7ms/step | 10.6 samples/s +Step 45810/50462 | Loss: 2.4931 | LR: 1.07e-05 | 5634.3ms/step | 10.6 samples/s +Step 45820/50462 | Loss: 2.4883 | LR: 1.07e-05 | 5633.7ms/step | 10.7 samples/s +Step 45830/50462 | Loss: 2.4950 | LR: 1.06e-05 | 5629.3ms/step | 10.7 samples/s +Step 45840/50462 | Loss: 2.5082 | LR: 1.06e-05 | 5631.6ms/step | 10.7 samples/s +Step 45850/50462 | Loss: 2.4906 | LR: 1.05e-05 | 5630.3ms/step | 10.7 samples/s +Step 45860/50462 | Loss: 2.5150 | LR: 1.05e-05 | 5630.0ms/step | 10.7 samples/s +Step 45870/50462 | Loss: 2.4999 | LR: 1.05e-05 | 5629.0ms/step | 10.7 samples/s +Step 45880/50462 | Loss: 2.5006 | LR: 1.04e-05 | 5626.1ms/step | 10.7 samples/s +Step 45890/50462 | Loss: 2.4635 | LR: 1.04e-05 | 5629.7ms/step | 10.7 samples/s +Step 45900/50462 | Loss: 2.4866 | LR: 1.03e-05 | 5626.4ms/step | 10.7 samples/s +Step 45910/50462 | Loss: 2.5164 | LR: 1.03e-05 | 5624.6ms/step | 10.7 samples/s +Step 45920/50462 | Loss: 2.4694 | LR: 1.02e-05 | 5629.2ms/step | 10.7 samples/s +Step 45930/50462 | Loss: 2.4812 | LR: 1.02e-05 | 5508.2ms/step | 10.9 samples/s +Step 45940/50462 | Loss: 2.5004 | LR: 1.01e-05 | 5122.9ms/step | 11.7 samples/s +Step 45950/50462 | Loss: 2.4873 | LR: 1.01e-05 | 5637.3ms/step | 10.6 samples/s +Step 45960/50462 | Loss: 2.4731 | LR: 1.01e-05 | 5651.3ms/step | 10.6 samples/s +Step 45970/50462 | Loss: 2.4792 | LR: 1.00e-05 | 5649.4ms/step | 10.6 samples/s +Step 45980/50462 | Loss: 2.5228 | LR: 9.96e-06 | 5652.0ms/step | 10.6 samples/s +Step 45990/50462 | Loss: 2.4765 | LR: 9.92e-06 | 5647.2ms/step | 10.6 samples/s +Step 46000/50462 | Loss: 2.4909 | LR: 9.87e-06 | 5646.0ms/step | 10.6 samples/s +Validation at step 46000: Loss = 2.5015, Perplexity = 12.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-46000 +Step 46010/50462 | Loss: 2.4936 | LR: 9.83e-06 | 9321.3ms/step | 6.4 samples/s +Step 46020/50462 | Loss: 2.4798 | LR: 9.79e-06 | 5672.4ms/step | 10.6 samples/s +Step 46030/50462 | Loss: 2.5040 | LR: 9.74e-06 | 5673.5ms/step | 10.6 samples/s +Step 46040/50462 | Loss: 2.5040 | LR: 9.70e-06 | 5672.2ms/step | 10.6 samples/s +Step 46050/50462 | Loss: 2.4866 | LR: 9.65e-06 | 5669.8ms/step | 10.6 samples/s +Step 46060/50462 | Loss: 2.4814 | LR: 9.61e-06 | 5674.8ms/step | 10.6 samples/s +Step 46070/50462 | Loss: 2.5007 | LR: 9.57e-06 | 5672.3ms/step | 10.6 samples/s +Step 46080/50462 | Loss: 2.4963 | LR: 9.52e-06 | 5673.5ms/step | 10.6 samples/s +Step 46090/50462 | Loss: 2.4652 | LR: 9.48e-06 | 5673.8ms/step | 10.6 samples/s +Step 46100/50462 | Loss: 2.4914 | LR: 9.44e-06 | 5676.7ms/step | 10.6 samples/s +Step 46110/50462 | Loss: 2.4870 | LR: 9.40e-06 | 5677.9ms/step | 10.6 samples/s +Step 46120/50462 | Loss: 2.4816 | LR: 9.35e-06 | 5675.3ms/step | 10.6 samples/s +Step 46130/50462 | Loss: 2.4827 | LR: 9.31e-06 | 5674.6ms/step | 10.6 samples/s +Step 46140/50462 | Loss: 2.4837 | LR: 9.27e-06 | 5674.2ms/step | 10.6 samples/s +Step 46150/50462 | Loss: 2.4822 | LR: 9.22e-06 | 5673.9ms/step | 10.6 samples/s +Step 46160/50462 | Loss: 2.4995 | LR: 9.18e-06 | 5671.7ms/step | 10.6 samples/s +Step 46170/50462 | Loss: 2.4999 | LR: 9.14e-06 | 5672.8ms/step | 10.6 samples/s +Step 46180/50462 | Loss: 2.4699 | LR: 9.10e-06 | 5675.8ms/step | 10.6 samples/s +Step 46190/50462 | Loss: 2.4707 | LR: 9.06e-06 | 5672.4ms/step | 10.6 samples/s +Step 46200/50462 | Loss: 2.4785 | LR: 9.01e-06 | 5674.9ms/step | 10.6 samples/s +Step 46210/50462 | Loss: 2.4944 | LR: 8.97e-06 | 5675.2ms/step | 10.6 samples/s +Step 46220/50462 | Loss: 2.4973 | LR: 8.93e-06 | 5675.1ms/step | 10.6 samples/s +Step 46230/50462 | Loss: 2.5043 | LR: 8.89e-06 | 5673.3ms/step | 10.6 samples/s +Step 46240/50462 | Loss: 2.5126 | LR: 8.85e-06 | 5676.8ms/step | 10.6 samples/s +Step 46250/50462 | Loss: 2.5101 | LR: 8.80e-06 | 5671.8ms/step | 10.6 samples/s +Validation at step 46250: Loss = 2.5015, Perplexity = 12.20 +Step 46260/50462 | Loss: 2.5099 | LR: 8.76e-06 | 8497.4ms/step | 7.1 samples/s +Step 46270/50462 | Loss: 2.4972 | LR: 8.72e-06 | 5676.3ms/step | 10.6 samples/s +Step 46280/50462 | Loss: 2.5284 | LR: 8.68e-06 | 5670.8ms/step | 10.6 samples/s +Step 46290/50462 | Loss: 2.5147 | LR: 8.64e-06 | 5673.3ms/step | 10.6 samples/s +Step 46300/50462 | Loss: 2.4853 | LR: 8.60e-06 | 5673.7ms/step | 10.6 samples/s +Step 46310/50462 | Loss: 2.4931 | LR: 8.56e-06 | 5673.0ms/step | 10.6 samples/s +Step 46320/50462 | Loss: 2.5181 | LR: 8.52e-06 | 5673.6ms/step | 10.6 samples/s +Step 46330/50462 | Loss: 2.4983 | LR: 8.47e-06 | 5673.5ms/step | 10.6 samples/s +Step 46340/50462 | Loss: 2.4975 | LR: 8.43e-06 | 5674.5ms/step | 10.6 samples/s +Step 46350/50462 | Loss: 2.5185 | LR: 8.39e-06 | 5674.7ms/step | 10.6 samples/s +Step 46360/50462 | Loss: 2.4814 | LR: 8.35e-06 | 5673.2ms/step | 10.6 samples/s +Step 46370/50462 | Loss: 2.4672 | LR: 8.31e-06 | 5672.2ms/step | 10.6 samples/s +Step 46380/50462 | Loss: 2.4919 | LR: 8.27e-06 | 5673.8ms/step | 10.6 samples/s +Step 46390/50462 | Loss: 2.4819 | LR: 8.23e-06 | 5670.2ms/step | 10.6 samples/s +Step 46400/50462 | Loss: 2.4733 | LR: 8.19e-06 | 5676.6ms/step | 10.6 samples/s +Step 46410/50462 | Loss: 2.4858 | LR: 8.15e-06 | 5675.3ms/step | 10.6 samples/s +Step 46420/50462 | Loss: 2.4845 | LR: 8.11e-06 | 5672.0ms/step | 10.6 samples/s +Step 46430/50462 | Loss: 2.5058 | LR: 8.07e-06 | 5675.4ms/step | 10.6 samples/s +Step 46440/50462 | Loss: 2.4886 | LR: 8.03e-06 | 5673.6ms/step | 10.6 samples/s +Step 46450/50462 | Loss: 2.5146 | LR: 7.99e-06 | 5672.0ms/step | 10.6 samples/s +Step 46460/50462 | Loss: 2.4924 | LR: 7.95e-06 | 5671.7ms/step | 10.6 samples/s +Step 46470/50462 | Loss: 2.4840 | LR: 7.91e-06 | 5670.9ms/step | 10.6 samples/s +Step 46480/50462 | Loss: 2.4796 | LR: 7.87e-06 | 5677.4ms/step | 10.6 samples/s +Step 46490/50462 | Loss: 2.4747 | LR: 7.83e-06 | 5671.1ms/step | 10.6 samples/s +Step 46500/50462 | Loss: 2.4923 | LR: 7.80e-06 | 5674.0ms/step | 10.6 samples/s +Validation at step 46500: Loss = 2.5015, Perplexity = 12.20 +Step 46510/50462 | Loss: 2.4873 | LR: 7.76e-06 | 8487.4ms/step | 7.1 samples/s +Step 46520/50462 | Loss: 2.5108 | LR: 7.72e-06 | 5669.2ms/step | 10.6 samples/s +Step 46530/50462 | Loss: 2.4893 | LR: 7.68e-06 | 5671.9ms/step | 10.6 samples/s +Step 46540/50462 | Loss: 2.5034 | LR: 7.64e-06 | 5675.3ms/step | 10.6 samples/s +Step 46550/50462 | Loss: 2.4638 | LR: 7.60e-06 | 5670.6ms/step | 10.6 samples/s +Step 46560/50462 | Loss: 2.4814 | LR: 7.56e-06 | 5670.9ms/step | 10.6 samples/s +Step 46570/50462 | Loss: 2.4918 | LR: 7.52e-06 | 5671.8ms/step | 10.6 samples/s +Step 46580/50462 | Loss: 2.4839 | LR: 7.49e-06 | 5668.9ms/step | 10.6 samples/s +Step 46590/50462 | Loss: 2.4757 | LR: 7.45e-06 | 5672.7ms/step | 10.6 samples/s +Step 46600/50462 | Loss: 2.4913 | LR: 7.41e-06 | 5668.4ms/step | 10.6 samples/s +Step 46610/50462 | Loss: 2.4905 | LR: 7.37e-06 | 5666.4ms/step | 10.6 samples/s +Step 46620/50462 | Loss: 2.4715 | LR: 7.33e-06 | 5667.3ms/step | 10.6 samples/s +Step 46630/50462 | Loss: 2.4867 | LR: 7.29e-06 | 5666.4ms/step | 10.6 samples/s +Step 46640/50462 | Loss: 2.4987 | LR: 7.26e-06 | 5670.3ms/step | 10.6 samples/s +Step 46650/50462 | Loss: 2.4908 | LR: 7.22e-06 | 5668.4ms/step | 10.6 samples/s +Step 46660/50462 | Loss: 2.4810 | LR: 7.18e-06 | 5671.5ms/step | 10.6 samples/s +Step 46670/50462 | Loss: 2.4956 | LR: 7.14e-06 | 5670.0ms/step | 10.6 samples/s +Step 46680/50462 | Loss: 2.5041 | LR: 7.11e-06 | 5668.3ms/step | 10.6 samples/s +Step 46690/50462 | Loss: 2.5213 | LR: 7.07e-06 | 5668.7ms/step | 10.6 samples/s +Step 46700/50462 | Loss: 2.4819 | LR: 7.03e-06 | 5673.5ms/step | 10.6 samples/s +Step 46710/50462 | Loss: 2.5139 | LR: 6.99e-06 | 5670.8ms/step | 10.6 samples/s +Step 46720/50462 | Loss: 2.4977 | LR: 6.96e-06 | 5672.7ms/step | 10.6 samples/s +Step 46730/50462 | Loss: 2.4778 | LR: 6.92e-06 | 5667.2ms/step | 10.6 samples/s +Step 46740/50462 | Loss: 2.5045 | LR: 6.88e-06 | 5668.1ms/step | 10.6 samples/s +Step 46750/50462 | Loss: 2.4872 | LR: 6.85e-06 | 5668.4ms/step | 10.6 samples/s +Validation at step 46750: Loss = 2.5015, Perplexity = 12.20 +Step 46760/50462 | Loss: 2.5018 | LR: 6.81e-06 | 8492.4ms/step | 7.1 samples/s +Step 46770/50462 | Loss: 2.4915 | LR: 6.77e-06 | 5671.0ms/step | 10.6 samples/s +Step 46780/50462 | Loss: 2.5172 | LR: 6.74e-06 | 5670.9ms/step | 10.6 samples/s +Step 46790/50462 | Loss: 2.4744 | LR: 6.70e-06 | 5670.4ms/step | 10.6 samples/s +Step 46800/50462 | Loss: 2.5112 | LR: 6.66e-06 | 5674.9ms/step | 10.6 samples/s +Step 46810/50462 | Loss: 2.5072 | LR: 6.63e-06 | 6122.7ms/step | 9.8 samples/s +Step 46820/50462 | Loss: 2.4712 | LR: 6.59e-06 | 5667.9ms/step | 10.6 samples/s +Step 46830/50462 | Loss: 2.4694 | LR: 6.56e-06 | 5670.2ms/step | 10.6 samples/s +Step 46840/50462 | Loss: 2.4950 | LR: 6.52e-06 | 5672.4ms/step | 10.6 samples/s +Step 46850/50462 | Loss: 2.4912 | LR: 6.48e-06 | 5668.7ms/step | 10.6 samples/s +Step 46860/50462 | Loss: 2.4913 | LR: 6.45e-06 | 5674.3ms/step | 10.6 samples/s +Step 46870/50462 | Loss: 2.4482 | LR: 6.41e-06 | 5669.7ms/step | 10.6 samples/s +Step 46880/50462 | Loss: 2.4847 | LR: 6.38e-06 | 5670.6ms/step | 10.6 samples/s +Step 46890/50462 | Loss: 2.4904 | LR: 6.34e-06 | 5664.0ms/step | 10.6 samples/s +Step 46900/50462 | Loss: 2.4821 | LR: 6.31e-06 | 5539.6ms/step | 10.8 samples/s +Step 46910/50462 | Loss: 2.4756 | LR: 6.27e-06 | 5161.1ms/step | 11.6 samples/s +Step 46920/50462 | Loss: 2.5137 | LR: 6.24e-06 | 5159.6ms/step | 11.6 samples/s +Step 46930/50462 | Loss: 2.4898 | LR: 6.20e-06 | 5172.4ms/step | 11.6 samples/s +Step 46940/50462 | Loss: 2.4910 | LR: 6.17e-06 | 5145.0ms/step | 11.7 samples/s +Step 46950/50462 | Loss: 2.4748 | LR: 6.13e-06 | 5663.2ms/step | 10.6 samples/s +Step 46960/50462 | Loss: 2.4978 | LR: 6.10e-06 | 5742.8ms/step | 10.4 samples/s +Step 46970/50462 | Loss: 2.5019 | LR: 6.06e-06 | 5739.1ms/step | 10.5 samples/s +Step 46980/50462 | Loss: 2.5006 | LR: 6.03e-06 | 5749.9ms/step | 10.4 samples/s +Step 46990/50462 | Loss: 2.5220 | LR: 5.99e-06 | 5753.4ms/step | 10.4 samples/s +Step 47000/50462 | Loss: 2.5013 | LR: 5.96e-06 | 5752.4ms/step | 10.4 samples/s +Validation at step 47000: Loss = 2.5015, Perplexity = 12.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-47000 +Step 47010/50462 | Loss: 2.5141 | LR: 5.93e-06 | 9443.4ms/step | 6.4 samples/s +Step 47020/50462 | Loss: 2.4899 | LR: 5.89e-06 | 5747.1ms/step | 10.4 samples/s +Step 47030/50462 | Loss: 2.5099 | LR: 5.86e-06 | 5748.0ms/step | 10.4 samples/s +Step 47040/50462 | Loss: 2.4552 | LR: 5.82e-06 | 5747.6ms/step | 10.4 samples/s +Step 47050/50462 | Loss: 2.4920 | LR: 5.79e-06 | 5747.3ms/step | 10.4 samples/s +Step 47060/50462 | Loss: 2.4899 | LR: 5.76e-06 | 5746.9ms/step | 10.4 samples/s +Step 47070/50462 | Loss: 2.4831 | LR: 5.72e-06 | 5750.3ms/step | 10.4 samples/s +Step 47080/50462 | Loss: 2.4907 | LR: 5.69e-06 | 5750.7ms/step | 10.4 samples/s +Step 47090/50462 | Loss: 2.4856 | LR: 5.65e-06 | 5747.2ms/step | 10.4 samples/s +Step 47100/50462 | Loss: 2.4892 | LR: 5.62e-06 | 5751.1ms/step | 10.4 samples/s +Step 47110/50462 | Loss: 2.4933 | LR: 5.59e-06 | 5749.1ms/step | 10.4 samples/s +Step 47120/50462 | Loss: 2.4438 | LR: 5.56e-06 | 5755.2ms/step | 10.4 samples/s +Step 47130/50462 | Loss: 2.4906 | LR: 5.52e-06 | 5747.2ms/step | 10.4 samples/s +Step 47140/50462 | Loss: 2.5026 | LR: 5.49e-06 | 5746.9ms/step | 10.4 samples/s +Step 47150/50462 | Loss: 2.4594 | LR: 5.46e-06 | 5746.3ms/step | 10.4 samples/s +Step 47160/50462 | Loss: 2.5013 | LR: 5.42e-06 | 5747.4ms/step | 10.4 samples/s +Step 47170/50462 | Loss: 2.4835 | LR: 5.39e-06 | 5746.4ms/step | 10.4 samples/s +Step 47180/50462 | Loss: 2.4943 | LR: 5.36e-06 | 5743.8ms/step | 10.4 samples/s +Step 47190/50462 | Loss: 2.4879 | LR: 5.33e-06 | 5747.1ms/step | 10.4 samples/s +Step 47200/50462 | Loss: 2.4860 | LR: 5.29e-06 | 5746.6ms/step | 10.4 samples/s +Step 47210/50462 | Loss: 2.4590 | LR: 5.26e-06 | 5750.9ms/step | 10.4 samples/s +Step 47220/50462 | Loss: 2.5008 | LR: 5.23e-06 | 5748.7ms/step | 10.4 samples/s +Step 47230/50462 | Loss: 2.4986 | LR: 5.20e-06 | 5744.1ms/step | 10.4 samples/s +Step 47240/50462 | Loss: 2.4672 | LR: 5.16e-06 | 5742.5ms/step | 10.4 samples/s +Step 47250/50462 | Loss: 2.4972 | LR: 5.13e-06 | 5746.4ms/step | 10.4 samples/s +Validation at step 47250: Loss = 2.5015, Perplexity = 12.20 +Step 47260/50462 | Loss: 2.4922 | LR: 5.10e-06 | 8601.2ms/step | 7.0 samples/s +Step 47270/50462 | Loss: 2.5037 | LR: 5.07e-06 | 5747.5ms/step | 10.4 samples/s +Step 47280/50462 | Loss: 2.4767 | LR: 5.04e-06 | 5743.3ms/step | 10.4 samples/s +Step 47290/50462 | Loss: 2.5056 | LR: 5.01e-06 | 5750.5ms/step | 10.4 samples/s +Step 47300/50462 | Loss: 2.5040 | LR: 4.97e-06 | 5748.4ms/step | 10.4 samples/s +Step 47310/50462 | Loss: 2.4668 | LR: 4.94e-06 | 5752.8ms/step | 10.4 samples/s +Step 47320/50462 | Loss: 2.4832 | LR: 4.91e-06 | 5754.5ms/step | 10.4 samples/s +Step 47330/50462 | Loss: 2.4861 | LR: 4.88e-06 | 5752.3ms/step | 10.4 samples/s +Step 47340/50462 | Loss: 2.4504 | LR: 4.85e-06 | 5751.0ms/step | 10.4 samples/s +Step 47350/50462 | Loss: 2.4984 | LR: 4.82e-06 | 5750.9ms/step | 10.4 samples/s +Step 47360/50462 | Loss: 2.4694 | LR: 4.79e-06 | 5752.7ms/step | 10.4 samples/s +Step 47370/50462 | Loss: 2.5077 | LR: 4.76e-06 | 5752.8ms/step | 10.4 samples/s +Step 47380/50462 | Loss: 2.5002 | LR: 4.73e-06 | 5754.1ms/step | 10.4 samples/s +Step 47390/50462 | Loss: 2.4822 | LR: 4.70e-06 | 5754.3ms/step | 10.4 samples/s +Step 47400/50462 | Loss: 2.4863 | LR: 4.67e-06 | 5752.9ms/step | 10.4 samples/s +Step 47410/50462 | Loss: 2.4968 | LR: 4.64e-06 | 5751.5ms/step | 10.4 samples/s +Step 47420/50462 | Loss: 2.4488 | LR: 4.61e-06 | 5753.4ms/step | 10.4 samples/s +Step 47430/50462 | Loss: 2.4770 | LR: 4.58e-06 | 5749.1ms/step | 10.4 samples/s +Step 47440/50462 | Loss: 2.4857 | LR: 4.55e-06 | 5751.7ms/step | 10.4 samples/s +Step 47450/50462 | Loss: 2.4605 | LR: 4.52e-06 | 6259.0ms/step | 9.6 samples/s +Step 47460/50462 | Loss: 2.5049 | LR: 4.49e-06 | 5741.1ms/step | 10.5 samples/s +Step 47470/50462 | Loss: 2.5025 | LR: 4.46e-06 | 6274.3ms/step | 9.6 samples/s +Step 47480/50462 | Loss: 2.5011 | LR: 4.43e-06 | 5745.3ms/step | 10.4 samples/s +Step 47490/50462 | Loss: 2.4952 | LR: 4.40e-06 | 5742.7ms/step | 10.4 samples/s +Step 47500/50462 | Loss: 2.4857 | LR: 4.37e-06 | 5739.2ms/step | 10.5 samples/s +Validation at step 47500: Loss = 2.5015, Perplexity = 12.20 +Step 47510/50462 | Loss: 2.4568 | LR: 4.34e-06 | 8596.7ms/step | 7.0 samples/s +Step 47520/50462 | Loss: 2.5046 | LR: 4.31e-06 | 5745.7ms/step | 10.4 samples/s +Step 47530/50462 | Loss: 2.4998 | LR: 4.28e-06 | 5741.5ms/step | 10.5 samples/s +Step 47540/50462 | Loss: 2.4676 | LR: 4.25e-06 | 5739.4ms/step | 10.5 samples/s +Step 47550/50462 | Loss: 2.4972 | LR: 4.22e-06 | 5740.9ms/step | 10.5 samples/s +Step 47560/50462 | Loss: 2.4832 | LR: 4.19e-06 | 5743.2ms/step | 10.4 samples/s +Step 47570/50462 | Loss: 2.4928 | LR: 4.16e-06 | 5739.8ms/step | 10.5 samples/s +Step 47580/50462 | Loss: 2.5290 | LR: 4.13e-06 | 5741.2ms/step | 10.5 samples/s +Step 47590/50462 | Loss: 2.4993 | LR: 4.11e-06 | 5740.4ms/step | 10.5 samples/s +Step 47600/50462 | Loss: 2.4909 | LR: 4.08e-06 | 5734.4ms/step | 10.5 samples/s +Step 47610/50462 | Loss: 2.4703 | LR: 4.05e-06 | 5726.8ms/step | 10.5 samples/s +Step 47620/50462 | Loss: 2.4839 | LR: 4.02e-06 | 5725.8ms/step | 10.5 samples/s +Step 47630/50462 | Loss: 2.5111 | LR: 3.99e-06 | 5733.6ms/step | 10.5 samples/s +Step 47640/50462 | Loss: 2.4666 | LR: 3.97e-06 | 5717.0ms/step | 10.5 samples/s +Step 47650/50462 | Loss: 2.4852 | LR: 3.94e-06 | 5724.5ms/step | 10.5 samples/s +Step 47660/50462 | Loss: 2.4848 | LR: 3.91e-06 | 5718.9ms/step | 10.5 samples/s +Step 47670/50462 | Loss: 2.4767 | LR: 3.88e-06 | 5708.7ms/step | 10.5 samples/s +Step 47680/50462 | Loss: 2.4944 | LR: 3.85e-06 | 5727.5ms/step | 10.5 samples/s +Step 47690/50462 | Loss: 2.4906 | LR: 3.83e-06 | 5726.9ms/step | 10.5 samples/s +Step 47700/50462 | Loss: 2.4980 | LR: 3.80e-06 | 5710.1ms/step | 10.5 samples/s +Step 47710/50462 | Loss: 2.4645 | LR: 3.77e-06 | 5704.5ms/step | 10.5 samples/s +Step 47720/50462 | Loss: 2.4912 | LR: 3.74e-06 | 5713.5ms/step | 10.5 samples/s +Step 47730/50462 | Loss: 2.4924 | LR: 3.72e-06 | 5713.9ms/step | 10.5 samples/s +Step 47740/50462 | Loss: 2.4774 | LR: 3.69e-06 | 5715.9ms/step | 10.5 samples/s +Step 47750/50462 | Loss: 2.5039 | LR: 3.66e-06 | 5731.2ms/step | 10.5 samples/s +Validation at step 47750: Loss = 2.5015, Perplexity = 12.20 +Step 47760/50462 | Loss: 2.4614 | LR: 3.64e-06 | 8578.6ms/step | 7.0 samples/s +Step 47770/50462 | Loss: 2.5234 | LR: 3.61e-06 | 5731.2ms/step | 10.5 samples/s +Step 47780/50462 | Loss: 2.4815 | LR: 3.58e-06 | 5733.2ms/step | 10.5 samples/s +Step 47790/50462 | Loss: 2.4756 | LR: 3.56e-06 | 5731.1ms/step | 10.5 samples/s +Step 47800/50462 | Loss: 2.4686 | LR: 3.53e-06 | 5734.4ms/step | 10.5 samples/s +Step 47810/50462 | Loss: 2.5014 | LR: 3.50e-06 | 5726.7ms/step | 10.5 samples/s +Step 47820/50462 | Loss: 2.4944 | LR: 3.48e-06 | 5728.1ms/step | 10.5 samples/s +Step 47830/50462 | Loss: 2.4864 | LR: 3.45e-06 | 5726.4ms/step | 10.5 samples/s +Step 47840/50462 | Loss: 2.5097 | LR: 3.42e-06 | 5724.9ms/step | 10.5 samples/s +Step 47850/50462 | Loss: 2.4997 | LR: 3.40e-06 | 5721.4ms/step | 10.5 samples/s +Step 47860/50462 | Loss: 2.5019 | LR: 3.37e-06 | 5718.3ms/step | 10.5 samples/s +Step 47870/50462 | Loss: 2.4963 | LR: 3.35e-06 | 5717.7ms/step | 10.5 samples/s +Step 47880/50462 | Loss: 2.4502 | LR: 3.32e-06 | 5714.1ms/step | 10.5 samples/s +Step 47890/50462 | Loss: 2.5087 | LR: 3.30e-06 | 5715.9ms/step | 10.5 samples/s +Step 47900/50462 | Loss: 2.4966 | LR: 3.27e-06 | 5711.2ms/step | 10.5 samples/s +Step 47910/50462 | Loss: 2.4960 | LR: 3.24e-06 | 5698.8ms/step | 10.5 samples/s +Step 47920/50462 | Loss: 2.4702 | LR: 3.22e-06 | 5687.3ms/step | 10.5 samples/s +Step 47930/50462 | Loss: 2.5081 | LR: 3.19e-06 | 5345.8ms/step | 11.2 samples/s +Step 47940/50462 | Loss: 2.4844 | LR: 3.17e-06 | 5369.7ms/step | 11.2 samples/s +Step 47950/50462 | Loss: 2.4910 | LR: 3.14e-06 | 5657.7ms/step | 10.6 samples/s +Step 47960/50462 | Loss: 2.4580 | LR: 3.12e-06 | 5665.9ms/step | 10.6 samples/s +Step 47970/50462 | Loss: 2.4945 | LR: 3.09e-06 | 5667.8ms/step | 10.6 samples/s +Step 47980/50462 | Loss: 2.4908 | LR: 3.07e-06 | 5665.7ms/step | 10.6 samples/s +Step 47990/50462 | Loss: 2.4739 | LR: 3.04e-06 | 5666.2ms/step | 10.6 samples/s +Step 48000/50462 | Loss: 2.4838 | LR: 3.02e-06 | 5664.4ms/step | 10.6 samples/s +Validation at step 48000: Loss = 2.5015, Perplexity = 12.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-48000 +Step 48010/50462 | Loss: 2.4975 | LR: 3.00e-06 | 9341.5ms/step | 6.4 samples/s +Step 48020/50462 | Loss: 2.5255 | LR: 2.97e-06 | 5653.6ms/step | 10.6 samples/s +Step 48030/50462 | Loss: 2.4675 | LR: 2.95e-06 | 5646.5ms/step | 10.6 samples/s +Step 48040/50462 | Loss: 2.4807 | LR: 2.92e-06 | 5648.5ms/step | 10.6 samples/s +Step 48050/50462 | Loss: 2.4737 | LR: 2.90e-06 | 5650.5ms/step | 10.6 samples/s +Step 48060/50462 | Loss: 2.4903 | LR: 2.87e-06 | 5650.0ms/step | 10.6 samples/s +Step 48070/50462 | Loss: 2.4937 | LR: 2.85e-06 | 5665.0ms/step | 10.6 samples/s +Step 48080/50462 | Loss: 2.5175 | LR: 2.83e-06 | 5660.6ms/step | 10.6 samples/s +Step 48090/50462 | Loss: 2.4930 | LR: 2.80e-06 | 5661.6ms/step | 10.6 samples/s +Step 48100/50462 | Loss: 2.4970 | LR: 2.78e-06 | 5646.2ms/step | 10.6 samples/s +Step 48110/50462 | Loss: 2.4860 | LR: 2.76e-06 | 5639.3ms/step | 10.6 samples/s +Step 48120/50462 | Loss: 2.4833 | LR: 2.73e-06 | 5639.2ms/step | 10.6 samples/s +Step 48130/50462 | Loss: 2.4847 | LR: 2.71e-06 | 5638.8ms/step | 10.6 samples/s +Step 48140/50462 | Loss: 2.4672 | LR: 2.69e-06 | 5127.0ms/step | 11.7 samples/s +Step 48150/50462 | Loss: 2.4800 | LR: 2.66e-06 | 5099.6ms/step | 11.8 samples/s +Step 48160/50462 | Loss: 2.4996 | LR: 2.64e-06 | 5495.0ms/step | 10.9 samples/s +Step 48170/50462 | Loss: 2.4797 | LR: 2.62e-06 | 5701.4ms/step | 10.5 samples/s +Step 48180/50462 | Loss: 2.4917 | LR: 2.60e-06 | 5705.0ms/step | 10.5 samples/s +Step 48190/50462 | Loss: 2.4879 | LR: 2.57e-06 | 5704.6ms/step | 10.5 samples/s +Step 48200/50462 | Loss: 2.4977 | LR: 2.55e-06 | 5708.9ms/step | 10.5 samples/s +Step 48210/50462 | Loss: 2.4734 | LR: 2.53e-06 | 5713.1ms/step | 10.5 samples/s +Step 48220/50462 | Loss: 2.4790 | LR: 2.51e-06 | 5715.8ms/step | 10.5 samples/s +Step 48230/50462 | Loss: 2.4869 | LR: 2.48e-06 | 5713.6ms/step | 10.5 samples/s +Step 48240/50462 | Loss: 2.4849 | LR: 2.46e-06 | 5709.9ms/step | 10.5 samples/s +Step 48250/50462 | Loss: 2.5240 | LR: 2.44e-06 | 5706.4ms/step | 10.5 samples/s +Validation at step 48250: Loss = 2.5015, Perplexity = 12.20 +Step 48260/50462 | Loss: 2.4607 | LR: 2.42e-06 | 8545.5ms/step | 7.0 samples/s +Step 48270/50462 | Loss: 2.4867 | LR: 2.39e-06 | 5704.8ms/step | 10.5 samples/s +Step 48280/50462 | Loss: 2.4948 | LR: 2.37e-06 | 5700.2ms/step | 10.5 samples/s +Step 48290/50462 | Loss: 2.4830 | LR: 2.35e-06 | 5705.0ms/step | 10.5 samples/s +Step 48300/50462 | Loss: 2.4762 | LR: 2.33e-06 | 5699.6ms/step | 10.5 samples/s +Step 48310/50462 | Loss: 2.4960 | LR: 2.31e-06 | 5700.9ms/step | 10.5 samples/s +Step 48320/50462 | Loss: 2.4953 | LR: 2.29e-06 | 5703.1ms/step | 10.5 samples/s +Step 48330/50462 | Loss: 2.4777 | LR: 2.27e-06 | 5693.6ms/step | 10.5 samples/s +Step 48340/50462 | Loss: 2.5013 | LR: 2.24e-06 | 5688.7ms/step | 10.5 samples/s +Step 48350/50462 | Loss: 2.5148 | LR: 2.22e-06 | 5681.0ms/step | 10.6 samples/s +Step 48360/50462 | Loss: 2.4711 | LR: 2.20e-06 | 5454.5ms/step | 11.0 samples/s +Step 48370/50462 | Loss: 2.5091 | LR: 2.18e-06 | 5427.7ms/step | 11.1 samples/s +Step 48380/50462 | Loss: 2.4751 | LR: 2.16e-06 | 5645.5ms/step | 10.6 samples/s +Step 48390/50462 | Loss: 2.5236 | LR: 2.14e-06 | 5642.3ms/step | 10.6 samples/s +Step 48400/50462 | Loss: 2.4778 | LR: 2.12e-06 | 5646.0ms/step | 10.6 samples/s +Step 48410/50462 | Loss: 2.5065 | LR: 2.10e-06 | 5645.3ms/step | 10.6 samples/s +Step 48420/50462 | Loss: 2.4821 | LR: 2.08e-06 | 5646.8ms/step | 10.6 samples/s +Step 48430/50462 | Loss: 2.4794 | LR: 2.06e-06 | 5639.2ms/step | 10.6 samples/s +Step 48440/50462 | Loss: 2.4968 | LR: 2.04e-06 | 5641.5ms/step | 10.6 samples/s +Step 48450/50462 | Loss: 2.4624 | LR: 2.02e-06 | 5643.6ms/step | 10.6 samples/s +Step 48460/50462 | Loss: 2.5101 | LR: 2.00e-06 | 5651.2ms/step | 10.6 samples/s +Step 48470/50462 | Loss: 2.4977 | LR: 1.98e-06 | 5639.0ms/step | 10.6 samples/s +Step 48480/50462 | Loss: 2.4813 | LR: 1.96e-06 | 5164.8ms/step | 11.6 samples/s +Step 48490/50462 | Loss: 2.4887 | LR: 1.94e-06 | 5298.3ms/step | 11.3 samples/s +Step 48500/50462 | Loss: 2.4577 | LR: 1.92e-06 | 5703.0ms/step | 10.5 samples/s +Validation at step 48500: Loss = 2.5015, Perplexity = 12.20 +Step 48510/50462 | Loss: 2.4916 | LR: 1.90e-06 | 8538.7ms/step | 7.0 samples/s +Step 48520/50462 | Loss: 2.4907 | LR: 1.88e-06 | 5725.8ms/step | 10.5 samples/s +Step 48530/50462 | Loss: 2.5081 | LR: 1.86e-06 | 5724.9ms/step | 10.5 samples/s +Step 48540/50462 | Loss: 2.4645 | LR: 1.84e-06 | 5721.4ms/step | 10.5 samples/s +Step 48550/50462 | Loss: 2.4737 | LR: 1.82e-06 | 5720.9ms/step | 10.5 samples/s +Step 48560/50462 | Loss: 2.4852 | LR: 1.80e-06 | 5703.2ms/step | 10.5 samples/s +Step 48570/50462 | Loss: 2.4871 | LR: 1.78e-06 | 5706.0ms/step | 10.5 samples/s +Step 48580/50462 | Loss: 2.4793 | LR: 1.77e-06 | 5713.2ms/step | 10.5 samples/s +Step 48590/50462 | Loss: 2.5040 | LR: 1.75e-06 | 5688.5ms/step | 10.5 samples/s +Step 48600/50462 | Loss: 2.4955 | LR: 1.73e-06 | 5348.5ms/step | 11.2 samples/s +Step 48610/50462 | Loss: 2.4806 | LR: 1.71e-06 | 5572.2ms/step | 10.8 samples/s +Step 48620/50462 | Loss: 2.4865 | LR: 1.69e-06 | 5638.8ms/step | 10.6 samples/s +Step 48630/50462 | Loss: 2.5088 | LR: 1.67e-06 | 5640.1ms/step | 10.6 samples/s +Step 48640/50462 | Loss: 2.4902 | LR: 1.66e-06 | 5644.0ms/step | 10.6 samples/s +Step 48650/50462 | Loss: 2.5064 | LR: 1.64e-06 | 5644.5ms/step | 10.6 samples/s +Step 48660/50462 | Loss: 2.5025 | LR: 1.62e-06 | 5645.3ms/step | 10.6 samples/s +Step 48670/50462 | Loss: 2.5037 | LR: 1.60e-06 | 5650.0ms/step | 10.6 samples/s +Step 48680/50462 | Loss: 2.4862 | LR: 1.58e-06 | 5650.4ms/step | 10.6 samples/s +Step 48690/50462 | Loss: 2.5003 | LR: 1.57e-06 | 5641.5ms/step | 10.6 samples/s +Step 48700/50462 | Loss: 2.4592 | LR: 1.55e-06 | 5223.0ms/step | 11.5 samples/s +Step 48710/50462 | Loss: 2.4828 | LR: 1.53e-06 | 5102.1ms/step | 11.8 samples/s +Step 48720/50462 | Loss: 2.5045 | LR: 1.51e-06 | 5146.4ms/step | 11.7 samples/s +Step 48730/50462 | Loss: 2.4898 | LR: 1.50e-06 | 5600.7ms/step | 10.7 samples/s +Step 48740/50462 | Loss: 2.4665 | LR: 1.48e-06 | 5614.5ms/step | 10.7 samples/s +Step 48750/50462 | Loss: 2.5109 | LR: 1.46e-06 | 5622.4ms/step | 10.7 samples/s +Validation at step 48750: Loss = 2.5015, Perplexity = 12.20 +Step 48760/50462 | Loss: 2.5217 | LR: 1.44e-06 | 8407.7ms/step | 7.1 samples/s +Step 48770/50462 | Loss: 2.4618 | LR: 1.43e-06 | 5616.7ms/step | 10.7 samples/s +Step 48780/50462 | Loss: 2.4841 | LR: 1.41e-06 | 5614.5ms/step | 10.7 samples/s +Step 48790/50462 | Loss: 2.4918 | LR: 1.39e-06 | 5610.3ms/step | 10.7 samples/s +Step 48800/50462 | Loss: 2.4718 | LR: 1.38e-06 | 5614.7ms/step | 10.7 samples/s +Step 48810/50462 | Loss: 2.4809 | LR: 1.36e-06 | 5619.9ms/step | 10.7 samples/s +Step 48820/50462 | Loss: 2.4863 | LR: 1.34e-06 | 5610.8ms/step | 10.7 samples/s +Step 48830/50462 | Loss: 2.5027 | LR: 1.33e-06 | 5613.9ms/step | 10.7 samples/s +Step 48840/50462 | Loss: 2.5154 | LR: 1.31e-06 | 5608.2ms/step | 10.7 samples/s +Step 48850/50462 | Loss: 2.4878 | LR: 1.30e-06 | 5611.7ms/step | 10.7 samples/s +Step 48860/50462 | Loss: 2.4534 | LR: 1.28e-06 | 5624.2ms/step | 10.7 samples/s +Step 48870/50462 | Loss: 2.4965 | LR: 1.26e-06 | 6057.8ms/step | 9.9 samples/s +Step 48880/50462 | Loss: 2.4785 | LR: 1.25e-06 | 5607.9ms/step | 10.7 samples/s +Step 48890/50462 | Loss: 2.5103 | LR: 1.23e-06 | 5618.4ms/step | 10.7 samples/s +Step 48900/50462 | Loss: 2.4881 | LR: 1.22e-06 | 5621.9ms/step | 10.7 samples/s +Step 48910/50462 | Loss: 2.5021 | LR: 1.20e-06 | 5622.1ms/step | 10.7 samples/s +Step 48920/50462 | Loss: 2.4878 | LR: 1.19e-06 | 5619.8ms/step | 10.7 samples/s +Step 48930/50462 | Loss: 2.4843 | LR: 1.17e-06 | 5622.1ms/step | 10.7 samples/s +Step 48940/50462 | Loss: 2.4647 | LR: 1.16e-06 | 5622.0ms/step | 10.7 samples/s +Step 48950/50462 | Loss: 2.4648 | LR: 1.14e-06 | 5625.1ms/step | 10.7 samples/s +Step 48960/50462 | Loss: 2.4965 | LR: 1.13e-06 | 5627.2ms/step | 10.7 samples/s +Step 48970/50462 | Loss: 2.4635 | LR: 1.11e-06 | 5621.9ms/step | 10.7 samples/s +Step 48980/50462 | Loss: 2.5320 | LR: 1.10e-06 | 5620.0ms/step | 10.7 samples/s +Step 48990/50462 | Loss: 2.5162 | LR: 1.08e-06 | 5621.1ms/step | 10.7 samples/s +Step 49000/50462 | Loss: 2.4976 | LR: 1.07e-06 | 5619.1ms/step | 10.7 samples/s +Validation at step 49000: Loss = 2.5015, Perplexity = 12.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-49000 +Step 49010/50462 | Loss: 2.4870 | LR: 1.05e-06 | 9310.1ms/step | 6.4 samples/s +Step 49020/50462 | Loss: 2.4945 | LR: 1.04e-06 | 5601.2ms/step | 10.7 samples/s +Step 49030/50462 | Loss: 2.4955 | LR: 1.02e-06 | 5606.5ms/step | 10.7 samples/s +Step 49040/50462 | Loss: 2.4924 | LR: 1.01e-06 | 5607.5ms/step | 10.7 samples/s +Step 49050/50462 | Loss: 2.4921 | LR: 9.95e-07 | 5606.0ms/step | 10.7 samples/s +Step 49060/50462 | Loss: 2.5118 | LR: 9.81e-07 | 5605.5ms/step | 10.7 samples/s +Step 49070/50462 | Loss: 2.5097 | LR: 9.67e-07 | 5603.4ms/step | 10.7 samples/s +Step 49080/50462 | Loss: 2.5115 | LR: 9.53e-07 | 5272.2ms/step | 11.4 samples/s +Step 49090/50462 | Loss: 2.4657 | LR: 9.39e-07 | 5127.2ms/step | 11.7 samples/s +Step 49100/50462 | Loss: 2.5140 | LR: 9.25e-07 | 5121.8ms/step | 11.7 samples/s +Step 49110/50462 | Loss: 2.4994 | LR: 9.12e-07 | 5122.6ms/step | 11.7 samples/s +Step 49120/50462 | Loss: 2.4688 | LR: 8.99e-07 | 5118.1ms/step | 11.7 samples/s +Step 49130/50462 | Loss: 2.4975 | LR: 8.85e-07 | 5132.4ms/step | 11.7 samples/s +Step 49140/50462 | Loss: 2.4699 | LR: 8.72e-07 | 5636.8ms/step | 10.6 samples/s +Step 49150/50462 | Loss: 2.5016 | LR: 8.59e-07 | 5728.7ms/step | 10.5 samples/s +Step 49160/50462 | Loss: 2.4905 | LR: 8.46e-07 | 5740.3ms/step | 10.5 samples/s +Step 49170/50462 | Loss: 2.4890 | LR: 8.33e-07 | 5754.5ms/step | 10.4 samples/s +Step 49180/50462 | Loss: 2.4787 | LR: 8.20e-07 | 5750.3ms/step | 10.4 samples/s +Step 49190/50462 | Loss: 2.4683 | LR: 8.07e-07 | 5743.6ms/step | 10.4 samples/s +Step 49200/50462 | Loss: 2.5213 | LR: 7.95e-07 | 5682.6ms/step | 10.6 samples/s +Step 49210/50462 | Loss: 2.4928 | LR: 7.82e-07 | 5654.7ms/step | 10.6 samples/s +Step 49220/50462 | Loss: 2.5022 | LR: 7.70e-07 | 5657.8ms/step | 10.6 samples/s +Step 49230/50462 | Loss: 2.5040 | LR: 7.57e-07 | 5640.5ms/step | 10.6 samples/s +Step 49240/50462 | Loss: 2.4944 | LR: 7.45e-07 | 5641.8ms/step | 10.6 samples/s +Step 49250/50462 | Loss: 2.4883 | LR: 7.33e-07 | 5658.4ms/step | 10.6 samples/s +Validation at step 49250: Loss = 2.5015, Perplexity = 12.20 +Step 49260/50462 | Loss: 2.4610 | LR: 7.21e-07 | 8504.1ms/step | 7.1 samples/s +Step 49270/50462 | Loss: 2.4804 | LR: 7.09e-07 | 5677.4ms/step | 10.6 samples/s +Step 49280/50462 | Loss: 2.4636 | LR: 6.97e-07 | 5674.8ms/step | 10.6 samples/s +Step 49290/50462 | Loss: 2.4852 | LR: 6.85e-07 | 5674.0ms/step | 10.6 samples/s +Step 49300/50462 | Loss: 2.4830 | LR: 6.74e-07 | 5677.5ms/step | 10.6 samples/s +Step 49310/50462 | Loss: 2.4888 | LR: 6.62e-07 | 5671.2ms/step | 10.6 samples/s +Step 49320/50462 | Loss: 2.4771 | LR: 6.51e-07 | 5670.6ms/step | 10.6 samples/s +Step 49330/50462 | Loss: 2.4864 | LR: 6.39e-07 | 5669.4ms/step | 10.6 samples/s +Step 49340/50462 | Loss: 2.4953 | LR: 6.28e-07 | 5670.7ms/step | 10.6 samples/s +Step 49350/50462 | Loss: 2.4986 | LR: 6.17e-07 | 5663.7ms/step | 10.6 samples/s +Step 49360/50462 | Loss: 2.5085 | LR: 6.06e-07 | 5666.2ms/step | 10.6 samples/s +Step 49370/50462 | Loss: 2.4910 | LR: 5.95e-07 | 5661.0ms/step | 10.6 samples/s +Step 49380/50462 | Loss: 2.5042 | LR: 5.84e-07 | 5663.4ms/step | 10.6 samples/s +Step 49390/50462 | Loss: 2.5037 | LR: 5.73e-07 | 5660.7ms/step | 10.6 samples/s +Step 49400/50462 | Loss: 2.5021 | LR: 5.63e-07 | 5655.9ms/step | 10.6 samples/s +Step 49410/50462 | Loss: 2.4924 | LR: 5.52e-07 | 5662.9ms/step | 10.6 samples/s +Step 49420/50462 | Loss: 2.4910 | LR: 5.42e-07 | 5657.8ms/step | 10.6 samples/s +Step 49430/50462 | Loss: 2.4707 | LR: 5.31e-07 | 5654.3ms/step | 10.6 samples/s +Step 49440/50462 | Loss: 2.4795 | LR: 5.21e-07 | 5654.8ms/step | 10.6 samples/s +Step 49450/50462 | Loss: 2.4668 | LR: 5.11e-07 | 5657.4ms/step | 10.6 samples/s +Step 49460/50462 | Loss: 2.5006 | LR: 5.01e-07 | 5655.6ms/step | 10.6 samples/s +Step 49470/50462 | Loss: 2.5024 | LR: 4.91e-07 | 5654.0ms/step | 10.6 samples/s +Step 49480/50462 | Loss: 2.4948 | LR: 4.81e-07 | 5652.1ms/step | 10.6 samples/s +Step 49490/50462 | Loss: 2.4765 | LR: 4.72e-07 | 5654.6ms/step | 10.6 samples/s +Step 49500/50462 | Loss: 2.4968 | LR: 4.62e-07 | 5645.3ms/step | 10.6 samples/s +Validation at step 49500: Loss = 2.5015, Perplexity = 12.20 +Step 49510/50462 | Loss: 2.5059 | LR: 4.52e-07 | 8449.5ms/step | 7.1 samples/s +Step 49520/50462 | Loss: 2.4864 | LR: 4.43e-07 | 5607.0ms/step | 10.7 samples/s +Step 49530/50462 | Loss: 2.4893 | LR: 4.34e-07 | 5205.9ms/step | 11.5 samples/s +Step 49540/50462 | Loss: 2.4816 | LR: 4.24e-07 | 5499.6ms/step | 10.9 samples/s +Step 49550/50462 | Loss: 2.4884 | LR: 4.15e-07 | 5671.2ms/step | 10.6 samples/s +Step 49560/50462 | Loss: 2.5037 | LR: 4.06e-07 | 5674.3ms/step | 10.6 samples/s +Step 49570/50462 | Loss: 2.5055 | LR: 3.97e-07 | 6092.5ms/step | 9.8 samples/s +Step 49580/50462 | Loss: 2.5054 | LR: 3.88e-07 | 6152.4ms/step | 9.8 samples/s +Step 49590/50462 | Loss: 2.4790 | LR: 3.79e-07 | 5655.5ms/step | 10.6 samples/s +Step 49600/50462 | Loss: 2.5097 | LR: 3.71e-07 | 5653.0ms/step | 10.6 samples/s +Step 49610/50462 | Loss: 2.4896 | LR: 3.62e-07 | 5656.7ms/step | 10.6 samples/s +Step 49620/50462 | Loss: 2.4884 | LR: 3.54e-07 | 5665.3ms/step | 10.6 samples/s +Step 49630/50462 | Loss: 2.4891 | LR: 3.45e-07 | 5667.6ms/step | 10.6 samples/s +Step 49640/50462 | Loss: 2.4916 | LR: 3.37e-07 | 5657.7ms/step | 10.6 samples/s +Step 49650/50462 | Loss: 2.4957 | LR: 3.29e-07 | 5667.3ms/step | 10.6 samples/s +Step 49660/50462 | Loss: 2.4780 | LR: 3.21e-07 | 5672.5ms/step | 10.6 samples/s +Step 49670/50462 | Loss: 2.4650 | LR: 3.13e-07 | 5670.2ms/step | 10.6 samples/s +Step 49680/50462 | Loss: 2.5101 | LR: 3.05e-07 | 5665.3ms/step | 10.6 samples/s +Step 49690/50462 | Loss: 2.5279 | LR: 2.97e-07 | 5665.6ms/step | 10.6 samples/s +Step 49700/50462 | Loss: 2.4991 | LR: 2.90e-07 | 5658.8ms/step | 10.6 samples/s +Step 49710/50462 | Loss: 2.4927 | LR: 2.82e-07 | 5665.6ms/step | 10.6 samples/s +Step 49720/50462 | Loss: 2.4828 | LR: 2.75e-07 | 5670.1ms/step | 10.6 samples/s +Step 49730/50462 | Loss: 2.4999 | LR: 2.67e-07 | 5671.2ms/step | 10.6 samples/s +Step 49740/50462 | Loss: 2.5200 | LR: 2.60e-07 | 5672.5ms/step | 10.6 samples/s +Step 49750/50462 | Loss: 2.4917 | LR: 2.53e-07 | 5671.2ms/step | 10.6 samples/s +Validation at step 49750: Loss = 2.5015, Perplexity = 12.20 +Step 49760/50462 | Loss: 2.5092 | LR: 2.46e-07 | 8493.8ms/step | 7.1 samples/s +Step 49770/50462 | Loss: 2.4984 | LR: 2.39e-07 | 5665.5ms/step | 10.6 samples/s +Step 49780/50462 | Loss: 2.4824 | LR: 2.32e-07 | 5664.1ms/step | 10.6 samples/s +Step 49790/50462 | Loss: 2.4758 | LR: 2.25e-07 | 5659.4ms/step | 10.6 samples/s +Step 49800/50462 | Loss: 2.5190 | LR: 2.19e-07 | 5649.4ms/step | 10.6 samples/s +Step 49810/50462 | Loss: 2.5053 | LR: 2.12e-07 | 5649.7ms/step | 10.6 samples/s +Step 49820/50462 | Loss: 2.5029 | LR: 2.06e-07 | 5652.5ms/step | 10.6 samples/s +Step 49830/50462 | Loss: 2.4914 | LR: 1.99e-07 | 5646.7ms/step | 10.6 samples/s +Step 49840/50462 | Loss: 2.5069 | LR: 1.93e-07 | 5658.8ms/step | 10.6 samples/s +Step 49850/50462 | Loss: 2.4829 | LR: 1.87e-07 | 5652.3ms/step | 10.6 samples/s +Step 49860/50462 | Loss: 2.4838 | LR: 1.81e-07 | 5649.8ms/step | 10.6 samples/s +Step 49870/50462 | Loss: 2.4968 | LR: 1.75e-07 | 5659.7ms/step | 10.6 samples/s +Step 49880/50462 | Loss: 2.4828 | LR: 1.69e-07 | 5650.9ms/step | 10.6 samples/s +Step 49890/50462 | Loss: 2.4924 | LR: 1.63e-07 | 5650.3ms/step | 10.6 samples/s +Step 49900/50462 | Loss: 2.5048 | LR: 1.58e-07 | 5651.1ms/step | 10.6 samples/s +Step 49910/50462 | Loss: 2.4907 | LR: 1.52e-07 | 5651.0ms/step | 10.6 samples/s +Step 49920/50462 | Loss: 2.5056 | LR: 1.47e-07 | 5648.4ms/step | 10.6 samples/s +Step 49930/50462 | Loss: 2.4766 | LR: 1.41e-07 | 5649.0ms/step | 10.6 samples/s +Step 49940/50462 | Loss: 2.4882 | LR: 1.36e-07 | 5649.7ms/step | 10.6 samples/s +Step 49950/50462 | Loss: 2.4505 | LR: 1.31e-07 | 5650.6ms/step | 10.6 samples/s +Step 49960/50462 | Loss: 2.4788 | LR: 1.26e-07 | 5647.9ms/step | 10.6 samples/s +Step 49970/50462 | Loss: 2.4707 | LR: 1.21e-07 | 5651.0ms/step | 10.6 samples/s +Step 49980/50462 | Loss: 2.4855 | LR: 1.16e-07 | 5648.6ms/step | 10.6 samples/s +Step 49990/50462 | Loss: 2.4633 | LR: 1.11e-07 | 5647.9ms/step | 10.6 samples/s +Step 50000/50462 | Loss: 2.4744 | LR: 1.07e-07 | 5650.6ms/step | 10.6 samples/s +Validation at step 50000: Loss = 2.5015, Perplexity = 12.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-50000 +Step 50010/50462 | Loss: 2.4701 | LR: 1.02e-07 | 9386.2ms/step | 6.4 samples/s +Step 50020/50462 | Loss: 2.4772 | LR: 9.75e-08 | 5672.5ms/step | 10.6 samples/s +Step 50030/50462 | Loss: 2.5076 | LR: 9.32e-08 | 5675.5ms/step | 10.6 samples/s +Step 50040/50462 | Loss: 2.5382 | LR: 8.89e-08 | 5678.2ms/step | 10.6 samples/s +Step 50050/50462 | Loss: 2.4927 | LR: 8.47e-08 | 5676.3ms/step | 10.6 samples/s +Step 50060/50462 | Loss: 2.4848 | LR: 8.07e-08 | 5669.5ms/step | 10.6 samples/s +Step 50070/50462 | Loss: 2.5052 | LR: 7.67e-08 | 5673.9ms/step | 10.6 samples/s +Step 50080/50462 | Loss: 2.4757 | LR: 7.28e-08 | 5670.2ms/step | 10.6 samples/s +Step 50090/50462 | Loss: 2.4692 | LR: 6.91e-08 | 5671.2ms/step | 10.6 samples/s +Step 50100/50462 | Loss: 2.4756 | LR: 6.54e-08 | 5674.6ms/step | 10.6 samples/s +Step 50110/50462 | Loss: 2.5144 | LR: 6.19e-08 | 5675.0ms/step | 10.6 samples/s +Step 50120/50462 | Loss: 2.4954 | LR: 5.84e-08 | 5671.1ms/step | 10.6 samples/s +Step 50130/50462 | Loss: 2.4966 | LR: 5.50e-08 | 5675.4ms/step | 10.6 samples/s +Step 50140/50462 | Loss: 2.5147 | LR: 5.18e-08 | 5673.5ms/step | 10.6 samples/s +Step 50150/50462 | Loss: 2.4701 | LR: 4.86e-08 | 5673.6ms/step | 10.6 samples/s +Step 50160/50462 | Loss: 2.4862 | LR: 4.55e-08 | 5673.2ms/step | 10.6 samples/s +Step 50170/50462 | Loss: 2.4532 | LR: 4.26e-08 | 5677.9ms/step | 10.6 samples/s +Step 50180/50462 | Loss: 2.4586 | LR: 3.97e-08 | 5672.9ms/step | 10.6 samples/s +Step 50190/50462 | Loss: 2.5053 | LR: 3.69e-08 | 5676.1ms/step | 10.6 samples/s +Step 50200/50462 | Loss: 2.4741 | LR: 3.43e-08 | 5672.8ms/step | 10.6 samples/s +Step 50210/50462 | Loss: 2.4997 | LR: 3.17e-08 | 5673.3ms/step | 10.6 samples/s +Step 50220/50462 | Loss: 2.4965 | LR: 2.92e-08 | 5674.1ms/step | 10.6 samples/s +Step 50230/50462 | Loss: 2.5137 | LR: 2.69e-08 | 5672.9ms/step | 10.6 samples/s +Step 50240/50462 | Loss: 2.4783 | LR: 2.46e-08 | 5673.2ms/step | 10.6 samples/s +Step 50250/50462 | Loss: 2.4720 | LR: 2.24e-08 | 5675.4ms/step | 10.6 samples/s +Validation at step 50250: Loss = 2.5015, Perplexity = 12.20 +Step 50260/50462 | Loss: 2.5186 | LR: 2.04e-08 | 8495.4ms/step | 7.1 samples/s +Step 50270/50462 | Loss: 2.5304 | LR: 1.84e-08 | 5670.7ms/step | 10.6 samples/s +Step 50280/50462 | Loss: 2.5018 | LR: 1.65e-08 | 5671.2ms/step | 10.6 samples/s +Step 50290/50462 | Loss: 2.5151 | LR: 1.48e-08 | 5668.7ms/step | 10.6 samples/s +Step 50300/50462 | Loss: 2.4700 | LR: 1.31e-08 | 5671.2ms/step | 10.6 samples/s +Step 50310/50462 | Loss: 2.4659 | LR: 1.15e-08 | 5669.1ms/step | 10.6 samples/s +Step 50320/50462 | Loss: 2.4855 | LR: 1.01e-08 | 5668.8ms/step | 10.6 samples/s +Step 50330/50462 | Loss: 2.5131 | LR: 8.70e-09 | 5668.9ms/step | 10.6 samples/s +Step 50340/50462 | Loss: 2.4940 | LR: 7.43e-09 | 5669.1ms/step | 10.6 samples/s +Step 50350/50462 | Loss: 2.4750 | LR: 6.26e-09 | 5667.3ms/step | 10.6 samples/s +Step 50360/50462 | Loss: 2.4990 | LR: 5.19e-09 | 5669.4ms/step | 10.6 samples/s +Step 50370/50462 | Loss: 2.4742 | LR: 4.23e-09 | 5670.2ms/step | 10.6 samples/s +Step 50380/50462 | Loss: 2.4661 | LR: 3.36e-09 | 5667.9ms/step | 10.6 samples/s +Step 50390/50462 | Loss: 2.5061 | LR: 2.59e-09 | 5671.2ms/step | 10.6 samples/s +Step 50400/50462 | Loss: 2.4896 | LR: 1.92e-09 | 5671.0ms/step | 10.6 samples/s +Step 50410/50462 | Loss: 2.4787 | LR: 1.35e-09 | 5673.9ms/step | 10.6 samples/s +Step 50420/50462 | Loss: 2.5067 | LR: 8.81e-10 | 5670.1ms/step | 10.6 samples/s +Step 50430/50462 | Loss: 2.4976 | LR: 5.11e-10 | 5671.9ms/step | 10.6 samples/s +Step 50440/50462 | Loss: 2.4672 | LR: 2.42e-10 | 5674.3ms/step | 10.6 samples/s +Step 50450/50462 | Loss: 2.5086 | LR: 7.19e-11 | 5667.3ms/step | 10.6 samples/s +Step 50460/50462 | Loss: 2.4806 | LR: 2.00e-12 | 5670.8ms/step | 10.6 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260605_220251/checkpoint-50462 +wandb: updating run metadata +wandb: uploading wandb-summary.json; uploading config.yaml; uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▇▆▆▆▅▅▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ██▇▆▆▅▄▄▄▄▃▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▂▁▁▁▁▂▁▂▁▁▁▁▂▇▇▇▇▇▇▂▇▇▇▇▇▇████▁▆▇▇██▆▇▇▇ +wandb: speed/samples_per_sec █████████▆▆▆▆▆▆▆▆▁▆▆█▆▆█▆▆▆▆▆▆▆██▅▆▆▆▆▆▆ +wandb: speed/steps_per_sec ███████████▂███████▁▂▆▆▆▆▇▆▆▆▆▆▆▇▆▆███▆▆ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.50149 +wandb: eval/perplexity 12.20063 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 5670.76755 +wandb: speed/samples_per_sec 10.58058 +wandb: speed/steps_per_sec 0.17634 +wandb: +4 ... +wandb: +wandb: 🚀 View run dense continued 10k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ns4ua5fr +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260606_174407-ns4ua5fr/logs +Training complete! diff --git a/epfl_final/log_files/cont_pretrain_16k.log b/epfl_final/log_files/cont_pretrain_16k.log new file mode 100644 index 0000000000000000000000000000000000000000..3be437076d08c274512a3472a944bb82907208b9 --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_16k.log @@ -0,0 +1,296 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000", + "num_epochs": 1, + "max_steps": 20005, + "per_device_batch_size": 3, + "gradient_accumulation_steps": 8, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 250, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 200, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense-resume-18k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: W&B API key is configured. Use `wandb login --relogin` to force relogin +wandb: setting up run qudv4nmq +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260513_151725-qudv4nmq +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense-resume-18k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qudv4nmq + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1024 + num_hidden_layers: 28 + num_attention_heads: 16 + num_key_value_heads: 8 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 596,049,920 + Trainable parameters: 596,049,920 + Non-trainable parameters: 0 + Model size (BF16): 1.19 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [3, 8192] -- -- True +├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True +│ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True +├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True +================================================================================================================================================================ +Total params: 751,632,384 +Trainable params: 751,632,384 +Non-trainable params: 0 +Total mult-adds (G): 2.25 +================================================================================================================================================================ +Input size (MB): 0.20 +Forward/backward pass size (MB): 63052.97 +Params size (MB): 1503.26 +Estimated Total Size (MB): 64556.43 +================================================================================================================================================================ +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003) +Dataset split: 1017602 train, 254401 validation +Training examples (packed, train split): 1017602 +Validation examples (packed, val split): 254401 +Per-device batch size: 3 +World size (GPUs): 1 +Gradient accumulation steps: 8 +Effective batch size (seqs/step): 24 +Sequence length: 8192 +Tokens per optimizer step: 196,608 +Micro-batches per epoch (per rank): 339200 +Optimizer steps per epoch: 42400 +Total optimizer steps: 20005 (capped by max_steps; 0.47 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000... +Warning: no RNG state file found for rank 0; RNG will not be restored. +Resumed from checkpoint at step 15999 (epoch 0, micro_step 0) +Resuming training from step 15999 +Starting training... +Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run. +Step 16000/20005 | Loss: 0.2859 | LR: 4.90e-05 | 61450.0ms/step | 0.4 samples/s +Validation at step 16000: Loss = 2.9095, Perplexity = 18.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000 +Step 16010/20005 | Loss: 2.8805 | LR: 4.88e-05 | 42322.3ms/step | 0.6 samples/s +Step 16020/20005 | Loss: 2.9052 | LR: 4.85e-05 | 5797.6ms/step | 4.1 samples/s +Step 16030/20005 | Loss: 2.8774 | LR: 4.83e-05 | 5801.3ms/step | 4.1 samples/s +Step 16040/20005 | Loss: 2.8951 | LR: 4.80e-05 | 5783.8ms/step | 4.1 samples/s +Step 16050/20005 | Loss: 2.8999 | LR: 4.78e-05 | 5774.8ms/step | 4.2 samples/s +Step 16060/20005 | Loss: 2.8975 | LR: 4.76e-05 | 5774.1ms/step | 4.2 samples/s +Step 16070/20005 | Loss: 2.9103 | LR: 4.73e-05 | 5786.3ms/step | 4.1 samples/s +Step 16080/20005 | Loss: 2.8558 | LR: 4.71e-05 | 5793.9ms/step | 4.1 samples/s +Step 16090/20005 | Loss: 2.8936 | LR: 4.69e-05 | 5796.0ms/step | 4.1 samples/s +Step 16100/20005 | Loss: 2.8871 | LR: 4.67e-05 | 5782.7ms/step | 4.2 samples/s +Step 16110/20005 | Loss: 2.8781 | LR: 4.64e-05 | 5777.7ms/step | 4.2 samples/s +Step 16120/20005 | Loss: 2.8904 | LR: 4.62e-05 | 5775.9ms/step | 4.2 samples/s +Step 16130/20005 | Loss: 2.8697 | LR: 4.60e-05 | 5791.7ms/step | 4.1 samples/s +Step 16140/20005 | Loss: 2.8868 | LR: 4.57e-05 | 5801.7ms/step | 4.1 samples/s +Step 16150/20005 | Loss: 2.8816 | LR: 4.55e-05 | 5800.1ms/step | 4.1 samples/s +Step 16160/20005 | Loss: 2.8591 | LR: 4.53e-05 | 5784.0ms/step | 4.1 samples/s +Step 16170/20005 | Loss: 2.8515 | LR: 4.50e-05 | 5782.4ms/step | 4.2 samples/s +Step 16180/20005 | Loss: 2.8859 | LR: 4.48e-05 | 5784.0ms/step | 4.1 samples/s +Step 16190/20005 | Loss: 2.8966 | LR: 4.46e-05 | 5787.5ms/step | 4.1 samples/s +Step 16200/20005 | Loss: 2.8766 | LR: 4.44e-05 | 5802.8ms/step | 4.1 samples/s +Step 16210/20005 | Loss: 2.9043 | LR: 4.41e-05 | 5805.9ms/step | 4.1 samples/s +Step 16220/20005 | Loss: 2.8939 | LR: 4.39e-05 | 5783.9ms/step | 4.1 samples/s +Step 16230/20005 | Loss: 2.8999 | LR: 4.37e-05 | 5777.6ms/step | 4.2 samples/s +Step 16240/20005 | Loss: 2.8962 | LR: 4.35e-05 | 5775.0ms/step | 4.2 samples/s +Step 16250/20005 | Loss: 2.8544 | LR: 4.32e-05 | 5791.8ms/step | 4.1 samples/s +Validation at step 16250: Loss = 2.8974, Perplexity = 18.13 +Step 16260/20005 | Loss: 2.8719 | LR: 4.30e-05 | 39967.3ms/step | 0.6 samples/s +Step 16270/20005 | Loss: 2.8932 | LR: 4.28e-05 | 5794.1ms/step | 4.1 samples/s +Step 16280/20005 | Loss: 2.8991 | LR: 4.26e-05 | 5778.4ms/step | 4.2 samples/s +Step 16290/20005 | Loss: 2.8963 | LR: 4.24e-05 | 5782.1ms/step | 4.2 samples/s +Step 16300/20005 | Loss: 2.8744 | LR: 4.21e-05 | 5779.4ms/step | 4.2 samples/s +Step 16310/20005 | Loss: 2.8959 | LR: 4.19e-05 | 5794.2ms/step | 4.1 samples/s +Step 16320/20005 | Loss: 2.8922 | LR: 4.17e-05 | 5805.9ms/step | 4.1 samples/s +Step 16330/20005 | Loss: 2.8783 | LR: 4.15e-05 | 5792.6ms/step | 4.1 samples/s +Step 16340/20005 | Loss: 2.8571 | LR: 4.13e-05 | 5780.2ms/step | 4.2 samples/s +Step 16350/20005 | Loss: 2.8927 | LR: 4.10e-05 | 5782.3ms/step | 4.2 samples/s +Step 16360/20005 | Loss: 2.8868 | LR: 4.08e-05 | 5774.0ms/step | 4.2 samples/s +Step 16370/20005 | Loss: 2.8835 | LR: 4.06e-05 | 5793.8ms/step | 4.1 samples/s +Step 16380/20005 | Loss: 2.8890 | LR: 4.04e-05 | 5805.2ms/step | 4.1 samples/s +Step 16390/20005 | Loss: 2.8688 | LR: 4.02e-05 | 5792.8ms/step | 4.1 samples/s +Step 16400/20005 | Loss: 2.8665 | LR: 3.99e-05 | 5775.8ms/step | 4.2 samples/s +Step 16410/20005 | Loss: 2.9046 | LR: 3.97e-05 | 5798.5ms/step | 4.1 samples/s +Step 16420/20005 | Loss: 2.8618 | LR: 3.95e-05 | 5777.6ms/step | 4.2 samples/s +Step 16430/20005 | Loss: 2.8637 | LR: 3.93e-05 | 5798.4ms/step | 4.1 samples/s +Step 16440/20005 | Loss: 2.8608 | LR: 3.91e-05 | 5808.3ms/step | 4.1 samples/s +Step 16450/20005 | Loss: 2.8862 | LR: 3.89e-05 | 5787.8ms/step | 4.1 samples/s +Step 16460/20005 | Loss: 2.8885 | LR: 3.87e-05 | 5775.7ms/step | 4.2 samples/s +Step 16470/20005 | Loss: 2.8833 | LR: 3.85e-05 | 5771.0ms/step | 4.2 samples/s +Step 16480/20005 | Loss: 2.8654 | LR: 3.82e-05 | 5777.6ms/step | 4.2 samples/s +Step 16490/20005 | Loss: 2.8883 | LR: 3.80e-05 | 5805.9ms/step | 4.1 samples/s +Step 16500/20005 | Loss: 2.8750 | LR: 3.78e-05 | 5847.9ms/step | 4.1 samples/s +Validation at step 16500: Loss = 2.8964, Perplexity = 18.11 +Step 16510/20005 | Loss: 2.8852 | LR: 3.76e-05 | 39988.2ms/step | 0.6 samples/s +Step 16520/20005 | Loss: 2.8601 | LR: 3.74e-05 | 5779.8ms/step | 4.2 samples/s +Step 16530/20005 | Loss: 2.8606 | LR: 3.72e-05 | 5778.4ms/step | 4.2 samples/s +Step 16540/20005 | Loss: 2.8567 | LR: 3.70e-05 | 5788.9ms/step | 4.1 samples/s +Step 16550/20005 | Loss: 2.9279 | LR: 3.68e-05 | 5799.2ms/step | 4.1 samples/s +Step 16560/20005 | Loss: 2.8896 | LR: 3.66e-05 | 5806.0ms/step | 4.1 samples/s +Step 16570/20005 | Loss: 2.8944 | LR: 3.64e-05 | 5789.4ms/step | 4.1 samples/s +Step 16580/20005 | Loss: 2.8497 | LR: 3.62e-05 | 5778.2ms/step | 4.2 samples/s +Step 16590/20005 | Loss: 2.8682 | LR: 3.59e-05 | 5787.9ms/step | 4.1 samples/s +Step 16600/20005 | Loss: 2.8877 | LR: 3.57e-05 | 5789.2ms/step | 4.1 samples/s +Step 16610/20005 | Loss: 2.8810 | LR: 3.55e-05 | 5805.3ms/step | 4.1 samples/s +Step 16620/20005 | Loss: 2.8817 | LR: 3.53e-05 | 5806.6ms/step | 4.1 samples/s +Step 16630/20005 | Loss: 2.8868 | LR: 3.51e-05 | 5785.3ms/step | 4.1 samples/s +Step 16640/20005 | Loss: 2.9033 | LR: 3.49e-05 | 5778.0ms/step | 4.2 samples/s +Step 16650/20005 | Loss: 2.9087 | LR: 3.47e-05 | 5774.1ms/step | 4.2 samples/s +Step 16660/20005 | Loss: 2.8753 | LR: 3.45e-05 | 5783.8ms/step | 4.1 samples/s +Step 16670/20005 | Loss: 2.8726 | LR: 3.43e-05 | 5806.0ms/step | 4.1 samples/s +Step 16680/20005 | Loss: 2.8741 | LR: 3.41e-05 | 5808.2ms/step | 4.1 samples/s +Step 16690/20005 | Loss: 2.8671 | LR: 3.39e-05 | 5792.3ms/step | 4.1 samples/s +Step 16700/20005 | Loss: 2.8762 | LR: 3.37e-05 | 5802.2ms/step | 4.1 samples/s +Step 16710/20005 | Loss: 2.8609 | LR: 3.35e-05 | 5801.5ms/step | 4.1 samples/s +Step 16720/20005 | Loss: 2.8764 | LR: 3.33e-05 | 5798.4ms/step | 4.1 samples/s +Step 16730/20005 | Loss: 2.8537 | LR: 3.31e-05 | 5804.1ms/step | 4.1 samples/s +Step 16740/20005 | Loss: 2.8709 | LR: 3.29e-05 | 5801.8ms/step | 4.1 samples/s +Step 16750/20005 | Loss: 2.8625 | LR: 3.27e-05 | 5785.3ms/step | 4.1 samples/s +Validation at step 16750: Loss = 2.8958, Perplexity = 18.10 +Step 16760/20005 | Loss: 2.8645 | LR: 3.25e-05 | 40056.8ms/step | 0.6 samples/s +Step 16770/20005 | Loss: 2.8947 | LR: 3.23e-05 | 5803.4ms/step | 4.1 samples/s +Step 16780/20005 | Loss: 2.8879 | LR: 3.21e-05 | 5797.2ms/step | 4.1 samples/s +Step 16790/20005 | Loss: 2.8779 | LR: 3.20e-05 | 5802.3ms/step | 4.1 samples/s +Step 16800/20005 | Loss: 2.8738 | LR: 3.18e-05 | 5812.2ms/step | 4.1 samples/s +Step 16810/20005 | Loss: 2.8759 | LR: 3.16e-05 | 5789.0ms/step | 4.1 samples/s +Step 16820/20005 | Loss: 2.8653 | LR: 3.14e-05 | 5787.6ms/step | 4.1 samples/s +Step 16830/20005 | Loss: 2.8641 | LR: 3.12e-05 | 5796.3ms/step | 4.1 samples/s +Step 16840/20005 | Loss: 2.8667 | LR: 3.10e-05 | 5800.3ms/step | 4.1 samples/s +Step 16850/20005 | Loss: 2.8774 | LR: 3.08e-05 | 5800.0ms/step | 4.1 samples/s +Step 16860/20005 | Loss: 2.8557 | LR: 3.06e-05 | 5810.7ms/step | 4.1 samples/s +Step 16870/20005 | Loss: 2.8982 | LR: 3.04e-05 | 5787.4ms/step | 4.1 samples/s +Step 16880/20005 | Loss: 2.8782 | LR: 3.02e-05 | 5794.7ms/step | 4.1 samples/s +Step 16890/20005 | Loss: 2.8829 | LR: 3.00e-05 | 5801.1ms/step | 4.1 samples/s +Step 16900/20005 | Loss: 2.8573 | LR: 2.98e-05 | 5800.6ms/step | 4.1 samples/s +Step 16910/20005 | Loss: 2.8912 | LR: 2.97e-05 | 5796.6ms/step | 4.1 samples/s +Step 16920/20005 | Loss: 2.8813 | LR: 2.95e-05 | 5808.5ms/step | 4.1 samples/s +Step 16930/20005 | Loss: 2.8525 | LR: 2.93e-05 | 5787.2ms/step | 4.1 samples/s +Step 16940/20005 | Loss: 2.8718 | LR: 2.91e-05 | 5777.5ms/step | 4.2 samples/s +Step 16950/20005 | Loss: 2.8829 | LR: 2.89e-05 | 5788.7ms/step | 4.1 samples/s +Step 16960/20005 | Loss: 2.8935 | LR: 2.87e-05 | 5801.4ms/step | 4.1 samples/s +Step 16970/20005 | Loss: 2.9022 | LR: 2.85e-05 | 5795.8ms/step | 4.1 samples/s +Step 16980/20005 | Loss: 2.8955 | LR: 2.84e-05 | 5799.3ms/step | 4.1 samples/s +Step 16990/20005 | Loss: 2.8606 | LR: 2.82e-05 | 5796.4ms/step | 4.1 samples/s +Step 17000/20005 | Loss: 2.8556 | LR: 2.80e-05 | 5822.8ms/step | 4.1 samples/s +Validation at step 17000: Loss = 2.8954, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-17000 +Step 17010/20005 | Loss: 2.8932 | LR: 2.78e-05 | 40472.1ms/step | 0.6 samples/s +Step 17020/20005 | Loss: 2.8715 | LR: 2.76e-05 | 5795.8ms/step | 4.1 samples/s +Step 17030/20005 | Loss: 2.8630 | LR: 2.74e-05 | 5784.2ms/step | 4.1 samples/s +Step 17040/20005 | Loss: 2.8649 | LR: 2.73e-05 | 5804.2ms/step | 4.1 samples/s +Step 17050/20005 | Loss: 2.8631 | LR: 2.71e-05 | 5803.9ms/step | 4.1 samples/s +Step 17060/20005 | Loss: 2.8951 | LR: 2.69e-05 | 5785.9ms/step | 4.1 samples/s +Step 17070/20005 | Loss: 2.8709 | LR: 2.67e-05 | 5773.8ms/step | 4.2 samples/s +Step 17080/20005 | Loss: 2.9004 | LR: 2.65e-05 | 5774.1ms/step | 4.2 samples/s +Step 17090/20005 | Loss: 2.8492 | LR: 2.64e-05 | 5778.0ms/step | 4.2 samples/s +Step 17100/20005 | Loss: 2.8993 | LR: 2.62e-05 | 5799.8ms/step | 4.1 samples/s +Step 17110/20005 | Loss: 2.9000 | LR: 2.60e-05 | 5806.2ms/step | 4.1 samples/s +Step 17120/20005 | Loss: 2.8823 | LR: 2.58e-05 | 5780.5ms/step | 4.2 samples/s +Step 17130/20005 | Loss: 2.8930 | LR: 2.57e-05 | 5777.5ms/step | 4.2 samples/s +Step 17140/20005 | Loss: 2.8838 | LR: 2.55e-05 | 5772.2ms/step | 4.2 samples/s +Step 17150/20005 | Loss: 2.8710 | LR: 2.53e-05 | 5776.7ms/step | 4.2 samples/s +Step 17160/20005 | Loss: 2.8906 | LR: 2.51e-05 | 5799.7ms/step | 4.1 samples/s +Step 17170/20005 | Loss: 2.8886 | LR: 2.50e-05 | 5802.9ms/step | 4.1 samples/s +Step 17180/20005 | Loss: 2.8920 | LR: 2.48e-05 | 5786.9ms/step | 4.1 samples/s +Step 17190/20005 | Loss: 2.8665 | LR: 2.46e-05 | 5775.9ms/step | 4.2 samples/s +Step 17200/20005 | Loss: 2.8719 | LR: 2.44e-05 | 5776.3ms/step | 4.2 samples/s +Step 17210/20005 | Loss: 2.8837 | LR: 2.43e-05 | 5777.9ms/step | 4.2 samples/s +Step 17220/20005 | Loss: 2.8843 | LR: 2.41e-05 | 5796.4ms/step | 4.1 samples/s +Step 17230/20005 | Loss: 2.8992 | LR: 2.39e-05 | 5804.3ms/step | 4.1 samples/s +Step 17240/20005 | Loss: 2.8704 | LR: 2.38e-05 | 5788.7ms/step | 4.1 samples/s +Step 17250/20005 | Loss: 2.8868 | LR: 2.36e-05 | 5776.6ms/step | 4.2 samples/s +Validation at step 17250: Loss = 2.8951, Perplexity = 18.08 diff --git a/epfl_final/log_files/cont_pretrain_19k.log b/epfl_final/log_files/cont_pretrain_19k.log new file mode 100644 index 0000000000000000000000000000000000000000..bcc9a87e04fb9d2b2c4b522d1231ac1a6edcac78 --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_19k.log @@ -0,0 +1,302 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000", + "num_epochs": 1, + "max_steps": 20005, + "per_device_batch_size": 3, + "gradient_accumulation_steps": 8, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 250, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 200, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense-resume-18k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260513_052440-i0f0xwcq +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense-resume-18k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i0f0xwcq + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1024 + num_hidden_layers: 28 + num_attention_heads: 16 + num_key_value_heads: 8 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 596,049,920 + Trainable parameters: 596,049,920 + Non-trainable parameters: 0 + Model size (BF16): 1.19 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [3, 8192] -- -- True +├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True +│ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True +├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True +================================================================================================================================================================ +Total params: 751,632,384 +Trainable params: 751,632,384 +Non-trainable params: 0 +Total mult-adds (G): 2.25 +================================================================================================================================================================ +Input size (MB): 0.20 +Forward/backward pass size (MB): 63052.97 +Params size (MB): 1503.26 +Estimated Total Size (MB): 64556.43 +================================================================================================================================================================ +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003) +Dataset split: 1017602 train, 254401 validation +Training examples (packed, train split): 1017602 +Validation examples (packed, val split): 254401 +Per-device batch size: 3 +World size (GPUs): 1 +Gradient accumulation steps: 8 +Effective batch size (seqs/step): 24 +Sequence length: 8192 +Tokens per optimizer step: 196,608 +Micro-batches per epoch (per rank): 339200 +Optimizer steps per epoch: 42400 +Total optimizer steps: 20005 (capped by max_steps; 0.47 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000... +Warning: no RNG state file found for rank 0; RNG will not be restored. +Resumed from checkpoint at step 18999 (epoch 0, micro_step 0) +Resuming training from step 18999 +Starting training... +Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run. +Step 19000/20005 | Loss: 0.2856 | LR: 3.18e-06 | 64856.9ms/step | 0.4 samples/s +Validation at step 19000: Loss = 2.9082, Perplexity = 18.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000 +Step 19010/20005 | Loss: 2.8841 | LR: 3.12e-06 | 43506.1ms/step | 0.6 samples/s +Step 19020/20005 | Loss: 2.9121 | LR: 3.05e-06 | 5798.6ms/step | 4.1 samples/s +Step 19030/20005 | Loss: 2.8848 | LR: 2.99e-06 | 5799.2ms/step | 4.1 samples/s +Step 19040/20005 | Loss: 2.9030 | LR: 2.93e-06 | 5796.6ms/step | 4.1 samples/s +Step 19050/20005 | Loss: 2.9076 | LR: 2.87e-06 | 5801.1ms/step | 4.1 samples/s +Step 19060/20005 | Loss: 2.9055 | LR: 2.81e-06 | 5802.7ms/step | 4.1 samples/s +Step 19070/20005 | Loss: 2.9181 | LR: 2.75e-06 | 5800.7ms/step | 4.1 samples/s +Step 19080/20005 | Loss: 2.8634 | LR: 2.69e-06 | 5803.0ms/step | 4.1 samples/s +Step 19090/20005 | Loss: 2.9011 | LR: 2.64e-06 | 5803.1ms/step | 4.1 samples/s +Step 19100/20005 | Loss: 2.8946 | LR: 2.58e-06 | 5801.6ms/step | 4.1 samples/s +Step 19110/20005 | Loss: 2.8855 | LR: 2.52e-06 | 5846.1ms/step | 4.1 samples/s +Step 19120/20005 | Loss: 2.8977 | LR: 2.47e-06 | 5802.8ms/step | 4.1 samples/s +Step 19130/20005 | Loss: 2.8775 | LR: 2.41e-06 | 5799.4ms/step | 4.1 samples/s +Step 19140/20005 | Loss: 2.8940 | LR: 2.36e-06 | 5801.6ms/step | 4.1 samples/s +Step 19150/20005 | Loss: 2.8891 | LR: 2.30e-06 | 5801.3ms/step | 4.1 samples/s +Step 19160/20005 | Loss: 2.8665 | LR: 2.25e-06 | 5803.0ms/step | 4.1 samples/s +Step 19170/20005 | Loss: 2.8587 | LR: 2.20e-06 | 5802.4ms/step | 4.1 samples/s +Step 19180/20005 | Loss: 2.8930 | LR: 2.14e-06 | 5798.3ms/step | 4.1 samples/s +Step 19190/20005 | Loss: 2.9038 | LR: 2.09e-06 | 5801.6ms/step | 4.1 samples/s +Step 19200/20005 | Loss: 2.8835 | LR: 2.04e-06 | 5799.8ms/step | 4.1 samples/s +Step 19210/20005 | Loss: 2.9115 | LR: 1.99e-06 | 5801.6ms/step | 4.1 samples/s +Step 19220/20005 | Loss: 2.9010 | LR: 1.94e-06 | 5800.3ms/step | 4.1 samples/s +Step 19230/20005 | Loss: 2.9071 | LR: 1.89e-06 | 5800.3ms/step | 4.1 samples/s +Step 19240/20005 | Loss: 2.9033 | LR: 1.84e-06 | 5802.5ms/step | 4.1 samples/s +Step 19250/20005 | Loss: 2.8618 | LR: 1.80e-06 | 5799.7ms/step | 4.1 samples/s +Validation at step 19250: Loss = 2.9042, Perplexity = 18.25 +Step 19260/20005 | Loss: 2.8789 | LR: 1.75e-06 | 40859.3ms/step | 0.6 samples/s +Step 19270/20005 | Loss: 2.9005 | LR: 1.70e-06 | 5798.0ms/step | 4.1 samples/s +Step 19280/20005 | Loss: 2.9064 | LR: 1.66e-06 | 5802.0ms/step | 4.1 samples/s +Step 19290/20005 | Loss: 2.9035 | LR: 1.61e-06 | 5802.4ms/step | 4.1 samples/s +Step 19300/20005 | Loss: 2.8814 | LR: 1.57e-06 | 5800.4ms/step | 4.1 samples/s +Step 19310/20005 | Loss: 2.9034 | LR: 1.52e-06 | 5799.9ms/step | 4.1 samples/s +Step 19320/20005 | Loss: 2.8996 | LR: 1.48e-06 | 5800.9ms/step | 4.1 samples/s +Step 19330/20005 | Loss: 2.8856 | LR: 1.43e-06 | 5802.4ms/step | 4.1 samples/s +Step 19340/20005 | Loss: 2.8643 | LR: 1.39e-06 | 5798.2ms/step | 4.1 samples/s +Step 19350/20005 | Loss: 2.8999 | LR: 1.35e-06 | 5801.1ms/step | 4.1 samples/s +Step 19360/20005 | Loss: 2.8943 | LR: 1.31e-06 | 5803.3ms/step | 4.1 samples/s +Step 19370/20005 | Loss: 2.8913 | LR: 1.27e-06 | 5801.3ms/step | 4.1 samples/s +Step 19380/20005 | Loss: 2.8962 | LR: 1.23e-06 | 5800.5ms/step | 4.1 samples/s +Step 19390/20005 | Loss: 2.8758 | LR: 1.19e-06 | 5802.7ms/step | 4.1 samples/s +Step 19400/20005 | Loss: 2.8741 | LR: 1.15e-06 | 5800.3ms/step | 4.1 samples/s +Step 19410/20005 | Loss: 2.9121 | LR: 1.11e-06 | 5798.2ms/step | 4.1 samples/s +Step 19420/20005 | Loss: 2.8689 | LR: 1.08e-06 | 5798.5ms/step | 4.1 samples/s +Step 19430/20005 | Loss: 2.8710 | LR: 1.04e-06 | 5798.4ms/step | 4.1 samples/s +Step 19440/20005 | Loss: 2.8682 | LR: 1.00e-06 | 5801.2ms/step | 4.1 samples/s +Step 19450/20005 | Loss: 2.8935 | LR: 9.70e-07 | 5801.5ms/step | 4.1 samples/s +Step 19460/20005 | Loss: 2.8957 | LR: 9.35e-07 | 5801.1ms/step | 4.1 samples/s +Step 19470/20005 | Loss: 2.8909 | LR: 9.01e-07 | 5798.9ms/step | 4.1 samples/s +Step 19480/20005 | Loss: 2.8730 | LR: 8.67e-07 | 5796.8ms/step | 4.1 samples/s +Step 19490/20005 | Loss: 2.8959 | LR: 8.35e-07 | 5797.7ms/step | 4.1 samples/s +Step 19500/20005 | Loss: 2.8824 | LR: 8.03e-07 | 5797.8ms/step | 4.1 samples/s +Validation at step 19500: Loss = 2.9036, Perplexity = 18.24 +Step 19510/20005 | Loss: 2.8928 | LR: 7.71e-07 | 40869.8ms/step | 0.6 samples/s +Step 19520/20005 | Loss: 2.8678 | LR: 7.40e-07 | 5801.5ms/step | 4.1 samples/s +Step 19530/20005 | Loss: 2.8681 | LR: 7.10e-07 | 5800.2ms/step | 4.1 samples/s +Step 19540/20005 | Loss: 2.8642 | LR: 6.80e-07 | 5798.5ms/step | 4.1 samples/s +Step 19550/20005 | Loss: 2.9356 | LR: 6.51e-07 | 5801.8ms/step | 4.1 samples/s +Step 19560/20005 | Loss: 2.8972 | LR: 6.23e-07 | 5802.1ms/step | 4.1 samples/s +Step 19570/20005 | Loss: 2.9023 | LR: 5.95e-07 | 5798.3ms/step | 4.1 samples/s +Step 19580/20005 | Loss: 2.8574 | LR: 5.68e-07 | 5836.4ms/step | 4.1 samples/s +Step 19590/20005 | Loss: 2.8756 | LR: 5.42e-07 | 5807.5ms/step | 4.1 samples/s +Step 19600/20005 | Loss: 2.8952 | LR: 5.16e-07 | 5800.3ms/step | 4.1 samples/s +Step 19610/20005 | Loss: 2.8886 | LR: 4.91e-07 | 5799.3ms/step | 4.1 samples/s +Step 19620/20005 | Loss: 2.8894 | LR: 4.66e-07 | 5803.7ms/step | 4.1 samples/s +Step 19630/20005 | Loss: 2.8946 | LR: 4.42e-07 | 5803.9ms/step | 4.1 samples/s +Step 19640/20005 | Loss: 2.9110 | LR: 4.19e-07 | 5802.1ms/step | 4.1 samples/s +Step 19650/20005 | Loss: 2.9165 | LR: 3.96e-07 | 5800.4ms/step | 4.1 samples/s +Step 19660/20005 | Loss: 2.8830 | LR: 3.74e-07 | 5804.4ms/step | 4.1 samples/s +Step 19670/20005 | Loss: 2.8803 | LR: 3.53e-07 | 5801.1ms/step | 4.1 samples/s +Step 19680/20005 | Loss: 2.8821 | LR: 3.32e-07 | 5802.8ms/step | 4.1 samples/s +Step 19690/20005 | Loss: 2.8748 | LR: 3.12e-07 | 5801.4ms/step | 4.1 samples/s +Step 19700/20005 | Loss: 2.8839 | LR: 2.92e-07 | 5802.6ms/step | 4.1 samples/s +Step 19710/20005 | Loss: 2.8690 | LR: 2.73e-07 | 5803.1ms/step | 4.1 samples/s +Step 19720/20005 | Loss: 2.8843 | LR: 2.55e-07 | 5799.9ms/step | 4.1 samples/s +Step 19730/20005 | Loss: 2.8619 | LR: 2.37e-07 | 5802.0ms/step | 4.1 samples/s +Step 19740/20005 | Loss: 2.8787 | LR: 2.20e-07 | 5803.5ms/step | 4.1 samples/s +Step 19750/20005 | Loss: 2.8705 | LR: 2.04e-07 | 5801.0ms/step | 4.1 samples/s +Validation at step 19750: Loss = 2.9035, Perplexity = 18.24 +Step 19760/20005 | Loss: 2.8728 | LR: 1.88e-07 | 40861.9ms/step | 0.6 samples/s +Step 19770/20005 | Loss: 2.9028 | LR: 1.73e-07 | 5803.2ms/step | 4.1 samples/s +Step 19780/20005 | Loss: 2.8959 | LR: 1.59e-07 | 5801.9ms/step | 4.1 samples/s +Step 19790/20005 | Loss: 2.8860 | LR: 1.45e-07 | 5800.9ms/step | 4.1 samples/s +Step 19800/20005 | Loss: 2.8817 | LR: 1.32e-07 | 5799.4ms/step | 4.1 samples/s +Step 19810/20005 | Loss: 2.8840 | LR: 1.19e-07 | 5798.4ms/step | 4.1 samples/s +Step 19820/20005 | Loss: 2.8737 | LR: 1.07e-07 | 5797.4ms/step | 4.1 samples/s +Step 19830/20005 | Loss: 2.8726 | LR: 9.57e-08 | 5800.9ms/step | 4.1 samples/s +Step 19840/20005 | Loss: 2.8749 | LR: 8.50e-08 | 5800.1ms/step | 4.1 samples/s +Step 19850/20005 | Loss: 2.8855 | LR: 7.50e-08 | 5798.7ms/step | 4.1 samples/s +Step 19860/20005 | Loss: 2.8639 | LR: 6.55e-08 | 5798.3ms/step | 4.1 samples/s +Step 19870/20005 | Loss: 2.9065 | LR: 5.68e-08 | 5798.5ms/step | 4.1 samples/s +Step 19880/20005 | Loss: 2.8865 | LR: 4.86e-08 | 5799.3ms/step | 4.1 samples/s +Step 19890/20005 | Loss: 2.8910 | LR: 4.11e-08 | 5801.9ms/step | 4.1 samples/s +Step 19900/20005 | Loss: 2.8658 | LR: 3.42e-08 | 5806.0ms/step | 4.1 samples/s +Step 19910/20005 | Loss: 2.8996 | LR: 2.79e-08 | 5799.8ms/step | 4.1 samples/s +Step 19920/20005 | Loss: 2.8895 | LR: 2.23e-08 | 5799.6ms/step | 4.1 samples/s +Step 19930/20005 | Loss: 2.8606 | LR: 1.73e-08 | 5802.8ms/step | 4.1 samples/s +Step 19940/20005 | Loss: 2.8804 | LR: 1.29e-08 | 5800.1ms/step | 4.1 samples/s +Step 19950/20005 | Loss: 2.8914 | LR: 9.22e-09 | 5803.8ms/step | 4.1 samples/s +Step 19960/20005 | Loss: 2.9020 | LR: 6.12e-09 | 5804.7ms/step | 4.1 samples/s +Step 19970/20005 | Loss: 2.9103 | LR: 3.65e-09 | 5801.9ms/step | 4.1 samples/s +Step 19980/20005 | Loss: 2.9041 | LR: 1.82e-09 | 5802.7ms/step | 4.1 samples/s +Step 19990/20005 | Loss: 2.8692 | LR: 6.20e-10 | 5801.4ms/step | 4.1 samples/s +Step 20000/20005 | Loss: 2.8643 | LR: 5.06e-11 | 5798.9ms/step | 4.1 samples/s +Validation at step 20000: Loss = 2.9035, Perplexity = 18.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-20000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-20005 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log; uploading config.yaml +wandb: uploading history steps 100-100, summary, console lines 210-210 +wandb: +wandb: Run history: +wandb: eval/loss █▂▁▁▁ +wandb: eval/perplexity █▂▁▁▁ +wandb: speed/ms_per_step █▁▁▁▁▁▁▁▁▁▁▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ▁██████████████████████████████▁████████ +wandb: speed/steps_per_sec ▁██████████████████████▁████████████████ +wandb: train/epoch ▁▁▁▁▂▂▂▃▃▃▃▃▃▃▃▄▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇████ +wandb: train/global_step ▁▁▂▂▂▂▂▂▃▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇███ +wandb: train/learning_rate ████▇▇▆▆▆▆▅▅▅▅▅▄▄▄▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁ +wandb: train/loss ▃▅▆▁▅▂▄▁▄▅▁▅▅▃▄▂▂▄▂▄▂▁█▄▅▄▄▆▃▂▁▃▄▃▁▃▄▅▆▂ +wandb: +wandb: Run summary: +wandb: eval/loss 2.90353 +wandb: eval/perplexity 18.23845 +wandb: speed/ms_per_step 5798.85206 +wandb: speed/samples_per_sec 4.13875 +wandb: speed/steps_per_sec 0.17245 +wandb: train/epoch 0.4717 +wandb: train/global_step 20000 +wandb: train/learning_rate 0.0 +wandb: train/loss 2.86429 +wandb: +wandb: 🚀 View run dense-resume-18k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i0f0xwcq +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260513_052440-i0f0xwcq/logs +Training complete! diff --git a/epfl_final/log_files/cont_pretrain_20k.log b/epfl_final/log_files/cont_pretrain_20k.log new file mode 100644 index 0000000000000000000000000000000000000000..151433d7e5d6a2086e1d1f8936f65698fcf7224e --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_20k.log @@ -0,0 +1,853 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-44000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 20k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-44000 +Checkpoint activations: FalseCheckpoint activations: False + +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-44000... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 44000 (epoch 0, micro_step 440000) +Resuming training from step 44000 +Resuming wandb run 8igthj1g +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 8igthj1g +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260604_030818-8igthj1g +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run dense continued 20k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8igthj1g +Starting training... +Validation at step 44000: Loss = 2.5048, Perplexity = 12.24 +wandb: WARNING Tried to log to step 44000 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 44001: dense (dense step interval) +Step 44010/50462 | Loss: 2.5147 | LR: 2.05e-05 | 76343.6ms/step | 0.8 samples/s +wandb: WARNING Tried to log to step 44010 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44020/50462 | Loss: 2.5009 | LR: 2.04e-05 | 5185.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44020 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44030/50462 | Loss: 2.4941 | LR: 2.04e-05 | 5192.2ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44030 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44040/50462 | Loss: 2.4883 | LR: 2.03e-05 | 5203.5ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44040 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44050/50462 | Loss: 2.5206 | LR: 2.02e-05 | 5190.3ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44050 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44060/50462 | Loss: 2.5191 | LR: 2.02e-05 | 5220.3ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44060 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44070/50462 | Loss: 2.5187 | LR: 2.01e-05 | 5193.4ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44070 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44080/50462 | Loss: 2.5006 | LR: 2.01e-05 | 5195.4ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44080 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44090/50462 | Loss: 2.4700 | LR: 2.00e-05 | 5199.3ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44090 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44100/50462 | Loss: 2.4884 | LR: 1.99e-05 | 5205.9ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44100 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44110/50462 | Loss: 2.4903 | LR: 1.99e-05 | 5203.3ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44110 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44120/50462 | Loss: 2.5063 | LR: 1.98e-05 | 5207.8ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44120 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44130/50462 | Loss: 2.5124 | LR: 1.97e-05 | 5256.2ms/step | 11.4 samples/s +wandb: WARNING Tried to log to step 44130 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44140/50462 | Loss: 2.4806 | LR: 1.97e-05 | 5196.0ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44140 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44150/50462 | Loss: 2.4756 | LR: 1.96e-05 | 5194.8ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44150 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44160/50462 | Loss: 2.5060 | LR: 1.96e-05 | 5217.2ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44160 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44170/50462 | Loss: 2.5094 | LR: 1.95e-05 | 5231.7ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44170 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44180/50462 | Loss: 2.4765 | LR: 1.94e-05 | 5201.0ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44180 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44190/50462 | Loss: 2.4826 | LR: 1.94e-05 | 5194.5ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44190 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44200/50462 | Loss: 2.5043 | LR: 1.93e-05 | 5259.2ms/step | 11.4 samples/s +wandb: WARNING Tried to log to step 44200 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44210/50462 | Loss: 2.4919 | LR: 1.93e-05 | 5196.4ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44210 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44220/50462 | Loss: 2.4980 | LR: 1.92e-05 | 5192.5ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44220 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44230/50462 | Loss: 2.5093 | LR: 1.91e-05 | 5196.8ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44230 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44240/50462 | Loss: 2.4970 | LR: 1.91e-05 | 5173.2ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44240 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44250/50462 | Loss: 2.5105 | LR: 1.90e-05 | 5175.3ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44250 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 44250: Loss = 2.5056, Perplexity = 12.25 +wandb: WARNING Tried to log to step 44250 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44260/50462 | Loss: 2.5054 | LR: 1.90e-05 | 7741.5ms/step | 7.8 samples/s +wandb: WARNING Tried to log to step 44260 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44270/50462 | Loss: 2.4754 | LR: 1.89e-05 | 5184.2ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44270 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44280/50462 | Loss: 2.4989 | LR: 1.88e-05 | 5175.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44280 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44290/50462 | Loss: 2.4949 | LR: 1.88e-05 | 5171.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44290 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44300/50462 | Loss: 2.4461 | LR: 1.87e-05 | 5189.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44300 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44310/50462 | Loss: 2.4764 | LR: 1.87e-05 | 5181.6ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44310 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44320/50462 | Loss: 2.5087 | LR: 1.86e-05 | 5192.7ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44320 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44330/50462 | Loss: 2.4992 | LR: 1.85e-05 | 5182.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44330 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44340/50462 | Loss: 2.4956 | LR: 1.85e-05 | 5172.1ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44340 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44350/50462 | Loss: 2.5002 | LR: 1.84e-05 | 5184.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44350 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44360/50462 | Loss: 2.4735 | LR: 1.84e-05 | 5181.6ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44360 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44370/50462 | Loss: 2.4993 | LR: 1.83e-05 | 5176.3ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44370 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44380/50462 | Loss: 2.4896 | LR: 1.82e-05 | 5203.1ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 44380 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44390/50462 | Loss: 2.5099 | LR: 1.82e-05 | 5183.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44390 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44400/50462 | Loss: 2.4760 | LR: 1.81e-05 | 5188.7ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44400 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44410/50462 | Loss: 2.4856 | LR: 1.81e-05 | 5189.1ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44410 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44420/50462 | Loss: 2.4958 | LR: 1.80e-05 | 5247.0ms/step | 11.4 samples/s +wandb: WARNING Tried to log to step 44420 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44430/50462 | Loss: 2.5030 | LR: 1.79e-05 | 5144.2ms/step | 11.7 samples/s +wandb: WARNING Tried to log to step 44430 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44440/50462 | Loss: 2.4919 | LR: 1.79e-05 | 5158.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44440 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44450/50462 | Loss: 2.4807 | LR: 1.78e-05 | 5172.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 44450 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 44460/50462 | Loss: 2.4960 | LR: 1.78e-05 | 5188.2ms/step | 11.6 samples/s +Step 44470/50462 | Loss: 2.5090 | LR: 1.77e-05 | 5172.5ms/step | 11.6 samples/s +Step 44480/50462 | Loss: 2.5232 | LR: 1.77e-05 | 5236.6ms/step | 11.5 samples/s +Step 44490/50462 | Loss: 2.5189 | LR: 1.76e-05 | 5192.7ms/step | 11.6 samples/s +Step 44500/50462 | Loss: 2.4821 | LR: 1.75e-05 | 5258.1ms/step | 11.4 samples/s +Validation at step 44500: Loss = 2.5053, Perplexity = 12.25 +Step 44510/50462 | Loss: 2.4710 | LR: 1.75e-05 | 7732.5ms/step | 7.8 samples/s +Step 44520/50462 | Loss: 2.4697 | LR: 1.74e-05 | 5188.2ms/step | 11.6 samples/s +Step 44530/50462 | Loss: 2.4962 | LR: 1.74e-05 | 5193.9ms/step | 11.6 samples/s +Step 44540/50462 | Loss: 2.4847 | LR: 1.73e-05 | 5192.1ms/step | 11.6 samples/s +Step 44550/50462 | Loss: 2.4967 | LR: 1.72e-05 | 5175.2ms/step | 11.6 samples/s +Step 44560/50462 | Loss: 2.5024 | LR: 1.72e-05 | 5169.8ms/step | 11.6 samples/s +Step 44570/50462 | Loss: 2.4888 | LR: 1.71e-05 | 5167.9ms/step | 11.6 samples/s +Step 44580/50462 | Loss: 2.4674 | LR: 1.71e-05 | 5196.7ms/step | 11.5 samples/s +Step 44590/50462 | Loss: 2.4856 | LR: 1.70e-05 | 5181.2ms/step | 11.6 samples/s +Step 44600/50462 | Loss: 2.5096 | LR: 1.70e-05 | 5187.5ms/step | 11.6 samples/s +Step 44610/50462 | Loss: 2.4910 | LR: 1.69e-05 | 5172.7ms/step | 11.6 samples/s +Step 44620/50462 | Loss: 2.5085 | LR: 1.68e-05 | 5170.6ms/step | 11.6 samples/s +Step 44630/50462 | Loss: 2.4936 | LR: 1.68e-05 | 5181.3ms/step | 11.6 samples/s +Step 44640/50462 | Loss: 2.4850 | LR: 1.67e-05 | 5175.1ms/step | 11.6 samples/s +Step 44650/50462 | Loss: 2.4734 | LR: 1.67e-05 | 5190.9ms/step | 11.6 samples/s +Step 44660/50462 | Loss: 2.5113 | LR: 1.66e-05 | 5196.8ms/step | 11.5 samples/s +Step 44670/50462 | Loss: 2.5205 | LR: 1.66e-05 | 5181.3ms/step | 11.6 samples/s +Step 44680/50462 | Loss: 2.5065 | LR: 1.65e-05 | 5176.2ms/step | 11.6 samples/s +Step 44690/50462 | Loss: 2.5030 | LR: 1.64e-05 | 5177.1ms/step | 11.6 samples/s +Step 44700/50462 | Loss: 2.5082 | LR: 1.64e-05 | 5174.3ms/step | 11.6 samples/s +Step 44710/50462 | Loss: 2.4924 | LR: 1.63e-05 | 5246.6ms/step | 11.4 samples/s +Step 44720/50462 | Loss: 2.4936 | LR: 1.63e-05 | 5178.5ms/step | 11.6 samples/s +Step 44730/50462 | Loss: 2.4819 | LR: 1.62e-05 | 5182.7ms/step | 11.6 samples/s +Step 44740/50462 | Loss: 2.4776 | LR: 1.62e-05 | 5184.7ms/step | 11.6 samples/s +Step 44750/50462 | Loss: 2.5021 | LR: 1.61e-05 | 5192.5ms/step | 11.6 samples/s +Validation at step 44750: Loss = 2.5052, Perplexity = 12.25 +Step 44760/50462 | Loss: 2.5177 | LR: 1.61e-05 | 7733.8ms/step | 7.8 samples/s +Step 44770/50462 | Loss: 2.4661 | LR: 1.60e-05 | 5187.1ms/step | 11.6 samples/s +Step 44780/50462 | Loss: 2.4948 | LR: 1.59e-05 | 5222.2ms/step | 11.5 samples/s +Step 44790/50462 | Loss: 2.4846 | LR: 1.59e-05 | 5191.7ms/step | 11.6 samples/s +Step 44800/50462 | Loss: 2.4857 | LR: 1.58e-05 | 5183.6ms/step | 11.6 samples/s +Step 44810/50462 | Loss: 2.5117 | LR: 1.58e-05 | 5224.6ms/step | 11.5 samples/s +Step 44820/50462 | Loss: 2.5013 | LR: 1.57e-05 | 5174.7ms/step | 11.6 samples/s +Step 44830/50462 | Loss: 2.5227 | LR: 1.57e-05 | 5169.5ms/step | 11.6 samples/s +Step 44840/50462 | Loss: 2.4790 | LR: 1.56e-05 | 5160.1ms/step | 11.6 samples/s +Step 44850/50462 | Loss: 2.4829 | LR: 1.56e-05 | 5169.4ms/step | 11.6 samples/s +Step 44860/50462 | Loss: 2.4823 | LR: 1.55e-05 | 5169.3ms/step | 11.6 samples/s +Step 44870/50462 | Loss: 2.4716 | LR: 1.54e-05 | 5178.9ms/step | 11.6 samples/s +Step 44880/50462 | Loss: 2.4956 | LR: 1.54e-05 | 5177.7ms/step | 11.6 samples/s +Step 44890/50462 | Loss: 2.4588 | LR: 1.53e-05 | 5170.5ms/step | 11.6 samples/s +Step 44900/50462 | Loss: 2.4590 | LR: 1.53e-05 | 5182.0ms/step | 11.6 samples/s +Step 44910/50462 | Loss: 2.4815 | LR: 1.52e-05 | 5182.6ms/step | 11.6 samples/s +Step 44920/50462 | Loss: 2.4839 | LR: 1.52e-05 | 5190.6ms/step | 11.6 samples/s +Step 44930/50462 | Loss: 2.4873 | LR: 1.51e-05 | 5184.1ms/step | 11.6 samples/s +Step 44940/50462 | Loss: 2.4897 | LR: 1.51e-05 | 5181.3ms/step | 11.6 samples/s +Step 44950/50462 | Loss: 2.4473 | LR: 1.50e-05 | 5190.5ms/step | 11.6 samples/s +Step 44960/50462 | Loss: 2.5261 | LR: 1.50e-05 | 5190.4ms/step | 11.6 samples/s +Step 44970/50462 | Loss: 2.4759 | LR: 1.49e-05 | 5179.4ms/step | 11.6 samples/s +Step 44980/50462 | Loss: 2.4877 | LR: 1.49e-05 | 5194.3ms/step | 11.6 samples/s +Step 44990/50462 | Loss: 2.5167 | LR: 1.48e-05 | 5185.6ms/step | 11.6 samples/s +Step 45000/50462 | Loss: 2.5076 | LR: 1.47e-05 | 5236.9ms/step | 11.5 samples/s +Validation at step 45000: Loss = 2.5051, Perplexity = 12.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-45000 +Step 45010/50462 | Loss: 2.4997 | LR: 1.47e-05 | 8739.1ms/step | 6.9 samples/s +Step 45020/50462 | Loss: 2.4840 | LR: 1.46e-05 | 5203.3ms/step | 11.5 samples/s +Step 45030/50462 | Loss: 2.4906 | LR: 1.46e-05 | 5188.3ms/step | 11.6 samples/s +Step 45040/50462 | Loss: 2.5029 | LR: 1.45e-05 | 5191.5ms/step | 11.6 samples/s +Step 45050/50462 | Loss: 2.4785 | LR: 1.45e-05 | 5200.7ms/step | 11.5 samples/s +Step 45060/50462 | Loss: 2.5011 | LR: 1.44e-05 | 5203.3ms/step | 11.5 samples/s +Step 45070/50462 | Loss: 2.4950 | LR: 1.44e-05 | 5193.8ms/step | 11.6 samples/s +Step 45080/50462 | Loss: 2.4868 | LR: 1.43e-05 | 5203.7ms/step | 11.5 samples/s +Step 45090/50462 | Loss: 2.4864 | LR: 1.43e-05 | 5250.3ms/step | 11.4 samples/s +Step 45100/50462 | Loss: 2.5209 | LR: 1.42e-05 | 5204.0ms/step | 11.5 samples/s +Step 45110/50462 | Loss: 2.5200 | LR: 1.42e-05 | 5244.6ms/step | 11.4 samples/s +Step 45120/50462 | Loss: 2.5027 | LR: 1.41e-05 | 5177.9ms/step | 11.6 samples/s +Step 45130/50462 | Loss: 2.5109 | LR: 1.41e-05 | 5186.4ms/step | 11.6 samples/s +Step 45140/50462 | Loss: 2.4801 | LR: 1.40e-05 | 5187.7ms/step | 11.6 samples/s +Step 45150/50462 | Loss: 2.5050 | LR: 1.40e-05 | 5187.6ms/step | 11.6 samples/s +Step 45160/50462 | Loss: 2.5108 | LR: 1.39e-05 | 5185.1ms/step | 11.6 samples/s +Step 45170/50462 | Loss: 2.4723 | LR: 1.39e-05 | 5201.9ms/step | 11.5 samples/s +Step 45180/50462 | Loss: 2.5006 | LR: 1.38e-05 | 5186.6ms/step | 11.6 samples/s +Step 45190/50462 | Loss: 2.4874 | LR: 1.37e-05 | 5197.7ms/step | 11.5 samples/s +Step 45200/50462 | Loss: 2.5110 | LR: 1.37e-05 | 5201.1ms/step | 11.5 samples/s +Step 45210/50462 | Loss: 2.4961 | LR: 1.36e-05 | 5189.7ms/step | 11.6 samples/s +Step 45220/50462 | Loss: 2.4821 | LR: 1.36e-05 | 5181.5ms/step | 11.6 samples/s +Step 45230/50462 | Loss: 2.4763 | LR: 1.35e-05 | 5186.5ms/step | 11.6 samples/s +Step 45240/50462 | Loss: 2.5118 | LR: 1.35e-05 | 5184.0ms/step | 11.6 samples/s +Step 45250/50462 | Loss: 2.4881 | LR: 1.34e-05 | 5178.8ms/step | 11.6 samples/s +Validation at step 45250: Loss = 2.5050, Perplexity = 12.24 +Step 45260/50462 | Loss: 2.4996 | LR: 1.34e-05 | 7763.9ms/step | 7.7 samples/s +Step 45270/50462 | Loss: 2.5095 | LR: 1.33e-05 | 5209.7ms/step | 11.5 samples/s +Step 45280/50462 | Loss: 2.4668 | LR: 1.33e-05 | 5200.2ms/step | 11.5 samples/s +Step 45290/50462 | Loss: 2.4931 | LR: 1.32e-05 | 5265.0ms/step | 11.4 samples/s +Step 45300/50462 | Loss: 2.5119 | LR: 1.32e-05 | 5197.1ms/step | 11.5 samples/s +Step 45310/50462 | Loss: 2.4799 | LR: 1.31e-05 | 5186.7ms/step | 11.6 samples/s +Step 45320/50462 | Loss: 2.4879 | LR: 1.31e-05 | 5193.1ms/step | 11.6 samples/s +Step 45330/50462 | Loss: 2.5046 | LR: 1.30e-05 | 5210.3ms/step | 11.5 samples/s +Step 45340/50462 | Loss: 2.4958 | LR: 1.30e-05 | 5190.2ms/step | 11.6 samples/s +Step 45350/50462 | Loss: 2.4928 | LR: 1.29e-05 | 5199.7ms/step | 11.5 samples/s +Step 45360/50462 | Loss: 2.5122 | LR: 1.29e-05 | 5209.8ms/step | 11.5 samples/s +Step 45370/50462 | Loss: 2.4854 | LR: 1.28e-05 | 5205.9ms/step | 11.5 samples/s +Step 45380/50462 | Loss: 2.4811 | LR: 1.28e-05 | 5198.9ms/step | 11.5 samples/s +Step 45390/50462 | Loss: 2.4878 | LR: 1.27e-05 | 5241.0ms/step | 11.4 samples/s +Step 45400/50462 | Loss: 2.4718 | LR: 1.27e-05 | 5198.8ms/step | 11.5 samples/s +Step 45410/50462 | Loss: 2.5171 | LR: 1.26e-05 | 5195.2ms/step | 11.5 samples/s +Step 45420/50462 | Loss: 2.4874 | LR: 1.26e-05 | 5222.4ms/step | 11.5 samples/s +Step 45430/50462 | Loss: 2.4626 | LR: 1.25e-05 | 5183.6ms/step | 11.6 samples/s +Step 45440/50462 | Loss: 2.5043 | LR: 1.25e-05 | 5179.3ms/step | 11.6 samples/s +Step 45450/50462 | Loss: 2.4944 | LR: 1.24e-05 | 5193.3ms/step | 11.6 samples/s +Step 45460/50462 | Loss: 2.4843 | LR: 1.24e-05 | 5192.8ms/step | 11.6 samples/s +Step 45470/50462 | Loss: 2.4933 | LR: 1.23e-05 | 5184.8ms/step | 11.6 samples/s +Step 45480/50462 | Loss: 2.4835 | LR: 1.23e-05 | 5184.9ms/step | 11.6 samples/s +Step 45490/50462 | Loss: 2.5048 | LR: 1.22e-05 | 5177.5ms/step | 11.6 samples/s +Step 45500/50462 | Loss: 2.5045 | LR: 1.22e-05 | 5185.4ms/step | 11.6 samples/s +Validation at step 45500: Loss = 2.5050, Perplexity = 12.24 +Step 45510/50462 | Loss: 2.4787 | LR: 1.21e-05 | 7725.2ms/step | 7.8 samples/s +Step 45520/50462 | Loss: 2.5250 | LR: 1.21e-05 | 5173.0ms/step | 11.6 samples/s +Step 45530/50462 | Loss: 2.4709 | LR: 1.20e-05 | 5184.5ms/step | 11.6 samples/s +Step 45540/50462 | Loss: 2.5193 | LR: 1.20e-05 | 5167.8ms/step | 11.6 samples/s +Step 45550/50462 | Loss: 2.4792 | LR: 1.19e-05 | 5167.7ms/step | 11.6 samples/s +Step 45560/50462 | Loss: 2.5038 | LR: 1.19e-05 | 5176.6ms/step | 11.6 samples/s +Step 45570/50462 | Loss: 2.5085 | LR: 1.19e-05 | 5178.5ms/step | 11.6 samples/s +Step 45580/50462 | Loss: 2.4852 | LR: 1.18e-05 | 5247.9ms/step | 11.4 samples/s +Step 45590/50462 | Loss: 2.4742 | LR: 1.18e-05 | 5219.2ms/step | 11.5 samples/s +Step 45600/50462 | Loss: 2.4828 | LR: 1.17e-05 | 5204.7ms/step | 11.5 samples/s +Step 45610/50462 | Loss: 2.4611 | LR: 1.17e-05 | 5194.6ms/step | 11.6 samples/s +Step 45620/50462 | Loss: 2.4929 | LR: 1.16e-05 | 5210.7ms/step | 11.5 samples/s +Step 45630/50462 | Loss: 2.5094 | LR: 1.16e-05 | 5207.5ms/step | 11.5 samples/s +Step 45640/50462 | Loss: 2.4878 | LR: 1.15e-05 | 5201.9ms/step | 11.5 samples/s +Step 45650/50462 | Loss: 2.5127 | LR: 1.15e-05 | 5203.4ms/step | 11.5 samples/s +Step 45660/50462 | Loss: 2.4982 | LR: 1.14e-05 | 5216.1ms/step | 11.5 samples/s +Step 45670/50462 | Loss: 2.4642 | LR: 1.14e-05 | 5218.4ms/step | 11.5 samples/s +Step 45680/50462 | Loss: 2.4839 | LR: 1.13e-05 | 5221.1ms/step | 11.5 samples/s +Step 45690/50462 | Loss: 2.5045 | LR: 1.13e-05 | 5246.8ms/step | 11.4 samples/s +Step 45700/50462 | Loss: 2.4966 | LR: 1.12e-05 | 5182.5ms/step | 11.6 samples/s +Step 45710/50462 | Loss: 2.4997 | LR: 1.12e-05 | 5187.8ms/step | 11.6 samples/s +Step 45720/50462 | Loss: 2.4908 | LR: 1.11e-05 | 5253.9ms/step | 11.4 samples/s +Step 45730/50462 | Loss: 2.4725 | LR: 1.11e-05 | 5201.6ms/step | 11.5 samples/s +Step 45740/50462 | Loss: 2.4864 | LR: 1.10e-05 | 5189.2ms/step | 11.6 samples/s +Step 45750/50462 | Loss: 2.5044 | LR: 1.10e-05 | 5204.2ms/step | 11.5 samples/s +Validation at step 45750: Loss = 2.5049, Perplexity = 12.24 +Step 45760/50462 | Loss: 2.4592 | LR: 1.10e-05 | 7774.5ms/step | 7.7 samples/s +Step 45770/50462 | Loss: 2.4911 | LR: 1.09e-05 | 5211.1ms/step | 11.5 samples/s +Step 45780/50462 | Loss: 2.4834 | LR: 1.09e-05 | 5198.5ms/step | 11.5 samples/s +Step 45790/50462 | Loss: 2.5220 | LR: 1.08e-05 | 5202.3ms/step | 11.5 samples/s +Step 45800/50462 | Loss: 2.4354 | LR: 1.08e-05 | 5204.0ms/step | 11.5 samples/s +Step 45810/50462 | Loss: 2.4962 | LR: 1.07e-05 | 5191.4ms/step | 11.6 samples/s +Step 45820/50462 | Loss: 2.4923 | LR: 1.07e-05 | 5195.5ms/step | 11.5 samples/s +Step 45830/50462 | Loss: 2.4986 | LR: 1.06e-05 | 5212.6ms/step | 11.5 samples/s +Step 45840/50462 | Loss: 2.5116 | LR: 1.06e-05 | 5201.5ms/step | 11.5 samples/s +Step 45850/50462 | Loss: 2.4936 | LR: 1.05e-05 | 5211.5ms/step | 11.5 samples/s +Step 45860/50462 | Loss: 2.5182 | LR: 1.05e-05 | 5206.8ms/step | 11.5 samples/s +Step 45870/50462 | Loss: 2.5032 | LR: 1.05e-05 | 5242.5ms/step | 11.4 samples/s +Step 45880/50462 | Loss: 2.5039 | LR: 1.04e-05 | 5184.8ms/step | 11.6 samples/s +Step 45890/50462 | Loss: 2.4665 | LR: 1.04e-05 | 5190.1ms/step | 11.6 samples/s +Step 45900/50462 | Loss: 2.4898 | LR: 1.03e-05 | 5178.0ms/step | 11.6 samples/s +Step 45910/50462 | Loss: 2.5200 | LR: 1.03e-05 | 5191.4ms/step | 11.6 samples/s +Step 45920/50462 | Loss: 2.4733 | LR: 1.02e-05 | 5184.1ms/step | 11.6 samples/s +Step 45930/50462 | Loss: 2.4848 | LR: 1.02e-05 | 5181.2ms/step | 11.6 samples/s +Step 45940/50462 | Loss: 2.5039 | LR: 1.01e-05 | 5173.1ms/step | 11.6 samples/s +Step 45950/50462 | Loss: 2.4910 | LR: 1.01e-05 | 5175.2ms/step | 11.6 samples/s +Step 45960/50462 | Loss: 2.4769 | LR: 1.01e-05 | 5181.8ms/step | 11.6 samples/s +Step 45970/50462 | Loss: 2.4824 | LR: 1.00e-05 | 5163.9ms/step | 11.6 samples/s +Step 45980/50462 | Loss: 2.5263 | LR: 9.96e-06 | 5187.7ms/step | 11.6 samples/s +Step 45990/50462 | Loss: 2.4801 | LR: 9.92e-06 | 5190.0ms/step | 11.6 samples/s +Step 46000/50462 | Loss: 2.4944 | LR: 9.87e-06 | 5252.5ms/step | 11.4 samples/s +Validation at step 46000: Loss = 2.5049, Perplexity = 12.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-46000 +Step 46010/50462 | Loss: 2.4972 | LR: 9.83e-06 | 8699.2ms/step | 6.9 samples/s +Step 46020/50462 | Loss: 2.4834 | LR: 9.79e-06 | 5214.3ms/step | 11.5 samples/s +Step 46030/50462 | Loss: 2.5078 | LR: 9.74e-06 | 5192.7ms/step | 11.6 samples/s +Step 46040/50462 | Loss: 2.5078 | LR: 9.70e-06 | 5201.9ms/step | 11.5 samples/s +Step 46050/50462 | Loss: 2.4901 | LR: 9.65e-06 | 5202.2ms/step | 11.5 samples/s +Step 46060/50462 | Loss: 2.4847 | LR: 9.61e-06 | 5204.8ms/step | 11.5 samples/s +Step 46070/50462 | Loss: 2.5043 | LR: 9.57e-06 | 5186.0ms/step | 11.6 samples/s +Step 46080/50462 | Loss: 2.4995 | LR: 9.52e-06 | 5212.7ms/step | 11.5 samples/s +Step 46090/50462 | Loss: 2.4684 | LR: 9.48e-06 | 5211.2ms/step | 11.5 samples/s +Step 46100/50462 | Loss: 2.4948 | LR: 9.44e-06 | 5192.0ms/step | 11.6 samples/s +Step 46110/50462 | Loss: 2.4900 | LR: 9.40e-06 | 5213.0ms/step | 11.5 samples/s +Step 46120/50462 | Loss: 2.4849 | LR: 9.35e-06 | 5202.5ms/step | 11.5 samples/s +Step 46130/50462 | Loss: 2.4863 | LR: 9.31e-06 | 5197.7ms/step | 11.5 samples/s +Step 46140/50462 | Loss: 2.4870 | LR: 9.27e-06 | 5204.4ms/step | 11.5 samples/s +Step 46150/50462 | Loss: 2.4853 | LR: 9.22e-06 | 5185.9ms/step | 11.6 samples/s +Step 46160/50462 | Loss: 2.5033 | LR: 9.18e-06 | 5251.5ms/step | 11.4 samples/s +Step 46170/50462 | Loss: 2.5035 | LR: 9.14e-06 | 5172.3ms/step | 11.6 samples/s +Step 46180/50462 | Loss: 2.4734 | LR: 9.10e-06 | 5176.3ms/step | 11.6 samples/s +Step 46190/50462 | Loss: 2.4736 | LR: 9.06e-06 | 5157.9ms/step | 11.6 samples/s +Step 46200/50462 | Loss: 2.4817 | LR: 9.01e-06 | 5154.3ms/step | 11.6 samples/s +Step 46210/50462 | Loss: 2.4980 | LR: 8.97e-06 | 5191.7ms/step | 11.6 samples/s +Step 46220/50462 | Loss: 2.5004 | LR: 8.93e-06 | 5176.2ms/step | 11.6 samples/s +Step 46230/50462 | Loss: 2.5075 | LR: 8.89e-06 | 5206.5ms/step | 11.5 samples/s +Step 46240/50462 | Loss: 2.5160 | LR: 8.85e-06 | 5202.9ms/step | 11.5 samples/s +Step 46250/50462 | Loss: 2.5136 | LR: 8.80e-06 | 5196.2ms/step | 11.5 samples/s +Validation at step 46250: Loss = 2.5049, Perplexity = 12.24 +Step 46260/50462 | Loss: 2.5136 | LR: 8.76e-06 | 7727.5ms/step | 7.8 samples/s +Step 46270/50462 | Loss: 2.5004 | LR: 8.72e-06 | 5190.9ms/step | 11.6 samples/s +Step 46280/50462 | Loss: 2.5317 | LR: 8.68e-06 | 5183.3ms/step | 11.6 samples/s +Step 46290/50462 | Loss: 2.5176 | LR: 8.64e-06 | 5179.9ms/step | 11.6 samples/s +Step 46300/50462 | Loss: 2.4887 | LR: 8.60e-06 | 5243.9ms/step | 11.4 samples/s +Step 46310/50462 | Loss: 2.4963 | LR: 8.56e-06 | 5175.4ms/step | 11.6 samples/s +Step 46320/50462 | Loss: 2.5217 | LR: 8.52e-06 | 5188.3ms/step | 11.6 samples/s +Step 46330/50462 | Loss: 2.5018 | LR: 8.47e-06 | 5224.0ms/step | 11.5 samples/s +Step 46340/50462 | Loss: 2.5011 | LR: 8.43e-06 | 5186.2ms/step | 11.6 samples/s +Step 46350/50462 | Loss: 2.5222 | LR: 8.39e-06 | 5164.1ms/step | 11.6 samples/s +Step 46360/50462 | Loss: 2.4844 | LR: 8.35e-06 | 5167.3ms/step | 11.6 samples/s +Step 46370/50462 | Loss: 2.4706 | LR: 8.31e-06 | 5174.8ms/step | 11.6 samples/s +Step 46380/50462 | Loss: 2.4955 | LR: 8.27e-06 | 5185.8ms/step | 11.6 samples/s +Step 46390/50462 | Loss: 2.4847 | LR: 8.23e-06 | 5173.9ms/step | 11.6 samples/s +Step 46400/50462 | Loss: 2.4765 | LR: 8.19e-06 | 5172.2ms/step | 11.6 samples/s +Step 46410/50462 | Loss: 2.4890 | LR: 8.15e-06 | 5183.7ms/step | 11.6 samples/s +Step 46420/50462 | Loss: 2.4881 | LR: 8.11e-06 | 5170.6ms/step | 11.6 samples/s +Step 46430/50462 | Loss: 2.5091 | LR: 8.07e-06 | 5174.4ms/step | 11.6 samples/s +Step 46440/50462 | Loss: 2.4920 | LR: 8.03e-06 | 5173.6ms/step | 11.6 samples/s +Step 46450/50462 | Loss: 2.5176 | LR: 7.99e-06 | 5232.9ms/step | 11.5 samples/s +Step 46460/50462 | Loss: 2.4957 | LR: 7.95e-06 | 5187.6ms/step | 11.6 samples/s +Step 46470/50462 | Loss: 2.4873 | LR: 7.91e-06 | 5177.5ms/step | 11.6 samples/s +Step 46480/50462 | Loss: 2.4829 | LR: 7.87e-06 | 5186.0ms/step | 11.6 samples/s +Step 46490/50462 | Loss: 2.4782 | LR: 7.83e-06 | 5176.1ms/step | 11.6 samples/s +Step 46500/50462 | Loss: 2.4953 | LR: 7.80e-06 | 5165.8ms/step | 11.6 samples/s +Validation at step 46500: Loss = 2.5049, Perplexity = 12.24 +Step 46510/50462 | Loss: 2.4902 | LR: 7.76e-06 | 7749.4ms/step | 7.7 samples/s +Step 46520/50462 | Loss: 2.5135 | LR: 7.72e-06 | 5187.4ms/step | 11.6 samples/s +Step 46530/50462 | Loss: 2.4931 | LR: 7.68e-06 | 5205.1ms/step | 11.5 samples/s +Step 46540/50462 | Loss: 2.5066 | LR: 7.64e-06 | 5192.6ms/step | 11.6 samples/s +Step 46550/50462 | Loss: 2.4674 | LR: 7.60e-06 | 5196.7ms/step | 11.5 samples/s +Step 46560/50462 | Loss: 2.4845 | LR: 7.56e-06 | 5181.3ms/step | 11.6 samples/s +Step 46570/50462 | Loss: 2.4955 | LR: 7.52e-06 | 5194.7ms/step | 11.6 samples/s +Step 46580/50462 | Loss: 2.4876 | LR: 7.49e-06 | 5191.6ms/step | 11.6 samples/s +Step 46590/50462 | Loss: 2.4797 | LR: 7.45e-06 | 5205.0ms/step | 11.5 samples/s +Step 46600/50462 | Loss: 2.4952 | LR: 7.41e-06 | 5199.4ms/step | 11.5 samples/s +Step 46610/50462 | Loss: 2.4932 | LR: 7.37e-06 | 5246.2ms/step | 11.4 samples/s +Step 46620/50462 | Loss: 2.4753 | LR: 7.33e-06 | 5163.2ms/step | 11.6 samples/s +Step 46630/50462 | Loss: 2.4900 | LR: 7.29e-06 | 5222.8ms/step | 11.5 samples/s +Step 46640/50462 | Loss: 2.5022 | LR: 7.26e-06 | 5201.0ms/step | 11.5 samples/s +Step 46650/50462 | Loss: 2.4942 | LR: 7.22e-06 | 5195.5ms/step | 11.5 samples/s +Step 46660/50462 | Loss: 2.4840 | LR: 7.18e-06 | 5195.6ms/step | 11.5 samples/s +Step 46670/50462 | Loss: 2.4988 | LR: 7.14e-06 | 5197.2ms/step | 11.5 samples/s +Step 46680/50462 | Loss: 2.5076 | LR: 7.11e-06 | 5190.2ms/step | 11.6 samples/s +Step 46690/50462 | Loss: 2.5246 | LR: 7.07e-06 | 5203.1ms/step | 11.5 samples/s +Step 46700/50462 | Loss: 2.4851 | LR: 7.03e-06 | 5193.3ms/step | 11.6 samples/s +Step 46710/50462 | Loss: 2.5175 | LR: 6.99e-06 | 5198.8ms/step | 11.5 samples/s +Step 46720/50462 | Loss: 2.5011 | LR: 6.96e-06 | 5196.4ms/step | 11.5 samples/s +Step 46730/50462 | Loss: 2.4808 | LR: 6.92e-06 | 5202.9ms/step | 11.5 samples/s +Step 46740/50462 | Loss: 2.5074 | LR: 6.88e-06 | 5191.6ms/step | 11.6 samples/s +Step 46750/50462 | Loss: 2.4906 | LR: 6.85e-06 | 5222.0ms/step | 11.5 samples/s +Validation at step 46750: Loss = 2.5049, Perplexity = 12.24 +Step 46760/50462 | Loss: 2.5052 | LR: 6.81e-06 | 7730.9ms/step | 7.8 samples/s +Step 46770/50462 | Loss: 2.4943 | LR: 6.77e-06 | 5159.1ms/step | 11.6 samples/s +Step 46780/50462 | Loss: 2.5204 | LR: 6.74e-06 | 5184.9ms/step | 11.6 samples/s +Step 46790/50462 | Loss: 2.4776 | LR: 6.70e-06 | 5170.3ms/step | 11.6 samples/s +Step 46800/50462 | Loss: 2.5151 | LR: 6.66e-06 | 5160.8ms/step | 11.6 samples/s +Step 46810/50462 | Loss: 2.5114 | LR: 6.63e-06 | 5181.9ms/step | 11.6 samples/s +Step 46820/50462 | Loss: 2.4747 | LR: 6.59e-06 | 5169.7ms/step | 11.6 samples/s +Step 46830/50462 | Loss: 2.4728 | LR: 6.56e-06 | 5176.0ms/step | 11.6 samples/s +Step 46840/50462 | Loss: 2.4985 | LR: 6.52e-06 | 5184.4ms/step | 11.6 samples/s +Step 46850/50462 | Loss: 2.4946 | LR: 6.48e-06 | 5174.9ms/step | 11.6 samples/s +Step 46860/50462 | Loss: 2.4941 | LR: 6.45e-06 | 5167.7ms/step | 11.6 samples/s +Step 46870/50462 | Loss: 2.4520 | LR: 6.41e-06 | 5172.2ms/step | 11.6 samples/s +Step 46880/50462 | Loss: 2.4878 | LR: 6.38e-06 | 5170.6ms/step | 11.6 samples/s +Step 46890/50462 | Loss: 2.4936 | LR: 6.34e-06 | 5169.1ms/step | 11.6 samples/s +Step 46900/50462 | Loss: 2.4857 | LR: 6.31e-06 | 5173.0ms/step | 11.6 samples/s +Step 46910/50462 | Loss: 2.4788 | LR: 6.27e-06 | 5219.4ms/step | 11.5 samples/s +Step 46920/50462 | Loss: 2.5168 | LR: 6.24e-06 | 5186.9ms/step | 11.6 samples/s +Step 46930/50462 | Loss: 2.4932 | LR: 6.20e-06 | 5234.2ms/step | 11.5 samples/s +Step 46940/50462 | Loss: 2.4948 | LR: 6.17e-06 | 5198.8ms/step | 11.5 samples/s +Step 46950/50462 | Loss: 2.4777 | LR: 6.13e-06 | 5164.3ms/step | 11.6 samples/s +Step 46960/50462 | Loss: 2.5014 | LR: 6.10e-06 | 5171.3ms/step | 11.6 samples/s +Step 46970/50462 | Loss: 2.5051 | LR: 6.06e-06 | 5153.7ms/step | 11.6 samples/s +Step 46980/50462 | Loss: 2.5043 | LR: 6.03e-06 | 5184.5ms/step | 11.6 samples/s +Step 46990/50462 | Loss: 2.5255 | LR: 5.99e-06 | 5179.1ms/step | 11.6 samples/s +Step 47000/50462 | Loss: 2.5053 | LR: 5.96e-06 | 5176.9ms/step | 11.6 samples/s +Validation at step 47000: Loss = 2.5048, Perplexity = 12.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-47000 +Step 47010/50462 | Loss: 2.5168 | LR: 5.93e-06 | 8707.4ms/step | 6.9 samples/s +Step 47020/50462 | Loss: 2.4936 | LR: 5.89e-06 | 5186.7ms/step | 11.6 samples/s +Step 47030/50462 | Loss: 2.5132 | LR: 5.86e-06 | 5240.4ms/step | 11.4 samples/s +Step 47040/50462 | Loss: 2.4586 | LR: 5.82e-06 | 5206.8ms/step | 11.5 samples/s +Step 47050/50462 | Loss: 2.4955 | LR: 5.79e-06 | 5205.5ms/step | 11.5 samples/s +Step 47060/50462 | Loss: 2.4934 | LR: 5.76e-06 | 5200.7ms/step | 11.5 samples/s +Step 47070/50462 | Loss: 2.4866 | LR: 5.72e-06 | 5200.7ms/step | 11.5 samples/s +Step 47080/50462 | Loss: 2.4942 | LR: 5.69e-06 | 5174.5ms/step | 11.6 samples/s +Step 47090/50462 | Loss: 2.4890 | LR: 5.65e-06 | 5190.4ms/step | 11.6 samples/s +Step 47100/50462 | Loss: 2.4929 | LR: 5.62e-06 | 5185.1ms/step | 11.6 samples/s +Step 47110/50462 | Loss: 2.4969 | LR: 5.59e-06 | 5213.4ms/step | 11.5 samples/s +Step 47120/50462 | Loss: 2.4472 | LR: 5.56e-06 | 5197.5ms/step | 11.5 samples/s +Step 47130/50462 | Loss: 2.4940 | LR: 5.52e-06 | 5202.0ms/step | 11.5 samples/s +Step 47140/50462 | Loss: 2.5065 | LR: 5.49e-06 | 5208.1ms/step | 11.5 samples/s +Step 47150/50462 | Loss: 2.4626 | LR: 5.46e-06 | 5173.6ms/step | 11.6 samples/s +Step 47160/50462 | Loss: 2.5041 | LR: 5.42e-06 | 5200.5ms/step | 11.5 samples/s +Step 47170/50462 | Loss: 2.4873 | LR: 5.39e-06 | 5184.8ms/step | 11.6 samples/s +Step 47180/50462 | Loss: 2.4973 | LR: 5.36e-06 | 5173.0ms/step | 11.6 samples/s +Step 47190/50462 | Loss: 2.4916 | LR: 5.33e-06 | 5193.3ms/step | 11.6 samples/s +Step 47200/50462 | Loss: 2.4893 | LR: 5.29e-06 | 5188.8ms/step | 11.6 samples/s +Step 47210/50462 | Loss: 2.4631 | LR: 5.26e-06 | 5203.6ms/step | 11.5 samples/s +Step 47220/50462 | Loss: 2.5042 | LR: 5.23e-06 | 5233.6ms/step | 11.5 samples/s +Step 47230/50462 | Loss: 2.5017 | LR: 5.20e-06 | 5182.9ms/step | 11.6 samples/s +Step 47240/50462 | Loss: 2.4710 | LR: 5.16e-06 | 5224.0ms/step | 11.5 samples/s +Step 47250/50462 | Loss: 2.5009 | LR: 5.13e-06 | 5167.7ms/step | 11.6 samples/s +Validation at step 47250: Loss = 2.5048, Perplexity = 12.24 +Step 47260/50462 | Loss: 2.4958 | LR: 5.10e-06 | 7725.9ms/step | 7.8 samples/s +Step 47270/50462 | Loss: 2.5072 | LR: 5.07e-06 | 5187.3ms/step | 11.6 samples/s +Step 47280/50462 | Loss: 2.4806 | LR: 5.04e-06 | 5190.9ms/step | 11.6 samples/s +Step 47290/50462 | Loss: 2.5090 | LR: 5.01e-06 | 5175.9ms/step | 11.6 samples/s +Step 47300/50462 | Loss: 2.5073 | LR: 4.97e-06 | 5173.6ms/step | 11.6 samples/s +Step 47310/50462 | Loss: 2.4698 | LR: 4.94e-06 | 5185.4ms/step | 11.6 samples/s +Step 47320/50462 | Loss: 2.4862 | LR: 4.91e-06 | 5171.5ms/step | 11.6 samples/s +Step 47330/50462 | Loss: 2.4897 | LR: 4.88e-06 | 5231.8ms/step | 11.5 samples/s +Step 47340/50462 | Loss: 2.4542 | LR: 4.85e-06 | 5171.5ms/step | 11.6 samples/s +Step 47350/50462 | Loss: 2.5018 | LR: 4.82e-06 | 5159.6ms/step | 11.6 samples/s +Step 47360/50462 | Loss: 2.4725 | LR: 4.79e-06 | 5173.0ms/step | 11.6 samples/s +Step 47370/50462 | Loss: 2.5110 | LR: 4.76e-06 | 5163.0ms/step | 11.6 samples/s +Step 47380/50462 | Loss: 2.5032 | LR: 4.73e-06 | 5183.0ms/step | 11.6 samples/s +Step 47390/50462 | Loss: 2.4857 | LR: 4.70e-06 | 5200.3ms/step | 11.5 samples/s +Step 47400/50462 | Loss: 2.4892 | LR: 4.67e-06 | 5184.6ms/step | 11.6 samples/s +Step 47410/50462 | Loss: 2.5004 | LR: 4.64e-06 | 5165.8ms/step | 11.6 samples/s +Step 47420/50462 | Loss: 2.4523 | LR: 4.61e-06 | 5167.7ms/step | 11.6 samples/s +Step 47430/50462 | Loss: 2.4803 | LR: 4.58e-06 | 5164.2ms/step | 11.6 samples/s +Step 47440/50462 | Loss: 2.4889 | LR: 4.55e-06 | 5163.8ms/step | 11.6 samples/s +Step 47450/50462 | Loss: 2.4637 | LR: 4.52e-06 | 5161.8ms/step | 11.6 samples/s +Step 47460/50462 | Loss: 2.5085 | LR: 4.49e-06 | 5151.5ms/step | 11.6 samples/s +Step 47470/50462 | Loss: 2.5061 | LR: 4.46e-06 | 5172.4ms/step | 11.6 samples/s +Step 47480/50462 | Loss: 2.5048 | LR: 4.43e-06 | 5182.2ms/step | 11.6 samples/s +Step 47490/50462 | Loss: 2.4981 | LR: 4.40e-06 | 5183.1ms/step | 11.6 samples/s +Step 47500/50462 | Loss: 2.4890 | LR: 4.37e-06 | 5169.8ms/step | 11.6 samples/s +Validation at step 47500: Loss = 2.5048, Perplexity = 12.24 +Step 47510/50462 | Loss: 2.4600 | LR: 4.34e-06 | 7722.4ms/step | 7.8 samples/s +Step 47520/50462 | Loss: 2.5080 | LR: 4.31e-06 | 5236.0ms/step | 11.5 samples/s +Step 47530/50462 | Loss: 2.5029 | LR: 4.28e-06 | 5157.6ms/step | 11.6 samples/s +Step 47540/50462 | Loss: 2.4708 | LR: 4.25e-06 | 5219.2ms/step | 11.5 samples/s +Step 47550/50462 | Loss: 2.5007 | LR: 4.22e-06 | 5219.3ms/step | 11.5 samples/s +Step 47560/50462 | Loss: 2.4864 | LR: 4.19e-06 | 5213.7ms/step | 11.5 samples/s +Step 47570/50462 | Loss: 2.4968 | LR: 4.16e-06 | 5190.2ms/step | 11.6 samples/s +Step 47580/50462 | Loss: 2.5322 | LR: 4.13e-06 | 5181.6ms/step | 11.6 samples/s +Step 47590/50462 | Loss: 2.5026 | LR: 4.11e-06 | 5203.0ms/step | 11.5 samples/s +Step 47600/50462 | Loss: 2.4937 | LR: 4.08e-06 | 5194.2ms/step | 11.6 samples/s +Step 47610/50462 | Loss: 2.4731 | LR: 4.05e-06 | 5197.8ms/step | 11.5 samples/s +Step 47620/50462 | Loss: 2.4871 | LR: 4.02e-06 | 5243.7ms/step | 11.4 samples/s +Step 47630/50462 | Loss: 2.5144 | LR: 3.99e-06 | 5199.7ms/step | 11.5 samples/s +Step 47640/50462 | Loss: 2.4698 | LR: 3.97e-06 | 5210.5ms/step | 11.5 samples/s +Step 47650/50462 | Loss: 2.4882 | LR: 3.94e-06 | 5209.7ms/step | 11.5 samples/s +Step 47660/50462 | Loss: 2.4879 | LR: 3.91e-06 | 5209.8ms/step | 11.5 samples/s +Step 47670/50462 | Loss: 2.4799 | LR: 3.88e-06 | 5204.8ms/step | 11.5 samples/s +Step 47680/50462 | Loss: 2.4980 | LR: 3.85e-06 | 5201.2ms/step | 11.5 samples/s +Step 47690/50462 | Loss: 2.4934 | LR: 3.83e-06 | 5195.1ms/step | 11.5 samples/s +Step 47700/50462 | Loss: 2.5015 | LR: 3.80e-06 | 5205.7ms/step | 11.5 samples/s +Step 47710/50462 | Loss: 2.4681 | LR: 3.77e-06 | 5196.7ms/step | 11.5 samples/s +Step 47720/50462 | Loss: 2.4946 | LR: 3.74e-06 | 5205.1ms/step | 11.5 samples/s +Step 47730/50462 | Loss: 2.4959 | LR: 3.72e-06 | 5214.4ms/step | 11.5 samples/s +Step 47740/50462 | Loss: 2.4806 | LR: 3.69e-06 | 5198.5ms/step | 11.5 samples/s +Step 47750/50462 | Loss: 2.5072 | LR: 3.66e-06 | 5208.7ms/step | 11.5 samples/s +Validation at step 47750: Loss = 2.5048, Perplexity = 12.24 +Step 47760/50462 | Loss: 2.4646 | LR: 3.64e-06 | 7766.7ms/step | 7.7 samples/s +Step 47770/50462 | Loss: 2.5274 | LR: 3.61e-06 | 5202.6ms/step | 11.5 samples/s +Step 47780/50462 | Loss: 2.4847 | LR: 3.58e-06 | 5200.7ms/step | 11.5 samples/s +Step 47790/50462 | Loss: 2.4790 | LR: 3.56e-06 | 5215.1ms/step | 11.5 samples/s +Step 47800/50462 | Loss: 2.4709 | LR: 3.53e-06 | 5206.1ms/step | 11.5 samples/s +Step 47810/50462 | Loss: 2.5045 | LR: 3.50e-06 | 5205.1ms/step | 11.5 samples/s +Step 47820/50462 | Loss: 2.4975 | LR: 3.48e-06 | 5204.7ms/step | 11.5 samples/s +Step 47830/50462 | Loss: 2.4900 | LR: 3.45e-06 | 5265.4ms/step | 11.4 samples/s +Step 47840/50462 | Loss: 2.5129 | LR: 3.42e-06 | 5202.3ms/step | 11.5 samples/s +Step 47850/50462 | Loss: 2.5031 | LR: 3.40e-06 | 5237.6ms/step | 11.5 samples/s +Step 47860/50462 | Loss: 2.5052 | LR: 3.37e-06 | 5183.5ms/step | 11.6 samples/s +Step 47870/50462 | Loss: 2.4996 | LR: 3.35e-06 | 5163.0ms/step | 11.6 samples/s +Step 47880/50462 | Loss: 2.4540 | LR: 3.32e-06 | 5172.5ms/step | 11.6 samples/s +Step 47890/50462 | Loss: 2.5120 | LR: 3.30e-06 | 5178.6ms/step | 11.6 samples/s +Step 47900/50462 | Loss: 2.4997 | LR: 3.27e-06 | 5170.5ms/step | 11.6 samples/s +Step 47910/50462 | Loss: 2.4997 | LR: 3.24e-06 | 5252.4ms/step | 11.4 samples/s +Step 47920/50462 | Loss: 2.4736 | LR: 3.22e-06 | 5212.3ms/step | 11.5 samples/s +Step 47930/50462 | Loss: 2.5108 | LR: 3.19e-06 | 5182.6ms/step | 11.6 samples/s +Step 47940/50462 | Loss: 2.4882 | LR: 3.17e-06 | 5194.5ms/step | 11.6 samples/s +Step 47950/50462 | Loss: 2.4943 | LR: 3.14e-06 | 5192.0ms/step | 11.6 samples/s +Step 47960/50462 | Loss: 2.4612 | LR: 3.12e-06 | 5213.9ms/step | 11.5 samples/s +Step 47970/50462 | Loss: 2.4978 | LR: 3.09e-06 | 5208.2ms/step | 11.5 samples/s +Step 47980/50462 | Loss: 2.4943 | LR: 3.07e-06 | 5158.7ms/step | 11.6 samples/s +Step 47990/50462 | Loss: 2.4774 | LR: 3.04e-06 | 5176.8ms/step | 11.6 samples/s +Step 48000/50462 | Loss: 2.4876 | LR: 3.02e-06 | 5183.8ms/step | 11.6 samples/s +Validation at step 48000: Loss = 2.5048, Perplexity = 12.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-48000 +Step 48010/50462 | Loss: 2.5007 | LR: 3.00e-06 | 8752.7ms/step | 6.9 samples/s +Step 48020/50462 | Loss: 2.5287 | LR: 2.97e-06 | 5155.9ms/step | 11.6 samples/s +Step 48030/50462 | Loss: 2.4702 | LR: 2.95e-06 | 5177.6ms/step | 11.6 samples/s +Step 48040/50462 | Loss: 2.4842 | LR: 2.92e-06 | 5172.5ms/step | 11.6 samples/s +Step 48050/50462 | Loss: 2.4769 | LR: 2.90e-06 | 5171.3ms/step | 11.6 samples/s +Step 48060/50462 | Loss: 2.4942 | LR: 2.87e-06 | 5193.9ms/step | 11.6 samples/s +Step 48070/50462 | Loss: 2.4968 | LR: 2.85e-06 | 5161.5ms/step | 11.6 samples/s +Step 48080/50462 | Loss: 2.5208 | LR: 2.83e-06 | 5177.3ms/step | 11.6 samples/s +Step 48090/50462 | Loss: 2.4963 | LR: 2.80e-06 | 5172.9ms/step | 11.6 samples/s +Step 48100/50462 | Loss: 2.4999 | LR: 2.78e-06 | 5170.7ms/step | 11.6 samples/s +Step 48110/50462 | Loss: 2.4892 | LR: 2.76e-06 | 5164.7ms/step | 11.6 samples/s +Step 48120/50462 | Loss: 2.4872 | LR: 2.73e-06 | 5173.8ms/step | 11.6 samples/s +Step 48130/50462 | Loss: 2.4885 | LR: 2.71e-06 | 5173.1ms/step | 11.6 samples/s +Step 48140/50462 | Loss: 2.4703 | LR: 2.69e-06 | 5220.6ms/step | 11.5 samples/s +Step 48150/50462 | Loss: 2.4834 | LR: 2.66e-06 | 5224.9ms/step | 11.5 samples/s +Step 48160/50462 | Loss: 2.5029 | LR: 2.64e-06 | 5215.3ms/step | 11.5 samples/s +Step 48170/50462 | Loss: 2.4828 | LR: 2.62e-06 | 5201.5ms/step | 11.5 samples/s +Step 48180/50462 | Loss: 2.4954 | LR: 2.60e-06 | 5208.1ms/step | 11.5 samples/s +Step 48190/50462 | Loss: 2.4914 | LR: 2.57e-06 | 5210.1ms/step | 11.5 samples/s +Step 48200/50462 | Loss: 2.5012 | LR: 2.55e-06 | 5246.8ms/step | 11.4 samples/s +Step 48210/50462 | Loss: 2.4770 | LR: 2.53e-06 | 5202.8ms/step | 11.5 samples/s +Step 48220/50462 | Loss: 2.4825 | LR: 2.51e-06 | 5209.3ms/step | 11.5 samples/s +Step 48230/50462 | Loss: 2.4899 | LR: 2.48e-06 | 5213.2ms/step | 11.5 samples/s +Step 48240/50462 | Loss: 2.4881 | LR: 2.46e-06 | 5211.1ms/step | 11.5 samples/s +Step 48250/50462 | Loss: 2.5273 | LR: 2.44e-06 | 5213.0ms/step | 11.5 samples/s +Validation at step 48250: Loss = 2.5048, Perplexity = 12.24 +Step 48260/50462 | Loss: 2.4645 | LR: 2.42e-06 | 7771.1ms/step | 7.7 samples/s +Step 48270/50462 | Loss: 2.4902 | LR: 2.39e-06 | 5210.3ms/step | 11.5 samples/s +Step 48280/50462 | Loss: 2.4982 | LR: 2.37e-06 | 5195.8ms/step | 11.5 samples/s +Step 48290/50462 | Loss: 2.4860 | LR: 2.35e-06 | 5223.5ms/step | 11.5 samples/s +Step 48300/50462 | Loss: 2.4793 | LR: 2.33e-06 | 5211.3ms/step | 11.5 samples/s +Step 48310/50462 | Loss: 2.4994 | LR: 2.31e-06 | 5208.5ms/step | 11.5 samples/s +Step 48320/50462 | Loss: 2.4982 | LR: 2.29e-06 | 5206.0ms/step | 11.5 samples/s +Step 48330/50462 | Loss: 2.4812 | LR: 2.27e-06 | 5184.7ms/step | 11.6 samples/s +Step 48340/50462 | Loss: 2.5049 | LR: 2.24e-06 | 5202.1ms/step | 11.5 samples/s +Step 48350/50462 | Loss: 2.5183 | LR: 2.22e-06 | 5204.0ms/step | 11.5 samples/s +Step 48360/50462 | Loss: 2.4743 | LR: 2.20e-06 | 5194.3ms/step | 11.6 samples/s +Step 48370/50462 | Loss: 2.5124 | LR: 2.18e-06 | 5192.6ms/step | 11.6 samples/s +Step 48380/50462 | Loss: 2.4784 | LR: 2.16e-06 | 5204.3ms/step | 11.5 samples/s +Step 48390/50462 | Loss: 2.5268 | LR: 2.14e-06 | 5189.8ms/step | 11.6 samples/s +Step 48400/50462 | Loss: 2.4811 | LR: 2.12e-06 | 5219.6ms/step | 11.5 samples/s +Step 48410/50462 | Loss: 2.5098 | LR: 2.10e-06 | 5200.8ms/step | 11.5 samples/s +Step 48420/50462 | Loss: 2.4861 | LR: 2.08e-06 | 5206.8ms/step | 11.5 samples/s +Step 48430/50462 | Loss: 2.4827 | LR: 2.06e-06 | 5208.2ms/step | 11.5 samples/s +Step 48440/50462 | Loss: 2.5002 | LR: 2.04e-06 | 5238.1ms/step | 11.5 samples/s +Step 48450/50462 | Loss: 2.4658 | LR: 2.02e-06 | 5187.7ms/step | 11.6 samples/s +Step 48460/50462 | Loss: 2.5132 | LR: 2.00e-06 | 5229.7ms/step | 11.5 samples/s +Step 48470/50462 | Loss: 2.5008 | LR: 1.98e-06 | 5162.1ms/step | 11.6 samples/s +Step 48480/50462 | Loss: 2.4844 | LR: 1.96e-06 | 5163.8ms/step | 11.6 samples/s +Step 48490/50462 | Loss: 2.4921 | LR: 1.94e-06 | 5230.6ms/step | 11.5 samples/s +Step 48500/50462 | Loss: 2.4614 | LR: 1.92e-06 | 5197.7ms/step | 11.5 samples/s +Validation at step 48500: Loss = 2.5048, Perplexity = 12.24 +Step 48510/50462 | Loss: 2.4943 | LR: 1.90e-06 | 7721.8ms/step | 7.8 samples/s +Step 48520/50462 | Loss: 2.4938 | LR: 1.88e-06 | 5186.9ms/step | 11.6 samples/s +Step 48530/50462 | Loss: 2.5110 | LR: 1.86e-06 | 5204.1ms/step | 11.5 samples/s +Step 48540/50462 | Loss: 2.4682 | LR: 1.84e-06 | 5184.9ms/step | 11.6 samples/s +Step 48550/50462 | Loss: 2.4775 | LR: 1.82e-06 | 5186.0ms/step | 11.6 samples/s +Step 48560/50462 | Loss: 2.4882 | LR: 1.80e-06 | 5193.3ms/step | 11.6 samples/s +Step 48570/50462 | Loss: 2.4897 | LR: 1.78e-06 | 5191.4ms/step | 11.6 samples/s +Step 48580/50462 | Loss: 2.4826 | LR: 1.77e-06 | 5212.3ms/step | 11.5 samples/s +Step 48590/50462 | Loss: 2.5070 | LR: 1.75e-06 | 5169.8ms/step | 11.6 samples/s +Step 48600/50462 | Loss: 2.4984 | LR: 1.73e-06 | 5183.8ms/step | 11.6 samples/s +Step 48610/50462 | Loss: 2.4841 | LR: 1.71e-06 | 5181.4ms/step | 11.6 samples/s +Step 48620/50462 | Loss: 2.4898 | LR: 1.69e-06 | 5183.7ms/step | 11.6 samples/s +Step 48630/50462 | Loss: 2.5119 | LR: 1.67e-06 | 5165.8ms/step | 11.6 samples/s +Step 48640/50462 | Loss: 2.4940 | LR: 1.66e-06 | 5208.5ms/step | 11.5 samples/s +Step 48650/50462 | Loss: 2.5105 | LR: 1.64e-06 | 5196.0ms/step | 11.5 samples/s +Step 48660/50462 | Loss: 2.5061 | LR: 1.62e-06 | 5178.1ms/step | 11.6 samples/s +Step 48670/50462 | Loss: 2.5075 | LR: 1.60e-06 | 5170.7ms/step | 11.6 samples/s +Step 48680/50462 | Loss: 2.4898 | LR: 1.58e-06 | 5183.1ms/step | 11.6 samples/s +Step 48690/50462 | Loss: 2.5035 | LR: 1.57e-06 | 5183.5ms/step | 11.6 samples/s +Step 48700/50462 | Loss: 2.4631 | LR: 1.55e-06 | 5179.1ms/step | 11.6 samples/s +Step 48710/50462 | Loss: 2.4862 | LR: 1.53e-06 | 5181.1ms/step | 11.6 samples/s +Step 48720/50462 | Loss: 2.5078 | LR: 1.51e-06 | 5189.1ms/step | 11.6 samples/s +Step 48730/50462 | Loss: 2.4936 | LR: 1.50e-06 | 5163.3ms/step | 11.6 samples/s +Step 48740/50462 | Loss: 2.4703 | LR: 1.48e-06 | 5186.0ms/step | 11.6 samples/s +Step 48750/50462 | Loss: 2.5140 | LR: 1.46e-06 | 5226.5ms/step | 11.5 samples/s +Validation at step 48750: Loss = 2.5048, Perplexity = 12.24 +Step 48760/50462 | Loss: 2.5248 | LR: 1.44e-06 | 7776.7ms/step | 7.7 samples/s +Step 48770/50462 | Loss: 2.4649 | LR: 1.43e-06 | 5180.9ms/step | 11.6 samples/s +Step 48780/50462 | Loss: 2.4872 | LR: 1.41e-06 | 5180.6ms/step | 11.6 samples/s +Step 48790/50462 | Loss: 2.4959 | LR: 1.39e-06 | 5256.3ms/step | 11.4 samples/s +Step 48800/50462 | Loss: 2.4750 | LR: 1.38e-06 | 5207.1ms/step | 11.5 samples/s +Step 48810/50462 | Loss: 2.4844 | LR: 1.36e-06 | 5178.3ms/step | 11.6 samples/s +Step 48820/50462 | Loss: 2.4896 | LR: 1.34e-06 | 5201.7ms/step | 11.5 samples/s +Step 48830/50462 | Loss: 2.5065 | LR: 1.33e-06 | 5206.8ms/step | 11.5 samples/s +Step 48840/50462 | Loss: 2.5192 | LR: 1.31e-06 | 5204.7ms/step | 11.5 samples/s +Step 48850/50462 | Loss: 2.4911 | LR: 1.30e-06 | 5200.3ms/step | 11.5 samples/s +Step 48860/50462 | Loss: 2.4568 | LR: 1.28e-06 | 5221.1ms/step | 11.5 samples/s +Step 48870/50462 | Loss: 2.4998 | LR: 1.26e-06 | 5204.1ms/step | 11.5 samples/s +Step 48880/50462 | Loss: 2.4823 | LR: 1.25e-06 | 5208.2ms/step | 11.5 samples/s +Step 48890/50462 | Loss: 2.5129 | LR: 1.23e-06 | 5196.6ms/step | 11.5 samples/s +Step 48900/50462 | Loss: 2.4909 | LR: 1.22e-06 | 5211.4ms/step | 11.5 samples/s +Step 48910/50462 | Loss: 2.5049 | LR: 1.20e-06 | 5209.2ms/step | 11.5 samples/s +Step 48920/50462 | Loss: 2.4914 | LR: 1.19e-06 | 5200.7ms/step | 11.5 samples/s +Step 48930/50462 | Loss: 2.4877 | LR: 1.17e-06 | 5207.8ms/step | 11.5 samples/s +Step 48940/50462 | Loss: 2.4682 | LR: 1.16e-06 | 5204.6ms/step | 11.5 samples/s +Step 48950/50462 | Loss: 2.4682 | LR: 1.14e-06 | 5180.3ms/step | 11.6 samples/s +Step 48960/50462 | Loss: 2.4996 | LR: 1.13e-06 | 5185.3ms/step | 11.6 samples/s +Step 48970/50462 | Loss: 2.4668 | LR: 1.11e-06 | 5171.7ms/step | 11.6 samples/s +Step 48980/50462 | Loss: 2.5349 | LR: 1.10e-06 | 5168.3ms/step | 11.6 samples/s +Step 48990/50462 | Loss: 2.5197 | LR: 1.08e-06 | 5179.3ms/step | 11.6 samples/s +Step 49000/50462 | Loss: 2.5014 | LR: 1.07e-06 | 5173.4ms/step | 11.6 samples/s +Validation at step 49000: Loss = 2.5048, Perplexity = 12.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-49000 +Step 49010/50462 | Loss: 2.4900 | LR: 1.05e-06 | 8715.7ms/step | 6.9 samples/s +Step 49020/50462 | Loss: 2.4978 | LR: 1.04e-06 | 5170.9ms/step | 11.6 samples/s +Step 49030/50462 | Loss: 2.4990 | LR: 1.02e-06 | 5160.1ms/step | 11.6 samples/s +Step 49040/50462 | Loss: 2.4951 | LR: 1.01e-06 | 5169.0ms/step | 11.6 samples/s +Step 49050/50462 | Loss: 2.4951 | LR: 9.95e-07 | 5225.9ms/step | 11.5 samples/s +Step 49060/50462 | Loss: 2.5158 | LR: 9.81e-07 | 5185.0ms/step | 11.6 samples/s +Step 49070/50462 | Loss: 2.5128 | LR: 9.67e-07 | 5295.2ms/step | 11.3 samples/s +Step 49080/50462 | Loss: 2.5147 | LR: 9.53e-07 | 5195.1ms/step | 11.5 samples/s +Step 49090/50462 | Loss: 2.4686 | LR: 9.39e-07 | 5198.3ms/step | 11.5 samples/s +Step 49100/50462 | Loss: 2.5177 | LR: 9.25e-07 | 5203.7ms/step | 11.5 samples/s +Step 49110/50462 | Loss: 2.5028 | LR: 9.12e-07 | 5186.9ms/step | 11.6 samples/s +Step 49120/50462 | Loss: 2.4722 | LR: 8.99e-07 | 5164.1ms/step | 11.6 samples/s +Step 49130/50462 | Loss: 2.5013 | LR: 8.85e-07 | 5185.9ms/step | 11.6 samples/s +Step 49140/50462 | Loss: 2.4733 | LR: 8.72e-07 | 5183.1ms/step | 11.6 samples/s +Step 49150/50462 | Loss: 2.5046 | LR: 8.59e-07 | 5199.8ms/step | 11.5 samples/s +Step 49160/50462 | Loss: 2.4944 | LR: 8.46e-07 | 5202.6ms/step | 11.5 samples/s +Step 49170/50462 | Loss: 2.4924 | LR: 8.33e-07 | 5199.8ms/step | 11.5 samples/s +Step 49180/50462 | Loss: 2.4823 | LR: 8.20e-07 | 5177.3ms/step | 11.6 samples/s +Step 49190/50462 | Loss: 2.4718 | LR: 8.07e-07 | 5188.1ms/step | 11.6 samples/s +Step 49200/50462 | Loss: 2.5251 | LR: 7.95e-07 | 5179.2ms/step | 11.6 samples/s +Step 49210/50462 | Loss: 2.4958 | LR: 7.82e-07 | 5186.4ms/step | 11.6 samples/s +Step 49220/50462 | Loss: 2.5059 | LR: 7.70e-07 | 5203.5ms/step | 11.5 samples/s +Step 49230/50462 | Loss: 2.5070 | LR: 7.57e-07 | 5172.4ms/step | 11.6 samples/s +Step 49240/50462 | Loss: 2.4977 | LR: 7.45e-07 | 5174.3ms/step | 11.6 samples/s +Step 49250/50462 | Loss: 2.4913 | LR: 7.33e-07 | 5186.9ms/step | 11.6 samples/s +Validation at step 49250: Loss = 2.5048, Perplexity = 12.24 +Step 49260/50462 | Loss: 2.4645 | LR: 7.21e-07 | 7761.1ms/step | 7.7 samples/s +Step 49270/50462 | Loss: 2.4834 | LR: 7.09e-07 | 5207.6ms/step | 11.5 samples/s +Step 49280/50462 | Loss: 2.4673 | LR: 6.97e-07 | 5211.2ms/step | 11.5 samples/s +Step 49290/50462 | Loss: 2.4887 | LR: 6.85e-07 | 5219.3ms/step | 11.5 samples/s +Step 49300/50462 | Loss: 2.4861 | LR: 6.74e-07 | 5200.0ms/step | 11.5 samples/s +Step 49310/50462 | Loss: 2.4923 | LR: 6.62e-07 | 5217.7ms/step | 11.5 samples/s +Step 49320/50462 | Loss: 2.4807 | LR: 6.51e-07 | 5205.4ms/step | 11.5 samples/s +Step 49330/50462 | Loss: 2.4891 | LR: 6.39e-07 | 5201.4ms/step | 11.5 samples/s +Step 49340/50462 | Loss: 2.4990 | LR: 6.28e-07 | 5198.8ms/step | 11.5 samples/s +Step 49350/50462 | Loss: 2.5018 | LR: 6.17e-07 | 5202.2ms/step | 11.5 samples/s +Step 49360/50462 | Loss: 2.5120 | LR: 6.06e-07 | 5254.6ms/step | 11.4 samples/s +Step 49370/50462 | Loss: 2.4944 | LR: 5.95e-07 | 5281.1ms/step | 11.4 samples/s +Step 49380/50462 | Loss: 2.5072 | LR: 5.84e-07 | 5204.4ms/step | 11.5 samples/s +Step 49390/50462 | Loss: 2.5067 | LR: 5.73e-07 | 5189.2ms/step | 11.6 samples/s +Step 49400/50462 | Loss: 2.5044 | LR: 5.63e-07 | 5201.3ms/step | 11.5 samples/s +Step 49410/50462 | Loss: 2.4963 | LR: 5.52e-07 | 5205.5ms/step | 11.5 samples/s +Step 49420/50462 | Loss: 2.4938 | LR: 5.42e-07 | 5201.7ms/step | 11.5 samples/s +Step 49430/50462 | Loss: 2.4733 | LR: 5.31e-07 | 5212.6ms/step | 11.5 samples/s +Step 49440/50462 | Loss: 2.4831 | LR: 5.21e-07 | 5194.1ms/step | 11.6 samples/s +Step 49450/50462 | Loss: 2.4697 | LR: 5.11e-07 | 5198.4ms/step | 11.5 samples/s +Step 49460/50462 | Loss: 2.5039 | LR: 5.01e-07 | 5198.0ms/step | 11.5 samples/s +Step 49470/50462 | Loss: 2.5056 | LR: 4.91e-07 | 5208.5ms/step | 11.5 samples/s +Step 49480/50462 | Loss: 2.4989 | LR: 4.81e-07 | 5201.2ms/step | 11.5 samples/s +Step 49490/50462 | Loss: 2.4792 | LR: 4.72e-07 | 5228.3ms/step | 11.5 samples/s +Step 49500/50462 | Loss: 2.5000 | LR: 4.62e-07 | 5213.2ms/step | 11.5 samples/s +Validation at step 49500: Loss = 2.5048, Perplexity = 12.24 +Step 49510/50462 | Loss: 2.5094 | LR: 4.52e-07 | 7766.6ms/step | 7.7 samples/s +Step 49520/50462 | Loss: 2.4892 | LR: 4.43e-07 | 5214.1ms/step | 11.5 samples/s +Step 49530/50462 | Loss: 2.4930 | LR: 4.34e-07 | 5219.1ms/step | 11.5 samples/s +Step 49540/50462 | Loss: 2.4851 | LR: 4.24e-07 | 5219.1ms/step | 11.5 samples/s +Step 49550/50462 | Loss: 2.4914 | LR: 4.15e-07 | 5211.8ms/step | 11.5 samples/s +Step 49560/50462 | Loss: 2.5069 | LR: 4.06e-07 | 5189.7ms/step | 11.6 samples/s +Step 49570/50462 | Loss: 2.5093 | LR: 3.97e-07 | 5196.5ms/step | 11.5 samples/s +Step 49580/50462 | Loss: 2.5088 | LR: 3.88e-07 | 5215.3ms/step | 11.5 samples/s +Step 49590/50462 | Loss: 2.4824 | LR: 3.79e-07 | 5231.3ms/step | 11.5 samples/s +Step 49600/50462 | Loss: 2.5132 | LR: 3.71e-07 | 5214.2ms/step | 11.5 samples/s +Step 49610/50462 | Loss: 2.4931 | LR: 3.62e-07 | 5218.9ms/step | 11.5 samples/s +Step 49620/50462 | Loss: 2.4921 | LR: 3.54e-07 | 5213.7ms/step | 11.5 samples/s +Step 49630/50462 | Loss: 2.4922 | LR: 3.45e-07 | 5202.0ms/step | 11.5 samples/s +Step 49640/50462 | Loss: 2.4949 | LR: 3.37e-07 | 5208.8ms/step | 11.5 samples/s +Step 49650/50462 | Loss: 2.4983 | LR: 3.29e-07 | 5212.5ms/step | 11.5 samples/s +Step 49660/50462 | Loss: 2.4813 | LR: 3.21e-07 | 5264.4ms/step | 11.4 samples/s +Step 49670/50462 | Loss: 2.4683 | LR: 3.13e-07 | 5246.9ms/step | 11.4 samples/s +Step 49680/50462 | Loss: 2.5133 | LR: 3.05e-07 | 5250.4ms/step | 11.4 samples/s +Step 49690/50462 | Loss: 2.5307 | LR: 2.97e-07 | 5206.3ms/step | 11.5 samples/s +Step 49700/50462 | Loss: 2.5030 | LR: 2.90e-07 | 5206.5ms/step | 11.5 samples/s +Step 49710/50462 | Loss: 2.4964 | LR: 2.82e-07 | 5202.6ms/step | 11.5 samples/s +Step 49720/50462 | Loss: 2.4867 | LR: 2.75e-07 | 5209.3ms/step | 11.5 samples/s +Step 49730/50462 | Loss: 2.5033 | LR: 2.67e-07 | 5211.5ms/step | 11.5 samples/s +Step 49740/50462 | Loss: 2.5229 | LR: 2.60e-07 | 5207.6ms/step | 11.5 samples/s +Step 49750/50462 | Loss: 2.4949 | LR: 2.53e-07 | 5204.6ms/step | 11.5 samples/s +Validation at step 49750: Loss = 2.5048, Perplexity = 12.24 +Step 49760/50462 | Loss: 2.5122 | LR: 2.46e-07 | 7766.1ms/step | 7.7 samples/s +Step 49770/50462 | Loss: 2.5014 | LR: 2.39e-07 | 5208.3ms/step | 11.5 samples/s +Step 49780/50462 | Loss: 2.4859 | LR: 2.32e-07 | 5203.7ms/step | 11.5 samples/s +Step 49790/50462 | Loss: 2.4796 | LR: 2.25e-07 | 5201.7ms/step | 11.5 samples/s +Step 49800/50462 | Loss: 2.5217 | LR: 2.19e-07 | 5198.6ms/step | 11.5 samples/s +Step 49810/50462 | Loss: 2.5085 | LR: 2.12e-07 | 5208.5ms/step | 11.5 samples/s +Step 49820/50462 | Loss: 2.5062 | LR: 2.06e-07 | 5196.0ms/step | 11.5 samples/s +Step 49830/50462 | Loss: 2.4951 | LR: 1.99e-07 | 5202.0ms/step | 11.5 samples/s +Step 49840/50462 | Loss: 2.5099 | LR: 1.93e-07 | 5208.0ms/step | 11.5 samples/s +Step 49850/50462 | Loss: 2.4864 | LR: 1.87e-07 | 5214.4ms/step | 11.5 samples/s +Step 49860/50462 | Loss: 2.4863 | LR: 1.81e-07 | 5209.4ms/step | 11.5 samples/s +Step 49870/50462 | Loss: 2.5005 | LR: 1.75e-07 | 5200.5ms/step | 11.5 samples/s +Step 49880/50462 | Loss: 2.4865 | LR: 1.69e-07 | 5217.1ms/step | 11.5 samples/s +Step 49890/50462 | Loss: 2.4959 | LR: 1.63e-07 | 5195.7ms/step | 11.5 samples/s +Step 49900/50462 | Loss: 2.5084 | LR: 1.58e-07 | 5207.5ms/step | 11.5 samples/s +Step 49910/50462 | Loss: 2.4943 | LR: 1.52e-07 | 5214.3ms/step | 11.5 samples/s +Step 49920/50462 | Loss: 2.5087 | LR: 1.47e-07 | 5206.3ms/step | 11.5 samples/s +Step 49930/50462 | Loss: 2.4796 | LR: 1.41e-07 | 5199.5ms/step | 11.5 samples/s +Step 49940/50462 | Loss: 2.4918 | LR: 1.36e-07 | 5213.2ms/step | 11.5 samples/s diff --git a/epfl_final/log_files/cont_pretrain_40k.log b/epfl_final/log_files/cont_pretrain_40k.log new file mode 100644 index 0000000000000000000000000000000000000000..db98ad24199293e5a0166aadebb0ba7439558ad4 --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_40k.log @@ -0,0 +1,1054 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260601_062617 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260601_062617", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-20000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 40k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-20000 +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-40000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 40000 (epoch 0, micro_step 400000) +Resuming training from step 40000 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run o2l7qkqb +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260601_064405-o2l7qkqb +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense continued 40k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/o2l7qkqb +Starting training... +Validation at step 40000: Loss = 2.5242, Perplexity = 12.48 +Switching attention mode for step 40001: dense (dense step interval) +Step 40010/50462 | Loss: 2.5457 | LR: 5.26e-05 | 59027.5ms/step | 1.0 samples/s +Step 40020/50462 | Loss: 2.5522 | LR: 5.25e-05 | 5181.0ms/step | 11.6 samples/s +Step 40030/50462 | Loss: 2.4929 | LR: 5.24e-05 | 5144.6ms/step | 11.7 samples/s +Step 40040/50462 | Loss: 2.5234 | LR: 5.23e-05 | 5136.0ms/step | 11.7 samples/s +Step 40050/50462 | Loss: 2.5101 | LR: 5.22e-05 | 5144.4ms/step | 11.7 samples/s +Step 40060/50462 | Loss: 2.5036 | LR: 5.21e-05 | 5150.3ms/step | 11.6 samples/s +Step 40070/50462 | Loss: 2.5129 | LR: 5.20e-05 | 5146.6ms/step | 11.7 samples/s +Step 40080/50462 | Loss: 2.4868 | LR: 5.19e-05 | 5146.8ms/step | 11.7 samples/s +Step 40090/50462 | Loss: 2.5215 | LR: 5.18e-05 | 5145.2ms/step | 11.7 samples/s +Step 40100/50462 | Loss: 2.5162 | LR: 5.17e-05 | 5138.6ms/step | 11.7 samples/s +Step 40110/50462 | Loss: 2.5050 | LR: 5.16e-05 | 5158.1ms/step | 11.6 samples/s +Step 40120/50462 | Loss: 2.5181 | LR: 5.15e-05 | 5153.1ms/step | 11.6 samples/s +Step 40130/50462 | Loss: 2.5002 | LR: 5.14e-05 | 5154.3ms/step | 11.6 samples/s +Step 40140/50462 | Loss: 2.4977 | LR: 5.13e-05 | 5150.9ms/step | 11.6 samples/s +Step 40150/50462 | Loss: 2.4753 | LR: 5.12e-05 | 5140.0ms/step | 11.7 samples/s +Step 40160/50462 | Loss: 2.4735 | LR: 5.11e-05 | 5148.0ms/step | 11.7 samples/s +Step 40170/50462 | Loss: 2.5115 | LR: 5.10e-05 | 5204.8ms/step | 11.5 samples/s +Step 40180/50462 | Loss: 2.5118 | LR: 5.09e-05 | 5174.2ms/step | 11.6 samples/s +Step 40190/50462 | Loss: 2.5083 | LR: 5.09e-05 | 5191.9ms/step | 11.6 samples/s +Step 40200/50462 | Loss: 2.5179 | LR: 5.08e-05 | 5170.1ms/step | 11.6 samples/s +Step 40210/50462 | Loss: 2.5223 | LR: 5.07e-05 | 5178.8ms/step | 11.6 samples/s +Step 40220/50462 | Loss: 2.5235 | LR: 5.06e-05 | 5176.5ms/step | 11.6 samples/s +Step 40230/50462 | Loss: 2.5141 | LR: 5.05e-05 | 5169.1ms/step | 11.6 samples/s +Step 40240/50462 | Loss: 2.5448 | LR: 5.04e-05 | 5179.6ms/step | 11.6 samples/s +Step 40250/50462 | Loss: 2.5040 | LR: 5.03e-05 | 5176.9ms/step | 11.6 samples/s +Validation at step 40250: Loss = 2.5156, Perplexity = 12.37 +Step 40260/50462 | Loss: 2.5107 | LR: 5.02e-05 | 7743.7ms/step | 7.7 samples/s +Step 40270/50462 | Loss: 2.4858 | LR: 5.01e-05 | 5152.6ms/step | 11.6 samples/s +Step 40280/50462 | Loss: 2.5092 | LR: 5.00e-05 | 5156.3ms/step | 11.6 samples/s +Step 40290/50462 | Loss: 2.5077 | LR: 4.99e-05 | 5168.6ms/step | 11.6 samples/s +Step 40300/50462 | Loss: 2.5032 | LR: 4.98e-05 | 5199.8ms/step | 11.5 samples/s +Step 40310/50462 | Loss: 2.4889 | LR: 4.97e-05 | 5169.4ms/step | 11.6 samples/s +Step 40320/50462 | Loss: 2.5072 | LR: 4.96e-05 | 5158.6ms/step | 11.6 samples/s +Step 40330/50462 | Loss: 2.5301 | LR: 4.95e-05 | 5166.1ms/step | 11.6 samples/s +Step 40340/50462 | Loss: 2.4905 | LR: 4.94e-05 | 5164.1ms/step | 11.6 samples/s +Step 40350/50462 | Loss: 2.4801 | LR: 4.93e-05 | 5175.1ms/step | 11.6 samples/s +Step 40360/50462 | Loss: 2.5102 | LR: 4.92e-05 | 5160.2ms/step | 11.6 samples/s +Step 40370/50462 | Loss: 2.5229 | LR: 4.91e-05 | 5163.1ms/step | 11.6 samples/s +Step 40380/50462 | Loss: 2.5087 | LR: 4.90e-05 | 5160.0ms/step | 11.6 samples/s +Step 40390/50462 | Loss: 2.5041 | LR: 4.90e-05 | 5165.9ms/step | 11.6 samples/s +Step 40400/50462 | Loss: 2.5157 | LR: 4.89e-05 | 5176.0ms/step | 11.6 samples/s +Step 40410/50462 | Loss: 2.5304 | LR: 4.88e-05 | 5175.0ms/step | 11.6 samples/s +Step 40420/50462 | Loss: 2.5288 | LR: 4.87e-05 | 5160.9ms/step | 11.6 samples/s +Step 40430/50462 | Loss: 2.4958 | LR: 4.86e-05 | 5199.4ms/step | 11.5 samples/s +Step 40440/50462 | Loss: 2.5184 | LR: 4.85e-05 | 5159.9ms/step | 11.6 samples/s +Step 40450/50462 | Loss: 2.5046 | LR: 4.84e-05 | 5145.1ms/step | 11.7 samples/s +Step 40460/50462 | Loss: 2.5366 | LR: 4.83e-05 | 5149.0ms/step | 11.7 samples/s +Step 40470/50462 | Loss: 2.4913 | LR: 4.82e-05 | 5145.6ms/step | 11.7 samples/s +Step 40480/50462 | Loss: 2.5023 | LR: 4.81e-05 | 5149.9ms/step | 11.7 samples/s +Step 40490/50462 | Loss: 2.5178 | LR: 4.80e-05 | 5140.5ms/step | 11.7 samples/s +Step 40500/50462 | Loss: 2.5079 | LR: 4.79e-05 | 5158.5ms/step | 11.6 samples/s +Validation at step 40500: Loss = 2.5150, Perplexity = 12.37 +Step 40510/50462 | Loss: 2.5065 | LR: 4.78e-05 | 7710.7ms/step | 7.8 samples/s +Step 40520/50462 | Loss: 2.4928 | LR: 4.77e-05 | 5161.4ms/step | 11.6 samples/s +Step 40530/50462 | Loss: 2.4956 | LR: 4.76e-05 | 5195.4ms/step | 11.5 samples/s +Step 40540/50462 | Loss: 2.4691 | LR: 4.76e-05 | 5153.7ms/step | 11.6 samples/s +Step 40550/50462 | Loss: 2.5244 | LR: 4.75e-05 | 5145.9ms/step | 11.7 samples/s +Step 40560/50462 | Loss: 2.5154 | LR: 4.74e-05 | 5149.2ms/step | 11.7 samples/s +Step 40570/50462 | Loss: 2.4906 | LR: 4.73e-05 | 5193.1ms/step | 11.6 samples/s +Step 40580/50462 | Loss: 2.5179 | LR: 4.72e-05 | 5157.6ms/step | 11.6 samples/s +Step 40590/50462 | Loss: 2.5019 | LR: 4.71e-05 | 5148.9ms/step | 11.7 samples/s +Step 40600/50462 | Loss: 2.5031 | LR: 4.70e-05 | 5149.1ms/step | 11.7 samples/s +Step 40610/50462 | Loss: 2.5216 | LR: 4.69e-05 | 5128.4ms/step | 11.7 samples/s +Step 40620/50462 | Loss: 2.4782 | LR: 4.68e-05 | 5136.6ms/step | 11.7 samples/s +Step 40630/50462 | Loss: 2.5000 | LR: 4.67e-05 | 5127.1ms/step | 11.7 samples/s +Step 40640/50462 | Loss: 2.5118 | LR: 4.66e-05 | 5135.5ms/step | 11.7 samples/s +Step 40650/50462 | Loss: 2.5093 | LR: 4.65e-05 | 5121.0ms/step | 11.7 samples/s +Step 40660/50462 | Loss: 2.5216 | LR: 4.64e-05 | 5127.8ms/step | 11.7 samples/s +Step 40670/50462 | Loss: 2.4989 | LR: 4.64e-05 | 5146.8ms/step | 11.7 samples/s +Step 40680/50462 | Loss: 2.5039 | LR: 4.63e-05 | 5156.4ms/step | 11.6 samples/s +Step 40690/50462 | Loss: 2.5001 | LR: 4.62e-05 | 5155.7ms/step | 11.6 samples/s +Step 40700/50462 | Loss: 2.4719 | LR: 4.61e-05 | 5197.2ms/step | 11.5 samples/s +Step 40710/50462 | Loss: 2.5122 | LR: 4.60e-05 | 5156.6ms/step | 11.6 samples/s +Step 40720/50462 | Loss: 2.4788 | LR: 4.59e-05 | 5153.1ms/step | 11.6 samples/s +Step 40730/50462 | Loss: 2.4901 | LR: 4.58e-05 | 5155.2ms/step | 11.6 samples/s +Step 40740/50462 | Loss: 2.4864 | LR: 4.57e-05 | 5152.4ms/step | 11.6 samples/s +Step 40750/50462 | Loss: 2.4964 | LR: 4.56e-05 | 5154.2ms/step | 11.6 samples/s +Validation at step 40750: Loss = 2.5144, Perplexity = 12.36 +Step 40760/50462 | Loss: 2.5211 | LR: 4.55e-05 | 7708.5ms/step | 7.8 samples/s +Step 40770/50462 | Loss: 2.5020 | LR: 4.54e-05 | 5161.6ms/step | 11.6 samples/s +Step 40780/50462 | Loss: 2.5177 | LR: 4.54e-05 | 5157.9ms/step | 11.6 samples/s +Step 40790/50462 | Loss: 2.5024 | LR: 4.53e-05 | 5150.3ms/step | 11.6 samples/s +Step 40800/50462 | Loss: 2.5103 | LR: 4.52e-05 | 5159.7ms/step | 11.6 samples/s +Step 40810/50462 | Loss: 2.4689 | LR: 4.51e-05 | 5190.0ms/step | 11.6 samples/s +Step 40820/50462 | Loss: 2.5425 | LR: 4.50e-05 | 5153.0ms/step | 11.6 samples/s +Step 40830/50462 | Loss: 2.4560 | LR: 4.49e-05 | 5159.6ms/step | 11.6 samples/s +Step 40840/50462 | Loss: 2.4798 | LR: 4.48e-05 | 5200.4ms/step | 11.5 samples/s +Step 40850/50462 | Loss: 2.5234 | LR: 4.47e-05 | 5161.4ms/step | 11.6 samples/s +Step 40860/50462 | Loss: 2.4707 | LR: 4.46e-05 | 5151.7ms/step | 11.6 samples/s +Step 40870/50462 | Loss: 2.5245 | LR: 4.45e-05 | 5166.3ms/step | 11.6 samples/s +Step 40880/50462 | Loss: 2.5316 | LR: 4.45e-05 | 5164.8ms/step | 11.6 samples/s +Step 40890/50462 | Loss: 2.4653 | LR: 4.44e-05 | 5162.8ms/step | 11.6 samples/s +Step 40900/50462 | Loss: 2.4903 | LR: 4.43e-05 | 5161.1ms/step | 11.6 samples/s +Step 40910/50462 | Loss: 2.5100 | LR: 4.42e-05 | 5168.1ms/step | 11.6 samples/s +Step 40920/50462 | Loss: 2.5193 | LR: 4.41e-05 | 5152.9ms/step | 11.6 samples/s +Step 40930/50462 | Loss: 2.5103 | LR: 4.40e-05 | 5163.4ms/step | 11.6 samples/s +Step 40940/50462 | Loss: 2.5274 | LR: 4.39e-05 | 5160.6ms/step | 11.6 samples/s +Step 40950/50462 | Loss: 2.4808 | LR: 4.38e-05 | 5158.2ms/step | 11.6 samples/s +Step 40960/50462 | Loss: 2.5118 | LR: 4.37e-05 | 5198.4ms/step | 11.5 samples/s +Step 40970/50462 | Loss: 2.5009 | LR: 4.36e-05 | 5162.9ms/step | 11.6 samples/s +Step 40980/50462 | Loss: 2.5012 | LR: 4.36e-05 | 5159.1ms/step | 11.6 samples/s +Step 40990/50462 | Loss: 2.4704 | LR: 4.35e-05 | 5163.5ms/step | 11.6 samples/s +Step 41000/50462 | Loss: 2.5149 | LR: 4.34e-05 | 5168.9ms/step | 11.6 samples/s +Validation at step 41000: Loss = 2.5141, Perplexity = 12.36 +Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-41000 +Step 41010/50462 | Loss: 2.5218 | LR: 4.33e-05 | 8630.8ms/step | 7.0 samples/s +Step 41020/50462 | Loss: 2.5162 | LR: 4.32e-05 | 5164.6ms/step | 11.6 samples/s +Step 41030/50462 | Loss: 2.5194 | LR: 4.31e-05 | 5170.4ms/step | 11.6 samples/s +Step 41040/50462 | Loss: 2.4829 | LR: 4.30e-05 | 5175.8ms/step | 11.6 samples/s +Step 41050/50462 | Loss: 2.5134 | LR: 4.29e-05 | 5181.1ms/step | 11.6 samples/s +Step 41060/50462 | Loss: 2.5192 | LR: 4.28e-05 | 5162.8ms/step | 11.6 samples/s +Step 41070/50462 | Loss: 2.4973 | LR: 4.28e-05 | 5149.7ms/step | 11.7 samples/s +Step 41080/50462 | Loss: 2.4898 | LR: 4.27e-05 | 5192.4ms/step | 11.6 samples/s +Step 41090/50462 | Loss: 2.4681 | LR: 4.26e-05 | 5157.7ms/step | 11.6 samples/s +Step 41100/50462 | Loss: 2.5034 | LR: 4.25e-05 | 5159.6ms/step | 11.6 samples/s +Step 41110/50462 | Loss: 2.5072 | LR: 4.24e-05 | 5158.6ms/step | 11.6 samples/s +Step 41120/50462 | Loss: 2.4971 | LR: 4.23e-05 | 5206.1ms/step | 11.5 samples/s +Step 41130/50462 | Loss: 2.5207 | LR: 4.22e-05 | 5163.8ms/step | 11.6 samples/s +Step 41140/50462 | Loss: 2.4933 | LR: 4.21e-05 | 5154.7ms/step | 11.6 samples/s +Step 41150/50462 | Loss: 2.5077 | LR: 4.21e-05 | 5152.6ms/step | 11.6 samples/s +Step 41160/50462 | Loss: 2.5045 | LR: 4.20e-05 | 5167.9ms/step | 11.6 samples/s +Step 41170/50462 | Loss: 2.5096 | LR: 4.19e-05 | 5169.6ms/step | 11.6 samples/s +Step 41180/50462 | Loss: 2.4950 | LR: 4.18e-05 | 5162.6ms/step | 11.6 samples/s +Step 41190/50462 | Loss: 2.4545 | LR: 4.17e-05 | 5163.8ms/step | 11.6 samples/s +Step 41200/50462 | Loss: 2.4911 | LR: 4.16e-05 | 5151.3ms/step | 11.6 samples/s +Step 41210/50462 | Loss: 2.4682 | LR: 4.15e-05 | 5165.6ms/step | 11.6 samples/s +Step 41220/50462 | Loss: 2.4851 | LR: 4.14e-05 | 5211.2ms/step | 11.5 samples/s +Step 41230/50462 | Loss: 2.5065 | LR: 4.14e-05 | 5149.2ms/step | 11.7 samples/s +Step 41240/50462 | Loss: 2.4966 | LR: 4.13e-05 | 5161.5ms/step | 11.6 samples/s +Step 41250/50462 | Loss: 2.5110 | LR: 4.12e-05 | 5177.9ms/step | 11.6 samples/s +Validation at step 41250: Loss = 2.5138, Perplexity = 12.35 +Step 41260/50462 | Loss: 2.5067 | LR: 4.11e-05 | 7728.5ms/step | 7.8 samples/s +Step 41270/50462 | Loss: 2.4809 | LR: 4.10e-05 | 5175.8ms/step | 11.6 samples/s +Step 41280/50462 | Loss: 2.4587 | LR: 4.09e-05 | 5151.2ms/step | 11.6 samples/s +Step 41290/50462 | Loss: 2.5209 | LR: 4.08e-05 | 5158.9ms/step | 11.6 samples/s +Step 41300/50462 | Loss: 2.5103 | LR: 4.07e-05 | 5155.7ms/step | 11.6 samples/s +Step 41310/50462 | Loss: 2.4724 | LR: 4.07e-05 | 5172.3ms/step | 11.6 samples/s +Step 41320/50462 | Loss: 2.5552 | LR: 4.06e-05 | 5156.7ms/step | 11.6 samples/s +Step 41330/50462 | Loss: 2.4914 | LR: 4.05e-05 | 5168.5ms/step | 11.6 samples/s +Step 41340/50462 | Loss: 2.4843 | LR: 4.04e-05 | 5146.6ms/step | 11.7 samples/s +Step 41350/50462 | Loss: 2.5042 | LR: 4.03e-05 | 5204.3ms/step | 11.5 samples/s +Step 41360/50462 | Loss: 2.5318 | LR: 4.02e-05 | 5162.0ms/step | 11.6 samples/s +Step 41370/50462 | Loss: 2.5036 | LR: 4.01e-05 | 5145.6ms/step | 11.7 samples/s +Step 41380/50462 | Loss: 2.4951 | LR: 4.01e-05 | 5165.5ms/step | 11.6 samples/s +Step 41390/50462 | Loss: 2.4720 | LR: 4.00e-05 | 5204.3ms/step | 11.5 samples/s +Step 41400/50462 | Loss: 2.5110 | LR: 3.99e-05 | 5156.0ms/step | 11.6 samples/s +Step 41410/50462 | Loss: 2.5178 | LR: 3.98e-05 | 5154.5ms/step | 11.6 samples/s +Step 41420/50462 | Loss: 2.4816 | LR: 3.97e-05 | 5165.7ms/step | 11.6 samples/s +Step 41430/50462 | Loss: 2.5131 | LR: 3.96e-05 | 5159.2ms/step | 11.6 samples/s +Step 41440/50462 | Loss: 2.4809 | LR: 3.95e-05 | 5148.4ms/step | 11.7 samples/s +Step 41450/50462 | Loss: 2.5018 | LR: 3.95e-05 | 5159.8ms/step | 11.6 samples/s +Step 41460/50462 | Loss: 2.4831 | LR: 3.94e-05 | 5162.8ms/step | 11.6 samples/s +Step 41470/50462 | Loss: 2.4636 | LR: 3.93e-05 | 5168.3ms/step | 11.6 samples/s +Step 41480/50462 | Loss: 2.5206 | LR: 3.92e-05 | 5200.9ms/step | 11.5 samples/s +Step 41490/50462 | Loss: 2.4754 | LR: 3.91e-05 | 5163.8ms/step | 11.6 samples/s +Step 41500/50462 | Loss: 2.5153 | LR: 3.90e-05 | 5167.1ms/step | 11.6 samples/s +Validation at step 41500: Loss = 2.5136, Perplexity = 12.35 +Step 41510/50462 | Loss: 2.4862 | LR: 3.90e-05 | 7719.3ms/step | 7.8 samples/s +Step 41520/50462 | Loss: 2.5013 | LR: 3.89e-05 | 5170.7ms/step | 11.6 samples/s +Step 41530/50462 | Loss: 2.5131 | LR: 3.88e-05 | 5175.4ms/step | 11.6 samples/s +Step 41540/50462 | Loss: 2.4814 | LR: 3.87e-05 | 5161.2ms/step | 11.6 samples/s +Step 41550/50462 | Loss: 2.5082 | LR: 3.86e-05 | 5180.6ms/step | 11.6 samples/s +Step 41560/50462 | Loss: 2.5193 | LR: 3.85e-05 | 5168.4ms/step | 11.6 samples/s +Step 41570/50462 | Loss: 2.5067 | LR: 3.84e-05 | 5151.7ms/step | 11.6 samples/s +Step 41580/50462 | Loss: 2.5125 | LR: 3.84e-05 | 5171.4ms/step | 11.6 samples/s +Step 41590/50462 | Loss: 2.4951 | LR: 3.83e-05 | 5169.3ms/step | 11.6 samples/s +Step 41600/50462 | Loss: 2.5089 | LR: 3.82e-05 | 5174.7ms/step | 11.6 samples/s +Step 41610/50462 | Loss: 2.4908 | LR: 3.81e-05 | 5156.6ms/step | 11.6 samples/s +Step 41620/50462 | Loss: 2.4971 | LR: 3.80e-05 | 5183.9ms/step | 11.6 samples/s +Step 41630/50462 | Loss: 2.4785 | LR: 3.79e-05 | 5208.6ms/step | 11.5 samples/s +Step 41640/50462 | Loss: 2.4947 | LR: 3.79e-05 | 5162.4ms/step | 11.6 samples/s +Step 41650/50462 | Loss: 2.4800 | LR: 3.78e-05 | 5168.0ms/step | 11.6 samples/s +Step 41660/50462 | Loss: 2.5093 | LR: 3.77e-05 | 5216.5ms/step | 11.5 samples/s +Step 41670/50462 | Loss: 2.5016 | LR: 3.76e-05 | 5170.0ms/step | 11.6 samples/s +Step 41680/50462 | Loss: 2.4876 | LR: 3.75e-05 | 5180.7ms/step | 11.6 samples/s +Step 41690/50462 | Loss: 2.4838 | LR: 3.74e-05 | 5169.9ms/step | 11.6 samples/s +Step 41700/50462 | Loss: 2.4951 | LR: 3.74e-05 | 5178.2ms/step | 11.6 samples/s +Step 41710/50462 | Loss: 2.4876 | LR: 3.73e-05 | 5165.2ms/step | 11.6 samples/s +Step 41720/50462 | Loss: 2.4835 | LR: 3.72e-05 | 5166.5ms/step | 11.6 samples/s +Step 41730/50462 | Loss: 2.4901 | LR: 3.71e-05 | 5177.8ms/step | 11.6 samples/s +Step 41740/50462 | Loss: 2.4828 | LR: 3.70e-05 | 5216.7ms/step | 11.5 samples/s +Step 41750/50462 | Loss: 2.4892 | LR: 3.69e-05 | 5159.7ms/step | 11.6 samples/s +Validation at step 41750: Loss = 2.5134, Perplexity = 12.35 +Step 41760/50462 | Loss: 2.5022 | LR: 3.69e-05 | 7700.7ms/step | 7.8 samples/s +Step 41770/50462 | Loss: 2.5089 | LR: 3.68e-05 | 5165.7ms/step | 11.6 samples/s +Step 41780/50462 | Loss: 2.4956 | LR: 3.67e-05 | 5164.8ms/step | 11.6 samples/s +Step 41790/50462 | Loss: 2.4814 | LR: 3.66e-05 | 5156.2ms/step | 11.6 samples/s +Step 41800/50462 | Loss: 2.4852 | LR: 3.65e-05 | 5155.4ms/step | 11.6 samples/s +Step 41810/50462 | Loss: 2.5201 | LR: 3.64e-05 | 5169.0ms/step | 11.6 samples/s +Step 41820/50462 | Loss: 2.4454 | LR: 3.64e-05 | 5153.5ms/step | 11.6 samples/s +Step 41830/50462 | Loss: 2.4929 | LR: 3.63e-05 | 5167.6ms/step | 11.6 samples/s +Step 41840/50462 | Loss: 2.4673 | LR: 3.62e-05 | 5171.4ms/step | 11.6 samples/s +Step 41850/50462 | Loss: 2.5099 | LR: 3.61e-05 | 5173.1ms/step | 11.6 samples/s +Step 41860/50462 | Loss: 2.4950 | LR: 3.60e-05 | 5189.0ms/step | 11.6 samples/s +Step 41870/50462 | Loss: 2.4939 | LR: 3.60e-05 | 5166.1ms/step | 11.6 samples/s +Step 41880/50462 | Loss: 2.5050 | LR: 3.59e-05 | 5187.6ms/step | 11.6 samples/s +Step 41890/50462 | Loss: 2.4969 | LR: 3.58e-05 | 5173.5ms/step | 11.6 samples/s +Step 41900/50462 | Loss: 2.5220 | LR: 3.57e-05 | 5216.8ms/step | 11.5 samples/s +Step 41910/50462 | Loss: 2.5135 | LR: 3.56e-05 | 5160.4ms/step | 11.6 samples/s +Step 41920/50462 | Loss: 2.5040 | LR: 3.55e-05 | 5151.6ms/step | 11.6 samples/s +Step 41930/50462 | Loss: 2.4838 | LR: 3.55e-05 | 5149.4ms/step | 11.7 samples/s +Step 41940/50462 | Loss: 2.5116 | LR: 3.54e-05 | 5218.6ms/step | 11.5 samples/s +Step 41950/50462 | Loss: 2.4828 | LR: 3.53e-05 | 5159.9ms/step | 11.6 samples/s +Step 41960/50462 | Loss: 2.5106 | LR: 3.52e-05 | 5172.8ms/step | 11.6 samples/s +Step 41970/50462 | Loss: 2.4976 | LR: 3.51e-05 | 5151.7ms/step | 11.6 samples/s +Step 41980/50462 | Loss: 2.4961 | LR: 3.51e-05 | 5148.7ms/step | 11.7 samples/s +Step 41990/50462 | Loss: 2.5281 | LR: 3.50e-05 | 5176.2ms/step | 11.6 samples/s +Step 42000/50462 | Loss: 2.5067 | LR: 3.49e-05 | 5142.5ms/step | 11.7 samples/s +Validation at step 42000: Loss = 2.5132, Perplexity = 12.34 +Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-42000 +Step 42010/50462 | Loss: 2.4752 | LR: 3.48e-05 | 8679.3ms/step | 6.9 samples/s +Step 42020/50462 | Loss: 2.5166 | LR: 3.47e-05 | 5136.5ms/step | 11.7 samples/s +Step 42030/50462 | Loss: 2.5355 | LR: 3.47e-05 | 5138.7ms/step | 11.7 samples/s +Step 42040/50462 | Loss: 2.5006 | LR: 3.46e-05 | 5156.3ms/step | 11.6 samples/s +Step 42050/50462 | Loss: 2.5031 | LR: 3.45e-05 | 5144.7ms/step | 11.7 samples/s +Step 42060/50462 | Loss: 2.5358 | LR: 3.44e-05 | 5156.3ms/step | 11.6 samples/s +Step 42070/50462 | Loss: 2.5169 | LR: 3.43e-05 | 5147.3ms/step | 11.7 samples/s +Step 42080/50462 | Loss: 2.4889 | LR: 3.43e-05 | 5144.5ms/step | 11.7 samples/s +Step 42090/50462 | Loss: 2.5203 | LR: 3.42e-05 | 5147.6ms/step | 11.7 samples/s +Step 42100/50462 | Loss: 2.5007 | LR: 3.41e-05 | 5152.5ms/step | 11.6 samples/s +Step 42110/50462 | Loss: 2.4686 | LR: 3.40e-05 | 5161.4ms/step | 11.6 samples/s +Step 42120/50462 | Loss: 2.5033 | LR: 3.39e-05 | 5154.0ms/step | 11.6 samples/s +Step 42130/50462 | Loss: 2.5165 | LR: 3.39e-05 | 5157.3ms/step | 11.6 samples/s +Step 42140/50462 | Loss: 2.5119 | LR: 3.38e-05 | 5144.0ms/step | 11.7 samples/s +Step 42150/50462 | Loss: 2.5080 | LR: 3.37e-05 | 5139.4ms/step | 11.7 samples/s +Step 42160/50462 | Loss: 2.4669 | LR: 3.36e-05 | 5149.1ms/step | 11.7 samples/s +Step 42170/50462 | Loss: 2.4929 | LR: 3.35e-05 | 5184.2ms/step | 11.6 samples/s +Step 42180/50462 | Loss: 2.4996 | LR: 3.35e-05 | 5151.0ms/step | 11.6 samples/s +Step 42190/50462 | Loss: 2.5298 | LR: 3.34e-05 | 5149.4ms/step | 11.7 samples/s +Step 42200/50462 | Loss: 2.4852 | LR: 3.33e-05 | 5153.8ms/step | 11.6 samples/s +Step 42210/50462 | Loss: 2.4907 | LR: 3.32e-05 | 5217.5ms/step | 11.5 samples/s +Step 42220/50462 | Loss: 2.5191 | LR: 3.32e-05 | 5148.5ms/step | 11.7 samples/s +Step 42230/50462 | Loss: 2.5049 | LR: 3.31e-05 | 5170.5ms/step | 11.6 samples/s +Step 42240/50462 | Loss: 2.4605 | LR: 3.30e-05 | 5147.1ms/step | 11.7 samples/s +Step 42250/50462 | Loss: 2.5105 | LR: 3.29e-05 | 5147.6ms/step | 11.7 samples/s +Validation at step 42250: Loss = 2.5130, Perplexity = 12.34 +Step 42260/50462 | Loss: 2.5174 | LR: 3.28e-05 | 7706.1ms/step | 7.8 samples/s +Step 42270/50462 | Loss: 2.5291 | LR: 3.28e-05 | 5218.1ms/step | 11.5 samples/s +Step 42280/50462 | Loss: 2.4986 | LR: 3.27e-05 | 5166.7ms/step | 11.6 samples/s +Step 42290/50462 | Loss: 2.5118 | LR: 3.26e-05 | 5166.1ms/step | 11.6 samples/s +Step 42300/50462 | Loss: 2.4987 | LR: 3.25e-05 | 5167.4ms/step | 11.6 samples/s +Step 42310/50462 | Loss: 2.4910 | LR: 3.24e-05 | 5153.6ms/step | 11.6 samples/s +Step 42320/50462 | Loss: 2.4724 | LR: 3.24e-05 | 5160.3ms/step | 11.6 samples/s +Step 42330/50462 | Loss: 2.5210 | LR: 3.23e-05 | 5166.3ms/step | 11.6 samples/s +Step 42340/50462 | Loss: 2.5282 | LR: 3.22e-05 | 5161.3ms/step | 11.6 samples/s +Step 42350/50462 | Loss: 2.4745 | LR: 3.21e-05 | 5165.0ms/step | 11.6 samples/s +Step 42360/50462 | Loss: 2.4851 | LR: 3.21e-05 | 5163.1ms/step | 11.6 samples/s +Step 42370/50462 | Loss: 2.4991 | LR: 3.20e-05 | 5164.2ms/step | 11.6 samples/s +Step 42380/50462 | Loss: 2.4853 | LR: 3.19e-05 | 5152.9ms/step | 11.6 samples/s +Step 42390/50462 | Loss: 2.5123 | LR: 3.18e-05 | 5159.2ms/step | 11.6 samples/s +Step 42400/50462 | Loss: 2.5210 | LR: 3.18e-05 | 5171.0ms/step | 11.6 samples/s +Step 42410/50462 | Loss: 2.5132 | LR: 3.17e-05 | 5168.4ms/step | 11.6 samples/s +Step 42420/50462 | Loss: 2.5069 | LR: 3.16e-05 | 5151.9ms/step | 11.6 samples/s +Step 42430/50462 | Loss: 2.4943 | LR: 3.15e-05 | 5178.7ms/step | 11.6 samples/s +Step 42440/50462 | Loss: 2.4927 | LR: 3.14e-05 | 5205.3ms/step | 11.5 samples/s +Step 42450/50462 | Loss: 2.5125 | LR: 3.14e-05 | 5155.1ms/step | 11.6 samples/s +Step 42460/50462 | Loss: 2.4925 | LR: 3.13e-05 | 5159.3ms/step | 11.6 samples/s +Step 42470/50462 | Loss: 2.5045 | LR: 3.12e-05 | 5160.3ms/step | 11.6 samples/s +Step 42480/50462 | Loss: 2.4600 | LR: 3.11e-05 | 5193.2ms/step | 11.6 samples/s +Step 42490/50462 | Loss: 2.5076 | LR: 3.11e-05 | 5154.0ms/step | 11.6 samples/s +Step 42500/50462 | Loss: 2.5170 | LR: 3.10e-05 | 5168.4ms/step | 11.6 samples/s +Validation at step 42500: Loss = 2.5129, Perplexity = 12.34 +Step 42510/50462 | Loss: 2.5126 | LR: 3.09e-05 | 7731.5ms/step | 7.8 samples/s +Step 42520/50462 | Loss: 2.4983 | LR: 3.08e-05 | 5179.7ms/step | 11.6 samples/s +Step 42530/50462 | Loss: 2.4910 | LR: 3.08e-05 | 5224.8ms/step | 11.5 samples/s +Step 42540/50462 | Loss: 2.4972 | LR: 3.07e-05 | 5181.2ms/step | 11.6 samples/s +Step 42550/50462 | Loss: 2.5209 | LR: 3.06e-05 | 5168.9ms/step | 11.6 samples/s +Step 42560/50462 | Loss: 2.5787 | LR: 3.05e-05 | 5170.3ms/step | 11.6 samples/s +Step 42570/50462 | Loss: 2.5091 | LR: 3.05e-05 | 5159.6ms/step | 11.6 samples/s +Step 42580/50462 | Loss: 2.5029 | LR: 3.04e-05 | 5166.3ms/step | 11.6 samples/s +Step 42590/50462 | Loss: 2.4970 | LR: 3.03e-05 | 5151.3ms/step | 11.6 samples/s +Step 42600/50462 | Loss: 2.5453 | LR: 3.02e-05 | 5168.0ms/step | 11.6 samples/s +Step 42610/50462 | Loss: 2.4944 | LR: 3.02e-05 | 5173.6ms/step | 11.6 samples/s +Step 42620/50462 | Loss: 2.4871 | LR: 3.01e-05 | 5152.0ms/step | 11.6 samples/s +Step 42630/50462 | Loss: 2.5191 | LR: 3.00e-05 | 5153.3ms/step | 11.6 samples/s +Step 42640/50462 | Loss: 2.5439 | LR: 2.99e-05 | 5148.9ms/step | 11.7 samples/s +Step 42650/50462 | Loss: 2.5139 | LR: 2.99e-05 | 5156.1ms/step | 11.6 samples/s +Step 42660/50462 | Loss: 2.5160 | LR: 2.98e-05 | 5173.8ms/step | 11.6 samples/s +Step 42670/50462 | Loss: 2.4854 | LR: 2.97e-05 | 5175.9ms/step | 11.6 samples/s +Step 42680/50462 | Loss: 2.4854 | LR: 2.96e-05 | 5176.2ms/step | 11.6 samples/s +Step 42690/50462 | Loss: 2.5066 | LR: 2.96e-05 | 5167.5ms/step | 11.6 samples/s +Step 42700/50462 | Loss: 2.5136 | LR: 2.95e-05 | 5177.2ms/step | 11.6 samples/s +Step 42710/50462 | Loss: 2.4962 | LR: 2.94e-05 | 5190.0ms/step | 11.6 samples/s +Step 42720/50462 | Loss: 2.4969 | LR: 2.93e-05 | 5193.1ms/step | 11.6 samples/s +Step 42730/50462 | Loss: 2.4975 | LR: 2.93e-05 | 5161.7ms/step | 11.6 samples/s +Step 42740/50462 | Loss: 2.5020 | LR: 2.92e-05 | 5156.0ms/step | 11.6 samples/s +Step 42750/50462 | Loss: 2.4985 | LR: 2.91e-05 | 5156.8ms/step | 11.6 samples/s +Validation at step 42750: Loss = 2.5128, Perplexity = 12.34 +Step 42760/50462 | Loss: 2.5120 | LR: 2.90e-05 | 7753.1ms/step | 7.7 samples/s +Step 42770/50462 | Loss: 2.5025 | LR: 2.90e-05 | 5168.1ms/step | 11.6 samples/s +Step 42780/50462 | Loss: 2.5001 | LR: 2.89e-05 | 5168.3ms/step | 11.6 samples/s +Step 42790/50462 | Loss: 2.5177 | LR: 2.88e-05 | 5199.6ms/step | 11.5 samples/s +Step 42800/50462 | Loss: 2.5179 | LR: 2.87e-05 | 5182.2ms/step | 11.6 samples/s +Step 42810/50462 | Loss: 2.5226 | LR: 2.87e-05 | 5195.8ms/step | 11.5 samples/s +Step 42820/50462 | Loss: 2.4943 | LR: 2.86e-05 | 5188.3ms/step | 11.6 samples/s +Step 42830/50462 | Loss: 2.4868 | LR: 2.85e-05 | 5183.4ms/step | 11.6 samples/s +Step 42840/50462 | Loss: 2.5150 | LR: 2.84e-05 | 5183.6ms/step | 11.6 samples/s +Step 42850/50462 | Loss: 2.5039 | LR: 2.84e-05 | 5187.0ms/step | 11.6 samples/s +Step 42860/50462 | Loss: 2.5320 | LR: 2.83e-05 | 5170.6ms/step | 11.6 samples/s +Step 42870/50462 | Loss: 2.5143 | LR: 2.82e-05 | 5194.1ms/step | 11.6 samples/s +Step 42880/50462 | Loss: 2.5097 | LR: 2.82e-05 | 5184.6ms/step | 11.6 samples/s +Step 42890/50462 | Loss: 2.4940 | LR: 2.81e-05 | 5167.8ms/step | 11.6 samples/s +Step 42900/50462 | Loss: 2.5022 | LR: 2.80e-05 | 5182.3ms/step | 11.6 samples/s +Step 42910/50462 | Loss: 2.5327 | LR: 2.79e-05 | 5185.3ms/step | 11.6 samples/s +Step 42920/50462 | Loss: 2.4916 | LR: 2.79e-05 | 5194.9ms/step | 11.5 samples/s +Step 42930/50462 | Loss: 2.4905 | LR: 2.78e-05 | 5178.8ms/step | 11.6 samples/s +Step 42940/50462 | Loss: 2.4804 | LR: 2.77e-05 | 5197.4ms/step | 11.5 samples/s +Step 42950/50462 | Loss: 2.5310 | LR: 2.76e-05 | 5192.0ms/step | 11.6 samples/s +Step 42960/50462 | Loss: 2.5145 | LR: 2.76e-05 | 5187.5ms/step | 11.6 samples/s +Step 42970/50462 | Loss: 2.4853 | LR: 2.75e-05 | 5180.6ms/step | 11.6 samples/s +Step 42980/50462 | Loss: 2.4996 | LR: 2.74e-05 | 5176.5ms/step | 11.6 samples/s +Step 42990/50462 | Loss: 2.5069 | LR: 2.74e-05 | 5211.4ms/step | 11.5 samples/s +Step 43000/50462 | Loss: 2.4844 | LR: 2.73e-05 | 5191.0ms/step | 11.6 samples/s +Validation at step 43000: Loss = 2.5127, Perplexity = 12.34 +Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-43000 +Step 43010/50462 | Loss: 2.5256 | LR: 2.72e-05 | 8678.3ms/step | 6.9 samples/s +Step 43020/50462 | Loss: 2.4983 | LR: 2.71e-05 | 5156.8ms/step | 11.6 samples/s +Step 43030/50462 | Loss: 2.5038 | LR: 2.71e-05 | 5211.8ms/step | 11.5 samples/s +Step 43040/50462 | Loss: 2.4917 | LR: 2.70e-05 | 5185.6ms/step | 11.6 samples/s +Step 43050/50462 | Loss: 2.5126 | LR: 2.69e-05 | 5180.7ms/step | 11.6 samples/s +Step 43060/50462 | Loss: 2.4868 | LR: 2.69e-05 | 5221.2ms/step | 11.5 samples/s +Step 43070/50462 | Loss: 2.5046 | LR: 2.68e-05 | 5193.1ms/step | 11.6 samples/s +Step 43080/50462 | Loss: 2.4839 | LR: 2.67e-05 | 5185.3ms/step | 11.6 samples/s +Step 43090/50462 | Loss: 2.4793 | LR: 2.66e-05 | 5190.8ms/step | 11.6 samples/s +Step 43100/50462 | Loss: 2.4791 | LR: 2.66e-05 | 5171.0ms/step | 11.6 samples/s +Step 43110/50462 | Loss: 2.4869 | LR: 2.65e-05 | 5180.3ms/step | 11.6 samples/s +Step 43120/50462 | Loss: 2.5018 | LR: 2.64e-05 | 5185.6ms/step | 11.6 samples/s +Step 43130/50462 | Loss: 2.5119 | LR: 2.64e-05 | 5184.2ms/step | 11.6 samples/s +Step 43140/50462 | Loss: 2.4820 | LR: 2.63e-05 | 5195.7ms/step | 11.5 samples/s +Step 43150/50462 | Loss: 2.4840 | LR: 2.62e-05 | 5183.8ms/step | 11.6 samples/s +Step 43160/50462 | Loss: 2.5231 | LR: 2.61e-05 | 5180.5ms/step | 11.6 samples/s +Step 43170/50462 | Loss: 2.5147 | LR: 2.61e-05 | 5184.1ms/step | 11.6 samples/s +Step 43180/50462 | Loss: 2.4685 | LR: 2.60e-05 | 5180.9ms/step | 11.6 samples/s +Step 43190/50462 | Loss: 2.5239 | LR: 2.59e-05 | 5176.6ms/step | 11.6 samples/s +Step 43200/50462 | Loss: 2.5007 | LR: 2.59e-05 | 5180.6ms/step | 11.6 samples/s +Step 43210/50462 | Loss: 2.5074 | LR: 2.58e-05 | 5153.7ms/step | 11.6 samples/s +Step 43220/50462 | Loss: 2.5150 | LR: 2.57e-05 | 5177.0ms/step | 11.6 samples/s +Step 43230/50462 | Loss: 2.5001 | LR: 2.57e-05 | 5161.6ms/step | 11.6 samples/s +Step 43240/50462 | Loss: 2.5207 | LR: 2.56e-05 | 5163.0ms/step | 11.6 samples/s +Step 43250/50462 | Loss: 2.4714 | LR: 2.55e-05 | 5149.6ms/step | 11.7 samples/s +Validation at step 43250: Loss = 2.5126, Perplexity = 12.34 +Step 43260/50462 | Loss: 2.4732 | LR: 2.54e-05 | 7757.1ms/step | 7.7 samples/s +Step 43270/50462 | Loss: 2.4846 | LR: 2.54e-05 | 5158.3ms/step | 11.6 samples/s +Step 43280/50462 | Loss: 2.5060 | LR: 2.53e-05 | 5167.4ms/step | 11.6 samples/s +Step 43290/50462 | Loss: 2.4857 | LR: 2.52e-05 | 5171.7ms/step | 11.6 samples/s +Step 43300/50462 | Loss: 2.5292 | LR: 2.52e-05 | 5196.2ms/step | 11.5 samples/s +Step 43310/50462 | Loss: 2.4945 | LR: 2.51e-05 | 5162.5ms/step | 11.6 samples/s +Step 43320/50462 | Loss: 2.4972 | LR: 2.50e-05 | 5201.8ms/step | 11.5 samples/s +Step 43330/50462 | Loss: 2.4832 | LR: 2.50e-05 | 5166.9ms/step | 11.6 samples/s +Step 43340/50462 | Loss: 2.4954 | LR: 2.49e-05 | 5158.2ms/step | 11.6 samples/s +Step 43350/50462 | Loss: 2.4815 | LR: 2.48e-05 | 5185.4ms/step | 11.6 samples/s +Step 43360/50462 | Loss: 2.5146 | LR: 2.48e-05 | 5161.7ms/step | 11.6 samples/s +Step 43370/50462 | Loss: 2.5055 | LR: 2.47e-05 | 5171.4ms/step | 11.6 samples/s +Step 43380/50462 | Loss: 2.4847 | LR: 2.46e-05 | 5157.9ms/step | 11.6 samples/s +Step 43390/50462 | Loss: 2.5132 | LR: 2.46e-05 | 5164.1ms/step | 11.6 samples/s +Step 43400/50462 | Loss: 2.4962 | LR: 2.45e-05 | 5159.4ms/step | 11.6 samples/s +Step 43410/50462 | Loss: 2.5158 | LR: 2.44e-05 | 5165.6ms/step | 11.6 samples/s +Step 43420/50462 | Loss: 2.4958 | LR: 2.44e-05 | 5173.5ms/step | 11.6 samples/s +Step 43430/50462 | Loss: 2.5009 | LR: 2.43e-05 | 5175.9ms/step | 11.6 samples/s +Step 43440/50462 | Loss: 2.4976 | LR: 2.42e-05 | 5150.8ms/step | 11.6 samples/s +Step 43450/50462 | Loss: 2.5220 | LR: 2.41e-05 | 5147.5ms/step | 11.7 samples/s +Step 43460/50462 | Loss: 2.5090 | LR: 2.41e-05 | 5155.4ms/step | 11.6 samples/s +Step 43470/50462 | Loss: 2.5244 | LR: 2.40e-05 | 5173.0ms/step | 11.6 samples/s +Step 43480/50462 | Loss: 2.5234 | LR: 2.39e-05 | 5170.2ms/step | 11.6 samples/s +Step 43490/50462 | Loss: 2.4919 | LR: 2.39e-05 | 5160.2ms/step | 11.6 samples/s +Step 43500/50462 | Loss: 2.5109 | LR: 2.38e-05 | 5153.3ms/step | 11.6 samples/s +Validation at step 43500: Loss = 2.5125, Perplexity = 12.34 +Step 43510/50462 | Loss: 2.5121 | LR: 2.37e-05 | 7705.5ms/step | 7.8 samples/s +Step 43520/50462 | Loss: 2.4758 | LR: 2.37e-05 | 5171.0ms/step | 11.6 samples/s +Step 43530/50462 | Loss: 2.4960 | LR: 2.36e-05 | 5195.5ms/step | 11.5 samples/s +Step 43540/50462 | Loss: 2.4936 | LR: 2.35e-05 | 5160.9ms/step | 11.6 samples/s +Step 43550/50462 | Loss: 2.5023 | LR: 2.35e-05 | 5168.2ms/step | 11.6 samples/s +Step 43560/50462 | Loss: 2.4811 | LR: 2.34e-05 | 5163.3ms/step | 11.6 samples/s +Step 43570/50462 | Loss: 2.5016 | LR: 2.33e-05 | 5168.3ms/step | 11.6 samples/s +Step 43580/50462 | Loss: 2.5301 | LR: 2.33e-05 | 5246.1ms/step | 11.4 samples/s +Step 43590/50462 | Loss: 2.4863 | LR: 2.32e-05 | 5164.2ms/step | 11.6 samples/s +Step 43600/50462 | Loss: 2.5211 | LR: 2.31e-05 | 5166.4ms/step | 11.6 samples/s +Step 43610/50462 | Loss: 2.5078 | LR: 2.31e-05 | 5169.4ms/step | 11.6 samples/s +Step 43620/50462 | Loss: 2.4907 | LR: 2.30e-05 | 5178.0ms/step | 11.6 samples/s +Step 43630/50462 | Loss: 2.5286 | LR: 2.29e-05 | 5163.2ms/step | 11.6 samples/s +Step 43640/50462 | Loss: 2.4902 | LR: 2.29e-05 | 5162.9ms/step | 11.6 samples/s +Step 43650/50462 | Loss: 2.5097 | LR: 2.28e-05 | 5167.1ms/step | 11.6 samples/s +Step 43660/50462 | Loss: 2.5227 | LR: 2.27e-05 | 5170.2ms/step | 11.6 samples/s +Step 43670/50462 | Loss: 2.5095 | LR: 2.27e-05 | 5164.6ms/step | 11.6 samples/s +Step 43680/50462 | Loss: 2.5139 | LR: 2.26e-05 | 5186.5ms/step | 11.6 samples/s +Step 43690/50462 | Loss: 2.5071 | LR: 2.25e-05 | 5177.8ms/step | 11.6 samples/s +Step 43700/50462 | Loss: 2.5241 | LR: 2.25e-05 | 5165.4ms/step | 11.6 samples/s +Step 43710/50462 | Loss: 2.5292 | LR: 2.24e-05 | 5162.2ms/step | 11.6 samples/s +Step 43720/50462 | Loss: 2.4852 | LR: 2.24e-05 | 5154.4ms/step | 11.6 samples/s +Step 43730/50462 | Loss: 2.4744 | LR: 2.23e-05 | 5140.4ms/step | 11.7 samples/s +Step 43740/50462 | Loss: 2.4986 | LR: 2.22e-05 | 5173.9ms/step | 11.6 samples/s +Step 43750/50462 | Loss: 2.4995 | LR: 2.22e-05 | 5179.9ms/step | 11.6 samples/s +Validation at step 43750: Loss = 2.5124, Perplexity = 12.34 +Step 43760/50462 | Loss: 2.5051 | LR: 2.21e-05 | 7735.0ms/step | 7.8 samples/s +Step 43770/50462 | Loss: 2.4901 | LR: 2.20e-05 | 5171.8ms/step | 11.6 samples/s +Step 43780/50462 | Loss: 2.4985 | LR: 2.20e-05 | 5168.2ms/step | 11.6 samples/s +Step 43790/50462 | Loss: 2.5069 | LR: 2.19e-05 | 5164.3ms/step | 11.6 samples/s +Step 43800/50462 | Loss: 2.5193 | LR: 2.18e-05 | 5162.2ms/step | 11.6 samples/s +Step 43810/50462 | Loss: 2.4630 | LR: 2.18e-05 | 5205.0ms/step | 11.5 samples/s +Step 43820/50462 | Loss: 2.4672 | LR: 2.17e-05 | 5170.2ms/step | 11.6 samples/s +Step 43830/50462 | Loss: 2.5112 | LR: 2.16e-05 | 5176.4ms/step | 11.6 samples/s +Step 43840/50462 | Loss: 2.5255 | LR: 2.16e-05 | 5211.8ms/step | 11.5 samples/s +Step 43850/50462 | Loss: 2.5218 | LR: 2.15e-05 | 5214.4ms/step | 11.5 samples/s +Step 43860/50462 | Loss: 2.5043 | LR: 2.14e-05 | 5171.2ms/step | 11.6 samples/s +Step 43870/50462 | Loss: 2.4938 | LR: 2.14e-05 | 5188.0ms/step | 11.6 samples/s +Step 43880/50462 | Loss: 2.5138 | LR: 2.13e-05 | 5171.2ms/step | 11.6 samples/s +Step 43890/50462 | Loss: 2.5056 | LR: 2.13e-05 | 5171.1ms/step | 11.6 samples/s +Step 43900/50462 | Loss: 2.5575 | LR: 2.12e-05 | 5165.6ms/step | 11.6 samples/s +Step 43910/50462 | Loss: 2.5240 | LR: 2.11e-05 | 5180.1ms/step | 11.6 samples/s +Step 43920/50462 | Loss: 2.4996 | LR: 2.11e-05 | 5169.8ms/step | 11.6 samples/s +Step 43930/50462 | Loss: 2.5250 | LR: 2.10e-05 | 5170.5ms/step | 11.6 samples/s +Step 43940/50462 | Loss: 2.5002 | LR: 2.09e-05 | 5176.1ms/step | 11.6 samples/s +Step 43950/50462 | Loss: 2.5125 | LR: 2.09e-05 | 5160.6ms/step | 11.6 samples/s +Step 43960/50462 | Loss: 2.4953 | LR: 2.08e-05 | 5175.9ms/step | 11.6 samples/s +Step 43970/50462 | Loss: 2.4909 | LR: 2.07e-05 | 5159.5ms/step | 11.6 samples/s +Step 43980/50462 | Loss: 2.4818 | LR: 2.07e-05 | 5157.6ms/step | 11.6 samples/s +Step 43990/50462 | Loss: 2.5192 | LR: 2.06e-05 | 5167.2ms/step | 11.6 samples/s +Step 44000/50462 | Loss: 2.5244 | LR: 2.06e-05 | 5164.9ms/step | 11.6 samples/s +Validation at step 44000: Loss = 2.5124, Perplexity = 12.33 +Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-44000 +Step 44010/50462 | Loss: 2.5095 | LR: 2.05e-05 | 8670.6ms/step | 6.9 samples/s +Step 44020/50462 | Loss: 2.5015 | LR: 2.04e-05 | 5152.2ms/step | 11.6 samples/s +Step 44030/50462 | Loss: 2.4981 | LR: 2.04e-05 | 5175.1ms/step | 11.6 samples/s +Step 44040/50462 | Loss: 2.4929 | LR: 2.03e-05 | 5152.4ms/step | 11.6 samples/s +Step 44050/50462 | Loss: 2.5256 | LR: 2.02e-05 | 5161.2ms/step | 11.6 samples/s +Step 44060/50462 | Loss: 2.5244 | LR: 2.02e-05 | 5178.8ms/step | 11.6 samples/s +Step 44070/50462 | Loss: 2.5240 | LR: 2.01e-05 | 5176.1ms/step | 11.6 samples/s +Step 44080/50462 | Loss: 2.5065 | LR: 2.01e-05 | 5216.6ms/step | 11.5 samples/s +Step 44090/50462 | Loss: 2.4756 | LR: 2.00e-05 | 5169.4ms/step | 11.6 samples/s +Step 44100/50462 | Loss: 2.4941 | LR: 1.99e-05 | 5195.9ms/step | 11.5 samples/s +Step 44110/50462 | Loss: 2.4963 | LR: 1.99e-05 | 5158.8ms/step | 11.6 samples/s +Step 44120/50462 | Loss: 2.5126 | LR: 1.98e-05 | 5160.9ms/step | 11.6 samples/s +Step 44130/50462 | Loss: 2.5185 | LR: 1.97e-05 | 5206.2ms/step | 11.5 samples/s +Step 44140/50462 | Loss: 2.4866 | LR: 1.97e-05 | 5156.2ms/step | 11.6 samples/s +Step 44150/50462 | Loss: 2.4821 | LR: 1.96e-05 | 5154.9ms/step | 11.6 samples/s +Step 44160/50462 | Loss: 2.5123 | LR: 1.96e-05 | 5145.1ms/step | 11.7 samples/s +Step 44170/50462 | Loss: 2.5159 | LR: 1.95e-05 | 5160.7ms/step | 11.6 samples/s +Step 44180/50462 | Loss: 2.4825 | LR: 1.94e-05 | 5164.1ms/step | 11.6 samples/s +Step 44190/50462 | Loss: 2.4891 | LR: 1.94e-05 | 5149.2ms/step | 11.7 samples/s +Step 44200/50462 | Loss: 2.5104 | LR: 1.93e-05 | 5146.7ms/step | 11.7 samples/s +Step 44210/50462 | Loss: 2.4985 | LR: 1.93e-05 | 5151.3ms/step | 11.6 samples/s +Step 44220/50462 | Loss: 2.5046 | LR: 1.92e-05 | 5156.1ms/step | 11.6 samples/s +Step 44230/50462 | Loss: 2.5162 | LR: 1.91e-05 | 5141.8ms/step | 11.7 samples/s +Step 44240/50462 | Loss: 2.5036 | LR: 1.91e-05 | 5160.0ms/step | 11.6 samples/s +Step 44250/50462 | Loss: 2.5171 | LR: 1.90e-05 | 5152.9ms/step | 11.6 samples/s +Validation at step 44250: Loss = 2.5123, Perplexity = 12.33 +Step 44260/50462 | Loss: 2.5115 | LR: 1.90e-05 | 7701.5ms/step | 7.8 samples/s +Step 44270/50462 | Loss: 2.4823 | LR: 1.89e-05 | 5127.6ms/step | 11.7 samples/s +Step 44280/50462 | Loss: 2.5057 | LR: 1.88e-05 | 5149.5ms/step | 11.7 samples/s +Step 44290/50462 | Loss: 2.5020 | LR: 1.88e-05 | 5160.4ms/step | 11.6 samples/s +Step 44300/50462 | Loss: 2.4532 | LR: 1.87e-05 | 5167.3ms/step | 11.6 samples/s +Step 44310/50462 | Loss: 2.4831 | LR: 1.87e-05 | 5135.7ms/step | 11.7 samples/s +Step 44320/50462 | Loss: 2.5155 | LR: 1.86e-05 | 5157.1ms/step | 11.6 samples/s +Step 44330/50462 | Loss: 2.5060 | LR: 1.85e-05 | 5139.4ms/step | 11.7 samples/s +Step 44340/50462 | Loss: 2.5022 | LR: 1.85e-05 | 5144.5ms/step | 11.7 samples/s +Step 44350/50462 | Loss: 2.5071 | LR: 1.84e-05 | 5186.8ms/step | 11.6 samples/s +Step 44360/50462 | Loss: 2.4802 | LR: 1.84e-05 | 5187.2ms/step | 11.6 samples/s +Step 44370/50462 | Loss: 2.5056 | LR: 1.83e-05 | 5139.5ms/step | 11.7 samples/s +Step 44380/50462 | Loss: 2.4965 | LR: 1.82e-05 | 5158.3ms/step | 11.6 samples/s +Step 44390/50462 | Loss: 2.5167 | LR: 1.82e-05 | 5154.7ms/step | 11.6 samples/s +Step 44400/50462 | Loss: 2.4830 | LR: 1.81e-05 | 5210.4ms/step | 11.5 samples/s +Step 44410/50462 | Loss: 2.4924 | LR: 1.81e-05 | 5157.1ms/step | 11.6 samples/s +Step 44420/50462 | Loss: 2.5024 | LR: 1.80e-05 | 5175.3ms/step | 11.6 samples/s +Step 44430/50462 | Loss: 2.5098 | LR: 1.79e-05 | 5167.0ms/step | 11.6 samples/s +Step 44440/50462 | Loss: 2.4990 | LR: 1.79e-05 | 5171.2ms/step | 11.6 samples/s +Step 44450/50462 | Loss: 2.4877 | LR: 1.78e-05 | 5161.6ms/step | 11.6 samples/s +Step 44460/50462 | Loss: 2.5033 | LR: 1.78e-05 | 5166.7ms/step | 11.6 samples/s +Step 44470/50462 | Loss: 2.5160 | LR: 1.77e-05 | 5161.7ms/step | 11.6 samples/s +Step 44480/50462 | Loss: 2.5302 | LR: 1.77e-05 | 5150.9ms/step | 11.6 samples/s +Step 44490/50462 | Loss: 2.5257 | LR: 1.76e-05 | 5181.0ms/step | 11.6 samples/s +Step 44500/50462 | Loss: 2.4886 | LR: 1.75e-05 | 5165.4ms/step | 11.6 samples/s +Validation at step 44500: Loss = 2.5123, Perplexity = 12.33 +Step 44510/50462 | Loss: 2.4781 | LR: 1.75e-05 | 7720.4ms/step | 7.8 samples/s +Step 44520/50462 | Loss: 2.4769 | LR: 1.74e-05 | 5174.8ms/step | 11.6 samples/s +Step 44530/50462 | Loss: 2.5030 | LR: 1.74e-05 | 5168.5ms/step | 11.6 samples/s +Step 44540/50462 | Loss: 2.4912 | LR: 1.73e-05 | 5175.9ms/step | 11.6 samples/s +Step 44550/50462 | Loss: 2.5036 | LR: 1.72e-05 | 5170.6ms/step | 11.6 samples/s +Step 44560/50462 | Loss: 2.5094 | LR: 1.72e-05 | 5179.0ms/step | 11.6 samples/s +Step 44570/50462 | Loss: 2.4958 | LR: 1.71e-05 | 5156.3ms/step | 11.6 samples/s +Step 44580/50462 | Loss: 2.4739 | LR: 1.71e-05 | 5185.6ms/step | 11.6 samples/s +Step 44590/50462 | Loss: 2.4925 | LR: 1.70e-05 | 5171.6ms/step | 11.6 samples/s +Step 44600/50462 | Loss: 2.5163 | LR: 1.70e-05 | 5163.0ms/step | 11.6 samples/s +Step 44610/50462 | Loss: 2.4981 | LR: 1.69e-05 | 5165.7ms/step | 11.6 samples/s +Step 44620/50462 | Loss: 2.5157 | LR: 1.68e-05 | 5165.0ms/step | 11.6 samples/s +Step 44630/50462 | Loss: 2.5007 | LR: 1.68e-05 | 5250.6ms/step | 11.4 samples/s +Step 44640/50462 | Loss: 2.4917 | LR: 1.67e-05 | 5166.7ms/step | 11.6 samples/s +Step 44650/50462 | Loss: 2.4803 | LR: 1.67e-05 | 5167.7ms/step | 11.6 samples/s +Step 44660/50462 | Loss: 2.5181 | LR: 1.66e-05 | 5180.8ms/step | 11.6 samples/s +Step 44670/50462 | Loss: 2.5270 | LR: 1.66e-05 | 5214.5ms/step | 11.5 samples/s +Step 44680/50462 | Loss: 2.5130 | LR: 1.65e-05 | 5177.8ms/step | 11.6 samples/s +Step 44690/50462 | Loss: 2.5101 | LR: 1.64e-05 | 5166.8ms/step | 11.6 samples/s +Step 44700/50462 | Loss: 2.5152 | LR: 1.64e-05 | 5178.8ms/step | 11.6 samples/s +Step 44710/50462 | Loss: 2.4996 | LR: 1.63e-05 | 5178.2ms/step | 11.6 samples/s +Step 44720/50462 | Loss: 2.5010 | LR: 1.63e-05 | 5161.9ms/step | 11.6 samples/s +Step 44730/50462 | Loss: 2.4889 | LR: 1.62e-05 | 5153.9ms/step | 11.6 samples/s +Step 44740/50462 | Loss: 2.4846 | LR: 1.62e-05 | 5171.4ms/step | 11.6 samples/s +Step 44750/50462 | Loss: 2.5089 | LR: 1.61e-05 | 5172.6ms/step | 11.6 samples/s +Validation at step 44750: Loss = 2.5122, Perplexity = 12.33 +Step 44760/50462 | Loss: 2.5245 | LR: 1.61e-05 | 7734.9ms/step | 7.8 samples/s +Step 44770/50462 | Loss: 2.4730 | LR: 1.60e-05 | 5172.9ms/step | 11.6 samples/s +Step 44780/50462 | Loss: 2.5015 | LR: 1.59e-05 | 5169.2ms/step | 11.6 samples/s +Step 44790/50462 | Loss: 2.4914 | LR: 1.59e-05 | 5166.6ms/step | 11.6 samples/s +Step 44800/50462 | Loss: 2.4930 | LR: 1.58e-05 | 5166.7ms/step | 11.6 samples/s +Step 44810/50462 | Loss: 2.5192 | LR: 1.58e-05 | 5152.5ms/step | 11.6 samples/s +Step 44820/50462 | Loss: 2.5088 | LR: 1.57e-05 | 5173.6ms/step | 11.6 samples/s +Step 44830/50462 | Loss: 2.5301 | LR: 1.57e-05 | 5167.2ms/step | 11.6 samples/s +Step 44840/50462 | Loss: 2.4858 | LR: 1.56e-05 | 5176.4ms/step | 11.6 samples/s +Step 44850/50462 | Loss: 2.4901 | LR: 1.56e-05 | 5176.0ms/step | 11.6 samples/s +Step 44860/50462 | Loss: 2.4888 | LR: 1.55e-05 | 5155.2ms/step | 11.6 samples/s +Step 44870/50462 | Loss: 2.4788 | LR: 1.54e-05 | 5152.3ms/step | 11.6 samples/s +Step 44880/50462 | Loss: 2.5031 | LR: 1.54e-05 | 5163.2ms/step | 11.6 samples/s +Step 44890/50462 | Loss: 2.4666 | LR: 1.53e-05 | 5208.8ms/step | 11.5 samples/s +Step 44900/50462 | Loss: 2.4664 | LR: 1.53e-05 | 5213.6ms/step | 11.5 samples/s +Step 44910/50462 | Loss: 2.4885 | LR: 1.52e-05 | 5170.2ms/step | 11.6 samples/s +Step 44920/50462 | Loss: 2.4909 | LR: 1.52e-05 | 5174.0ms/step | 11.6 samples/s +Step 44930/50462 | Loss: 2.4943 | LR: 1.51e-05 | 5171.5ms/step | 11.6 samples/s +Step 44940/50462 | Loss: 2.4966 | LR: 1.51e-05 | 5151.0ms/step | 11.6 samples/s +Step 44950/50462 | Loss: 2.4542 | LR: 1.50e-05 | 5207.3ms/step | 11.5 samples/s +Step 44960/50462 | Loss: 2.5331 | LR: 1.50e-05 | 5170.8ms/step | 11.6 samples/s +Step 44970/50462 | Loss: 2.4832 | LR: 1.49e-05 | 5154.2ms/step | 11.6 samples/s +Step 44980/50462 | Loss: 2.4949 | LR: 1.49e-05 | 5175.7ms/step | 11.6 samples/s +Step 44990/50462 | Loss: 2.5238 | LR: 1.48e-05 | 5177.0ms/step | 11.6 samples/s +Step 45000/50462 | Loss: 2.5144 | LR: 1.47e-05 | 5163.8ms/step | 11.6 samples/s +Validation at step 45000: Loss = 2.5122, Perplexity = 12.33 +Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-45000 +Step 45010/50462 | Loss: 2.5071 | LR: 1.47e-05 | 8679.3ms/step | 6.9 samples/s +Step 45020/50462 | Loss: 2.4908 | LR: 1.46e-05 | 5178.7ms/step | 11.6 samples/s +Step 45030/50462 | Loss: 2.4974 | LR: 1.46e-05 | 5161.3ms/step | 11.6 samples/s +Step 45040/50462 | Loss: 2.5102 | LR: 1.45e-05 | 5150.5ms/step | 11.6 samples/s +Step 45050/50462 | Loss: 2.4855 | LR: 1.45e-05 | 5167.6ms/step | 11.6 samples/s +Step 45060/50462 | Loss: 2.5082 | LR: 1.44e-05 | 5166.0ms/step | 11.6 samples/s +Step 45070/50462 | Loss: 2.5020 | LR: 1.44e-05 | 5171.3ms/step | 11.6 samples/s +Step 45080/50462 | Loss: 2.4941 | LR: 1.43e-05 | 5177.2ms/step | 11.6 samples/s +Step 45090/50462 | Loss: 2.4934 | LR: 1.43e-05 | 5165.3ms/step | 11.6 samples/s +Step 45100/50462 | Loss: 2.5278 | LR: 1.42e-05 | 5173.7ms/step | 11.6 samples/s +Step 45110/50462 | Loss: 2.5269 | LR: 1.42e-05 | 5155.8ms/step | 11.6 samples/s +Step 45120/50462 | Loss: 2.5095 | LR: 1.41e-05 | 5157.4ms/step | 11.6 samples/s +Step 45130/50462 | Loss: 2.5182 | LR: 1.41e-05 | 5160.8ms/step | 11.6 samples/s +Step 45140/50462 | Loss: 2.4870 | LR: 1.40e-05 | 5169.1ms/step | 11.6 samples/s +Step 45150/50462 | Loss: 2.5117 | LR: 1.40e-05 | 5204.0ms/step | 11.5 samples/s +Step 45160/50462 | Loss: 2.5183 | LR: 1.39e-05 | 5179.8ms/step | 11.6 samples/s +Step 45170/50462 | Loss: 2.4797 | LR: 1.39e-05 | 5212.8ms/step | 11.5 samples/s +Step 45180/50462 | Loss: 2.5078 | LR: 1.38e-05 | 5155.7ms/step | 11.6 samples/s +Step 45190/50462 | Loss: 2.4946 | LR: 1.37e-05 | 5152.7ms/step | 11.6 samples/s +Step 45200/50462 | Loss: 2.5181 | LR: 1.37e-05 | 5173.8ms/step | 11.6 samples/s +Step 45210/50462 | Loss: 2.5033 | LR: 1.36e-05 | 5157.2ms/step | 11.6 samples/s +Step 45220/50462 | Loss: 2.4889 | LR: 1.36e-05 | 5207.7ms/step | 11.5 samples/s +Step 45230/50462 | Loss: 2.4837 | LR: 1.35e-05 | 5166.6ms/step | 11.6 samples/s +Step 45240/50462 | Loss: 2.5189 | LR: 1.35e-05 | 5156.8ms/step | 11.6 samples/s +Step 45250/50462 | Loss: 2.4956 | LR: 1.34e-05 | 5164.6ms/step | 11.6 samples/s +Validation at step 45250: Loss = 2.5122, Perplexity = 12.33 +Step 45260/50462 | Loss: 2.5068 | LR: 1.34e-05 | 7704.6ms/step | 7.8 samples/s +Step 45270/50462 | Loss: 2.5164 | LR: 1.33e-05 | 5169.5ms/step | 11.6 samples/s +Step 45280/50462 | Loss: 2.4738 | LR: 1.33e-05 | 5170.7ms/step | 11.6 samples/s +Step 45290/50462 | Loss: 2.5001 | LR: 1.32e-05 | 5172.2ms/step | 11.6 samples/s +Step 45300/50462 | Loss: 2.5188 | LR: 1.32e-05 | 5171.7ms/step | 11.6 samples/s +Step 45310/50462 | Loss: 2.4871 | LR: 1.31e-05 | 5170.4ms/step | 11.6 samples/s +Step 45320/50462 | Loss: 2.4950 | LR: 1.31e-05 | 5184.4ms/step | 11.6 samples/s +Step 45330/50462 | Loss: 2.5115 | LR: 1.30e-05 | 5169.6ms/step | 11.6 samples/s +Step 45340/50462 | Loss: 2.5031 | LR: 1.30e-05 | 5176.2ms/step | 11.6 samples/s +Step 45350/50462 | Loss: 2.4997 | LR: 1.29e-05 | 5170.3ms/step | 11.6 samples/s +Step 45360/50462 | Loss: 2.5190 | LR: 1.29e-05 | 5175.3ms/step | 11.6 samples/s +Step 45370/50462 | Loss: 2.4927 | LR: 1.28e-05 | 5165.0ms/step | 11.6 samples/s +Step 45380/50462 | Loss: 2.4886 | LR: 1.28e-05 | 5163.1ms/step | 11.6 samples/s +Step 45390/50462 | Loss: 2.4950 | LR: 1.27e-05 | 5162.0ms/step | 11.6 samples/s +Step 45400/50462 | Loss: 2.4790 | LR: 1.27e-05 | 5191.3ms/step | 11.6 samples/s +Step 45410/50462 | Loss: 2.5239 | LR: 1.26e-05 | 5222.8ms/step | 11.5 samples/s +Step 45420/50462 | Loss: 2.4946 | LR: 1.26e-05 | 5158.9ms/step | 11.6 samples/s +Step 45430/50462 | Loss: 2.4696 | LR: 1.25e-05 | 5174.2ms/step | 11.6 samples/s +Step 45440/50462 | Loss: 2.5116 | LR: 1.25e-05 | 5207.5ms/step | 11.5 samples/s +Step 45450/50462 | Loss: 2.5015 | LR: 1.24e-05 | 5145.4ms/step | 11.7 samples/s +Step 45460/50462 | Loss: 2.4911 | LR: 1.24e-05 | 5159.3ms/step | 11.6 samples/s +Step 45470/50462 | Loss: 2.4999 | LR: 1.23e-05 | 5173.4ms/step | 11.6 samples/s +Step 45480/50462 | Loss: 2.4902 | LR: 1.23e-05 | 5159.5ms/step | 11.6 samples/s +Step 45490/50462 | Loss: 2.5119 | LR: 1.22e-05 | 5168.5ms/step | 11.6 samples/s +Step 45500/50462 | Loss: 2.5116 | LR: 1.22e-05 | 5206.0ms/step | 11.5 samples/s +Validation at step 45500: Loss = 2.5122, Perplexity = 12.33 +Step 45510/50462 | Loss: 2.4855 | LR: 1.21e-05 | 7711.9ms/step | 7.8 samples/s +Step 45520/50462 | Loss: 2.5318 | LR: 1.21e-05 | 5168.2ms/step | 11.6 samples/s +Step 45530/50462 | Loss: 2.4780 | LR: 1.20e-05 | 5172.1ms/step | 11.6 samples/s +Step 45540/50462 | Loss: 2.5264 | LR: 1.20e-05 | 5166.6ms/step | 11.6 samples/s +Step 45550/50462 | Loss: 2.4863 | LR: 1.19e-05 | 5166.1ms/step | 11.6 samples/s +Step 45560/50462 | Loss: 2.5110 | LR: 1.19e-05 | 5171.5ms/step | 11.6 samples/s +Step 45570/50462 | Loss: 2.5157 | LR: 1.19e-05 | 5155.1ms/step | 11.6 samples/s +Step 45580/50462 | Loss: 2.4926 | LR: 1.18e-05 | 5163.4ms/step | 11.6 samples/s +Step 45590/50462 | Loss: 2.4813 | LR: 1.18e-05 | 5165.5ms/step | 11.6 samples/s +Step 45600/50462 | Loss: 2.4902 | LR: 1.17e-05 | 5174.2ms/step | 11.6 samples/s +Step 45610/50462 | Loss: 2.4683 | LR: 1.17e-05 | 5153.4ms/step | 11.6 samples/s +Step 45620/50462 | Loss: 2.5001 | LR: 1.16e-05 | 5154.3ms/step | 11.6 samples/s +Step 45630/50462 | Loss: 2.5168 | LR: 1.16e-05 | 5168.5ms/step | 11.6 samples/s +Step 45640/50462 | Loss: 2.4951 | LR: 1.15e-05 | 5165.2ms/step | 11.6 samples/s +Step 45650/50462 | Loss: 2.5196 | LR: 1.15e-05 | 5158.8ms/step | 11.6 samples/s +Step 45660/50462 | Loss: 2.5050 | LR: 1.14e-05 | 5181.2ms/step | 11.6 samples/s +Step 45670/50462 | Loss: 2.4715 | LR: 1.14e-05 | 5167.4ms/step | 11.6 samples/s +Step 45680/50462 | Loss: 2.4913 | LR: 1.13e-05 | 5205.1ms/step | 11.5 samples/s +Step 45690/50462 | Loss: 2.5117 | LR: 1.13e-05 | 5158.2ms/step | 11.6 samples/s +Step 45700/50462 | Loss: 2.5041 | LR: 1.12e-05 | 5153.4ms/step | 11.6 samples/s +Step 45710/50462 | Loss: 2.5068 | LR: 1.12e-05 | 5164.0ms/step | 11.6 samples/s +Step 45720/50462 | Loss: 2.4981 | LR: 1.11e-05 | 5219.5ms/step | 11.5 samples/s +Step 45730/50462 | Loss: 2.4800 | LR: 1.11e-05 | 5165.7ms/step | 11.6 samples/s +Step 45740/50462 | Loss: 2.4934 | LR: 1.10e-05 | 5155.9ms/step | 11.6 samples/s +Step 45750/50462 | Loss: 2.5114 | LR: 1.10e-05 | 5164.5ms/step | 11.6 samples/s +Validation at step 45750: Loss = 2.5121, Perplexity = 12.33 +Step 45760/50462 | Loss: 2.4660 | LR: 1.10e-05 | 7716.8ms/step | 7.8 samples/s +Step 45770/50462 | Loss: 2.4983 | LR: 1.09e-05 | 5214.8ms/step | 11.5 samples/s +Step 45780/50462 | Loss: 2.4909 | LR: 1.09e-05 | 5154.2ms/step | 11.6 samples/s +Step 45790/50462 | Loss: 2.5291 | LR: 1.08e-05 | 5175.8ms/step | 11.6 samples/s +Step 45800/50462 | Loss: 2.4425 | LR: 1.08e-05 | 5178.7ms/step | 11.6 samples/s +Step 45810/50462 | Loss: 2.5032 | LR: 1.07e-05 | 5170.0ms/step | 11.6 samples/s +Step 45820/50462 | Loss: 2.4993 | LR: 1.07e-05 | 5165.5ms/step | 11.6 samples/s +Step 45830/50462 | Loss: 2.5056 | LR: 1.06e-05 | 5180.5ms/step | 11.6 samples/s +Step 45840/50462 | Loss: 2.5188 | LR: 1.06e-05 | 5169.1ms/step | 11.6 samples/s +Step 45850/50462 | Loss: 2.5006 | LR: 1.05e-05 | 5168.4ms/step | 11.6 samples/s +Step 45860/50462 | Loss: 2.5256 | LR: 1.05e-05 | 5157.1ms/step | 11.6 samples/s +Step 45870/50462 | Loss: 2.5105 | LR: 1.05e-05 | 5170.1ms/step | 11.6 samples/s +Step 45880/50462 | Loss: 2.5114 | LR: 1.04e-05 | 5159.2ms/step | 11.6 samples/s +Step 45890/50462 | Loss: 2.4737 | LR: 1.04e-05 | 5188.2ms/step | 11.6 samples/s +Step 45900/50462 | Loss: 2.4968 | LR: 1.03e-05 | 5173.0ms/step | 11.6 samples/s +Step 45910/50462 | Loss: 2.5271 | LR: 1.03e-05 | 5174.1ms/step | 11.6 samples/s +Step 45920/50462 | Loss: 2.4798 | LR: 1.02e-05 | 5170.7ms/step | 11.6 samples/s +Step 45930/50462 | Loss: 2.4918 | LR: 1.02e-05 | 5175.8ms/step | 11.6 samples/s +Step 45940/50462 | Loss: 2.5108 | LR: 1.01e-05 | 5204.0ms/step | 11.5 samples/s +Step 45950/50462 | Loss: 2.4982 | LR: 1.01e-05 | 5175.1ms/step | 11.6 samples/s +Step 45960/50462 | Loss: 2.4840 | LR: 1.01e-05 | 5173.7ms/step | 11.6 samples/s +Step 45970/50462 | Loss: 2.4895 | LR: 1.00e-05 | 5173.1ms/step | 11.6 samples/s +Step 45980/50462 | Loss: 2.5332 | LR: 9.96e-06 | 5171.5ms/step | 11.6 samples/s +Step 45990/50462 | Loss: 2.4876 | LR: 9.92e-06 | 5201.2ms/step | 11.5 samples/s +Step 46000/50462 | Loss: 2.5017 | LR: 9.87e-06 | 5176.3ms/step | 11.6 samples/s +Validation at step 46000: Loss = 2.5121, Perplexity = 12.33 +Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-46000 +Step 46010/50462 | Loss: 2.5043 | LR: 9.83e-06 | 8694.1ms/step | 6.9 samples/s +Step 46020/50462 | Loss: 2.4909 | LR: 9.79e-06 | 5162.0ms/step | 11.6 samples/s +Step 46030/50462 | Loss: 2.5147 | LR: 9.74e-06 | 5155.4ms/step | 11.6 samples/s +Step 46040/50462 | Loss: 2.5145 | LR: 9.70e-06 | 5208.4ms/step | 11.5 samples/s +Step 46050/50462 | Loss: 2.4978 | LR: 9.65e-06 | 5165.6ms/step | 11.6 samples/s +Step 46060/50462 | Loss: 2.4923 | LR: 9.61e-06 | 5163.9ms/step | 11.6 samples/s +Step 46070/50462 | Loss: 2.5114 | LR: 9.57e-06 | 5160.8ms/step | 11.6 samples/s +Step 46080/50462 | Loss: 2.5065 | LR: 9.52e-06 | 5154.8ms/step | 11.6 samples/s +Step 46090/50462 | Loss: 2.4753 | LR: 9.48e-06 | 5156.4ms/step | 11.6 samples/s +Step 46100/50462 | Loss: 2.5017 | LR: 9.44e-06 | 5155.8ms/step | 11.6 samples/s +Step 46110/50462 | Loss: 2.4972 | LR: 9.40e-06 | 5152.3ms/step | 11.6 samples/s +Step 46120/50462 | Loss: 2.4924 | LR: 9.35e-06 | 5165.7ms/step | 11.6 samples/s +Step 46130/50462 | Loss: 2.4939 | LR: 9.31e-06 | 5150.6ms/step | 11.6 samples/s +Step 46140/50462 | Loss: 2.4942 | LR: 9.27e-06 | 5162.9ms/step | 11.6 samples/s +Step 46150/50462 | Loss: 2.4926 | LR: 9.22e-06 | 5160.8ms/step | 11.6 samples/s +Step 46160/50462 | Loss: 2.5103 | LR: 9.18e-06 | 5146.4ms/step | 11.7 samples/s +Step 46170/50462 | Loss: 2.5108 | LR: 9.14e-06 | 5158.1ms/step | 11.6 samples/s +Step 46180/50462 | Loss: 2.4806 | LR: 9.10e-06 | 5135.5ms/step | 11.7 samples/s +Step 46190/50462 | Loss: 2.4809 | LR: 9.06e-06 | 5144.6ms/step | 11.7 samples/s +Step 46200/50462 | Loss: 2.4887 | LR: 9.01e-06 | 5204.1ms/step | 11.5 samples/s +Step 46210/50462 | Loss: 2.5053 | LR: 8.97e-06 | 5156.6ms/step | 11.6 samples/s +Step 46220/50462 | Loss: 2.5080 | LR: 8.93e-06 | 5161.9ms/step | 11.6 samples/s +Step 46230/50462 | Loss: 2.5142 | LR: 8.89e-06 | 5145.2ms/step | 11.7 samples/s +Step 46240/50462 | Loss: 2.5229 | LR: 8.85e-06 | 5159.1ms/step | 11.6 samples/s +Step 46250/50462 | Loss: 2.5206 | LR: 8.80e-06 | 5156.8ms/step | 11.6 samples/s +Validation at step 46250: Loss = 2.5121, Perplexity = 12.33 +Step 46260/50462 | Loss: 2.5207 | LR: 8.76e-06 | 7757.1ms/step | 7.7 samples/s +Step 46270/50462 | Loss: 2.5073 | LR: 8.72e-06 | 5188.2ms/step | 11.6 samples/s +Step 46280/50462 | Loss: 2.5386 | LR: 8.68e-06 | 5176.2ms/step | 11.6 samples/s +Step 46290/50462 | Loss: 2.5252 | LR: 8.64e-06 | 5168.7ms/step | 11.6 samples/s +Step 46300/50462 | Loss: 2.4959 | LR: 8.60e-06 | 5174.0ms/step | 11.6 samples/s +Step 46310/50462 | Loss: 2.5035 | LR: 8.56e-06 | 5168.8ms/step | 11.6 samples/s +Step 46320/50462 | Loss: 2.5289 | LR: 8.52e-06 | 5195.4ms/step | 11.5 samples/s +Step 46330/50462 | Loss: 2.5086 | LR: 8.47e-06 | 5155.3ms/step | 11.6 samples/s +Step 46340/50462 | Loss: 2.5083 | LR: 8.43e-06 | 5143.4ms/step | 11.7 samples/s +Step 46350/50462 | Loss: 2.5293 | LR: 8.39e-06 | 5147.4ms/step | 11.7 samples/s +Step 46360/50462 | Loss: 2.4921 | LR: 8.35e-06 | 5137.0ms/step | 11.7 samples/s +Step 46370/50462 | Loss: 2.4777 | LR: 8.31e-06 | 5134.7ms/step | 11.7 samples/s +Step 46380/50462 | Loss: 2.5026 | LR: 8.27e-06 | 5156.8ms/step | 11.6 samples/s +Step 46390/50462 | Loss: 2.4917 | LR: 8.23e-06 | 5148.3ms/step | 11.7 samples/s +Step 46400/50462 | Loss: 2.4833 | LR: 8.19e-06 | 5155.5ms/step | 11.6 samples/s +Step 46410/50462 | Loss: 2.4958 | LR: 8.15e-06 | 5139.3ms/step | 11.7 samples/s +Step 46420/50462 | Loss: 2.4957 | LR: 8.11e-06 | 5151.1ms/step | 11.6 samples/s +Step 46430/50462 | Loss: 2.5162 | LR: 8.07e-06 | 5155.7ms/step | 11.6 samples/s +Step 46440/50462 | Loss: 2.4991 | LR: 8.03e-06 | 5162.5ms/step | 11.6 samples/s +Step 46450/50462 | Loss: 2.5249 | LR: 7.99e-06 | 5141.0ms/step | 11.7 samples/s +Step 46460/50462 | Loss: 2.5032 | LR: 7.95e-06 | 5192.0ms/step | 11.6 samples/s +Step 46470/50462 | Loss: 2.4944 | LR: 7.91e-06 | 5157.5ms/step | 11.6 samples/s +Step 46480/50462 | Loss: 2.4895 | LR: 7.87e-06 | 5151.6ms/step | 11.6 samples/s +Step 46490/50462 | Loss: 2.4856 | LR: 7.83e-06 | 5151.3ms/step | 11.6 samples/s +Step 46500/50462 | Loss: 2.5026 | LR: 7.80e-06 | 5141.7ms/step | 11.7 samples/s +Validation at step 46500: Loss = 2.5121, Perplexity = 12.33 +Step 46510/50462 | Loss: 2.4972 | LR: 7.76e-06 | 7702.3ms/step | 7.8 samples/s +Step 46520/50462 | Loss: 2.5205 | LR: 7.72e-06 | 5165.0ms/step | 11.6 samples/s +Step 46530/50462 | Loss: 2.5005 | LR: 7.68e-06 | 5215.7ms/step | 11.5 samples/s +Step 46540/50462 | Loss: 2.5134 | LR: 7.64e-06 | 5162.7ms/step | 11.6 samples/s +Step 46550/50462 | Loss: 2.4748 | LR: 7.60e-06 | 5156.6ms/step | 11.6 samples/s +Step 46560/50462 | Loss: 2.4918 | LR: 7.56e-06 | 5166.6ms/step | 11.6 samples/s +Step 46570/50462 | Loss: 2.5024 | LR: 7.52e-06 | 5162.7ms/step | 11.6 samples/s +Step 46580/50462 | Loss: 2.4943 | LR: 7.49e-06 | 5165.8ms/step | 11.6 samples/s +Step 46590/50462 | Loss: 2.4869 | LR: 7.45e-06 | 5201.1ms/step | 11.5 samples/s +Step 46600/50462 | Loss: 2.5020 | LR: 7.41e-06 | 5182.2ms/step | 11.6 samples/s +Step 46610/50462 | Loss: 2.5003 | LR: 7.37e-06 | 5164.7ms/step | 11.6 samples/s +Step 46620/50462 | Loss: 2.4830 | LR: 7.33e-06 | 5165.9ms/step | 11.6 samples/s +Step 46630/50462 | Loss: 2.4972 | LR: 7.29e-06 | 5159.5ms/step | 11.6 samples/s +Step 46640/50462 | Loss: 2.5092 | LR: 7.26e-06 | 5163.1ms/step | 11.6 samples/s +Step 46650/50462 | Loss: 2.5013 | LR: 7.22e-06 | 5155.6ms/step | 11.6 samples/s +Step 46660/50462 | Loss: 2.4912 | LR: 7.18e-06 | 5141.4ms/step | 11.7 samples/s +Step 46670/50462 | Loss: 2.5060 | LR: 7.14e-06 | 5150.0ms/step | 11.7 samples/s +Step 46680/50462 | Loss: 2.5145 | LR: 7.11e-06 | 5165.3ms/step | 11.6 samples/s +Step 46690/50462 | Loss: 2.5314 | LR: 7.07e-06 | 5160.2ms/step | 11.6 samples/s +Step 46700/50462 | Loss: 2.4918 | LR: 7.03e-06 | 5153.5ms/step | 11.6 samples/s +Step 46710/50462 | Loss: 2.5247 | LR: 6.99e-06 | 5163.8ms/step | 11.6 samples/s +Step 46720/50462 | Loss: 2.5083 | LR: 6.96e-06 | 5206.7ms/step | 11.5 samples/s +Step 46730/50462 | Loss: 2.4882 | LR: 6.92e-06 | 5164.7ms/step | 11.6 samples/s +Step 46740/50462 | Loss: 2.5148 | LR: 6.88e-06 | 5160.4ms/step | 11.6 samples/s +Step 46750/50462 | Loss: 2.4976 | LR: 6.85e-06 | 5156.4ms/step | 11.6 samples/s +Validation at step 46750: Loss = 2.5121, Perplexity = 12.33 +Step 46760/50462 | Loss: 2.5122 | LR: 6.81e-06 | 7726.0ms/step | 7.8 samples/s +Step 46770/50462 | Loss: 2.5018 | LR: 6.77e-06 | 5178.9ms/step | 11.6 samples/s +Step 46780/50462 | Loss: 2.5271 | LR: 6.74e-06 | 5178.6ms/step | 11.6 samples/s +Step 46790/50462 | Loss: 2.4844 | LR: 6.70e-06 | 5154.5ms/step | 11.6 samples/s +Step 46800/50462 | Loss: 2.5224 | LR: 6.66e-06 | 5171.0ms/step | 11.6 samples/s +Step 46810/50462 | Loss: 2.5185 | LR: 6.63e-06 | 5215.6ms/step | 11.5 samples/s +Step 46820/50462 | Loss: 2.4818 | LR: 6.59e-06 | 5149.9ms/step | 11.7 samples/s +Step 46830/50462 | Loss: 2.4800 | LR: 6.56e-06 | 5153.9ms/step | 11.6 samples/s +Step 46840/50462 | Loss: 2.5056 | LR: 6.52e-06 | 5151.8ms/step | 11.6 samples/s +Step 46850/50462 | Loss: 2.5016 | LR: 6.48e-06 | 5156.7ms/step | 11.6 samples/s +Step 46860/50462 | Loss: 2.5011 | LR: 6.45e-06 | 5152.8ms/step | 11.6 samples/s +Step 46870/50462 | Loss: 2.4593 | LR: 6.41e-06 | 5206.8ms/step | 11.5 samples/s +Step 46880/50462 | Loss: 2.4948 | LR: 6.38e-06 | 5166.4ms/step | 11.6 samples/s +Step 46890/50462 | Loss: 2.5007 | LR: 6.34e-06 | 5153.6ms/step | 11.6 samples/s +Step 46900/50462 | Loss: 2.4931 | LR: 6.31e-06 | 5163.3ms/step | 11.6 samples/s +Step 46910/50462 | Loss: 2.4861 | LR: 6.27e-06 | 5160.7ms/step | 11.6 samples/s +Step 46920/50462 | Loss: 2.5241 | LR: 6.24e-06 | 5154.6ms/step | 11.6 samples/s +Step 46930/50462 | Loss: 2.5003 | LR: 6.20e-06 | 5165.4ms/step | 11.6 samples/s +Step 46940/50462 | Loss: 2.5022 | LR: 6.17e-06 | 5170.9ms/step | 11.6 samples/s +Step 46950/50462 | Loss: 2.4854 | LR: 6.13e-06 | 5158.9ms/step | 11.6 samples/s +Step 46960/50462 | Loss: 2.5086 | LR: 6.10e-06 | 5172.0ms/step | 11.6 samples/s +Step 46970/50462 | Loss: 2.5124 | LR: 6.06e-06 | 5157.0ms/step | 11.6 samples/s +Step 46980/50462 | Loss: 2.5115 | LR: 6.03e-06 | 5164.9ms/step | 11.6 samples/s +Step 46990/50462 | Loss: 2.5326 | LR: 5.99e-06 | 5197.7ms/step | 11.5 samples/s +Step 47000/50462 | Loss: 2.5122 | LR: 5.96e-06 | 5158.4ms/step | 11.6 samples/s +Validation at step 47000: Loss = 2.5121, Perplexity = 12.33 +Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-47000 +Step 47010/50462 | Loss: 2.5239 | LR: 5.93e-06 | 8654.8ms/step | 6.9 samples/s +Step 47020/50462 | Loss: 2.5006 | LR: 5.89e-06 | 5159.7ms/step | 11.6 samples/s +Step 47030/50462 | Loss: 2.5206 | LR: 5.86e-06 | 5163.1ms/step | 11.6 samples/s +Step 47040/50462 | Loss: 2.4658 | LR: 5.82e-06 | 5160.4ms/step | 11.6 samples/s +Step 47050/50462 | Loss: 2.5023 | LR: 5.79e-06 | 5148.0ms/step | 11.7 samples/s +Step 47060/50462 | Loss: 2.5007 | LR: 5.76e-06 | 5158.2ms/step | 11.6 samples/s +Step 47070/50462 | Loss: 2.4936 | LR: 5.72e-06 | 5158.2ms/step | 11.6 samples/s +Step 47080/50462 | Loss: 2.5014 | LR: 5.69e-06 | 5227.3ms/step | 11.5 samples/s +Step 47090/50462 | Loss: 2.4962 | LR: 5.65e-06 | 5167.7ms/step | 11.6 samples/s +Step 47100/50462 | Loss: 2.5002 | LR: 5.62e-06 | 5156.2ms/step | 11.6 samples/s +Step 47110/50462 | Loss: 2.5039 | LR: 5.59e-06 | 5156.0ms/step | 11.6 samples/s +Step 47120/50462 | Loss: 2.4543 | LR: 5.56e-06 | 5151.7ms/step | 11.6 samples/s +Step 47130/50462 | Loss: 2.5012 | LR: 5.52e-06 | 5134.2ms/step | 11.7 samples/s +Step 47140/50462 | Loss: 2.5135 | LR: 5.49e-06 | 5199.2ms/step | 11.5 samples/s +Step 47150/50462 | Loss: 2.4698 | LR: 5.46e-06 | 5175.1ms/step | 11.6 samples/s +Step 47160/50462 | Loss: 2.5117 | LR: 5.42e-06 | 5157.5ms/step | 11.6 samples/s +Step 47170/50462 | Loss: 2.4945 | LR: 5.39e-06 | 5137.3ms/step | 11.7 samples/s +Step 47180/50462 | Loss: 2.5041 | LR: 5.36e-06 | 5155.3ms/step | 11.6 samples/s +Step 47190/50462 | Loss: 2.4985 | LR: 5.33e-06 | 5161.5ms/step | 11.6 samples/s +Step 47200/50462 | Loss: 2.4966 | LR: 5.29e-06 | 5162.1ms/step | 11.6 samples/s +Step 47210/50462 | Loss: 2.4706 | LR: 5.26e-06 | 5167.6ms/step | 11.6 samples/s +Step 47220/50462 | Loss: 2.5113 | LR: 5.23e-06 | 5177.5ms/step | 11.6 samples/s +Step 47230/50462 | Loss: 2.5095 | LR: 5.20e-06 | 5167.2ms/step | 11.6 samples/s +Step 47240/50462 | Loss: 2.4779 | LR: 5.16e-06 | 5154.2ms/step | 11.6 samples/s +Step 47250/50462 | Loss: 2.5085 | LR: 5.13e-06 | 5232.1ms/step | 11.5 samples/s +Validation at step 47250: Loss = 2.5121, Perplexity = 12.33 +Step 47260/50462 | Loss: 2.5029 | LR: 5.10e-06 | 7708.3ms/step | 7.8 samples/s +Step 47270/50462 | Loss: 2.5147 | LR: 5.07e-06 | 5155.7ms/step | 11.6 samples/s +Step 47280/50462 | Loss: 2.4883 | LR: 5.04e-06 | 5175.8ms/step | 11.6 samples/s +Step 47290/50462 | Loss: 2.5163 | LR: 5.01e-06 | 5173.0ms/step | 11.6 samples/s +Step 47300/50462 | Loss: 2.5146 | LR: 4.97e-06 | 5144.1ms/step | 11.7 samples/s +Step 47310/50462 | Loss: 2.4767 | LR: 4.94e-06 | 5174.1ms/step | 11.6 samples/s +Step 47320/50462 | Loss: 2.4937 | LR: 4.91e-06 | 5156.7ms/step | 11.6 samples/s +Step 47330/50462 | Loss: 2.4967 | LR: 4.88e-06 | 5169.2ms/step | 11.6 samples/s +Step 47340/50462 | Loss: 2.4616 | LR: 4.85e-06 | 5178.0ms/step | 11.6 samples/s +Step 47350/50462 | Loss: 2.5092 | LR: 4.82e-06 | 5166.1ms/step | 11.6 samples/s +Step 47360/50462 | Loss: 2.4798 | LR: 4.79e-06 | 5229.1ms/step | 11.5 samples/s +Step 47370/50462 | Loss: 2.5180 | LR: 4.76e-06 | 5172.2ms/step | 11.6 samples/s +Step 47380/50462 | Loss: 2.5103 | LR: 4.73e-06 | 5156.9ms/step | 11.6 samples/s +Step 47390/50462 | Loss: 2.4930 | LR: 4.70e-06 | 5159.7ms/step | 11.6 samples/s +Step 47400/50462 | Loss: 2.4963 | LR: 4.67e-06 | 5155.3ms/step | 11.6 samples/s +Step 47410/50462 | Loss: 2.5076 | LR: 4.64e-06 | 5203.4ms/step | 11.5 samples/s +Step 47420/50462 | Loss: 2.4593 | LR: 4.61e-06 | 5153.7ms/step | 11.6 samples/s +Step 47430/50462 | Loss: 2.4873 | LR: 4.58e-06 | 5166.8ms/step | 11.6 samples/s +Step 47440/50462 | Loss: 2.4960 | LR: 4.55e-06 | 5165.0ms/step | 11.6 samples/s +Step 47450/50462 | Loss: 2.4708 | LR: 4.52e-06 | 5160.9ms/step | 11.6 samples/s +Step 47460/50462 | Loss: 2.5154 | LR: 4.49e-06 | 5161.0ms/step | 11.6 samples/s +Step 47470/50462 | Loss: 2.5133 | LR: 4.46e-06 | 5157.8ms/step | 11.6 samples/s +Step 47480/50462 | Loss: 2.5120 | LR: 4.43e-06 | 5165.1ms/step | 11.6 samples/s +Step 47490/50462 | Loss: 2.5050 | LR: 4.40e-06 | 5164.8ms/step | 11.6 samples/s +Step 47500/50462 | Loss: 2.4962 | LR: 4.37e-06 | 5162.8ms/step | 11.6 samples/s +Validation at step 47500: Loss = 2.5121, Perplexity = 12.33 +Step 47510/50462 | Loss: 2.4670 | LR: 4.34e-06 | 7753.0ms/step | 7.7 samples/s +Step 47520/50462 | Loss: 2.5150 | LR: 4.31e-06 | 5169.2ms/step | 11.6 samples/s +Step 47530/50462 | Loss: 2.5103 | LR: 4.28e-06 | 5163.0ms/step | 11.6 samples/s +Step 47540/50462 | Loss: 2.4787 | LR: 4.25e-06 | 5181.4ms/step | 11.6 samples/s +Step 47550/50462 | Loss: 2.5080 | LR: 4.22e-06 | 5166.9ms/step | 11.6 samples/s +Step 47560/50462 | Loss: 2.4934 | LR: 4.19e-06 | 5159.8ms/step | 11.6 samples/s +Step 47570/50462 | Loss: 2.5036 | LR: 4.16e-06 | 5169.6ms/step | 11.6 samples/s +Step 47580/50462 | Loss: 2.5389 | LR: 4.13e-06 | 5158.9ms/step | 11.6 samples/s +Step 47590/50462 | Loss: 2.5096 | LR: 4.11e-06 | 5141.6ms/step | 11.7 samples/s +Step 47600/50462 | Loss: 2.5005 | LR: 4.08e-06 | 5161.7ms/step | 11.6 samples/s +Step 47610/50462 | Loss: 2.4803 | LR: 4.05e-06 | 5154.9ms/step | 11.6 samples/s +Step 47620/50462 | Loss: 2.4942 | LR: 4.02e-06 | 5167.2ms/step | 11.6 samples/s +Step 47630/50462 | Loss: 2.5213 | LR: 3.99e-06 | 5207.7ms/step | 11.5 samples/s +Step 47640/50462 | Loss: 2.4764 | LR: 3.97e-06 | 5159.7ms/step | 11.6 samples/s +Step 47650/50462 | Loss: 2.4956 | LR: 3.94e-06 | 5153.7ms/step | 11.6 samples/s +Step 47660/50462 | Loss: 2.4947 | LR: 3.91e-06 | 5162.2ms/step | 11.6 samples/s +Step 47670/50462 | Loss: 2.4868 | LR: 3.88e-06 | 5164.1ms/step | 11.6 samples/s +Step 47680/50462 | Loss: 2.5052 | LR: 3.85e-06 | 5163.9ms/step | 11.6 samples/s +Step 47690/50462 | Loss: 2.5007 | LR: 3.83e-06 | 5197.5ms/step | 11.5 samples/s +Step 47700/50462 | Loss: 2.5088 | LR: 3.80e-06 | 5157.6ms/step | 11.6 samples/s +Step 47710/50462 | Loss: 2.4749 | LR: 3.77e-06 | 5158.7ms/step | 11.6 samples/s +Step 47720/50462 | Loss: 2.5018 | LR: 3.74e-06 | 5147.0ms/step | 11.7 samples/s +Step 47730/50462 | Loss: 2.5027 | LR: 3.72e-06 | 5164.5ms/step | 11.6 samples/s +Step 47740/50462 | Loss: 2.4877 | LR: 3.69e-06 | 5172.5ms/step | 11.6 samples/s +Step 47750/50462 | Loss: 2.5147 | LR: 3.66e-06 | 5163.1ms/step | 11.6 samples/s +Validation at step 47750: Loss = 2.5121, Perplexity = 12.33 +Step 47760/50462 | Loss: 2.4721 | LR: 3.64e-06 | 7694.0ms/step | 7.8 samples/s +Step 47770/50462 | Loss: 2.5342 | LR: 3.61e-06 | 5144.6ms/step | 11.7 samples/s +Step 47780/50462 | Loss: 2.4917 | LR: 3.58e-06 | 5216.3ms/step | 11.5 samples/s +Step 47790/50462 | Loss: 2.4864 | LR: 3.56e-06 | 5151.4ms/step | 11.6 samples/s +Step 47800/50462 | Loss: 2.4780 | LR: 3.53e-06 | 5157.5ms/step | 11.6 samples/s +Step 47810/50462 | Loss: 2.5113 | LR: 3.50e-06 | 5155.8ms/step | 11.6 samples/s +Step 47820/50462 | Loss: 2.5045 | LR: 3.48e-06 | 5152.0ms/step | 11.6 samples/s +Step 47830/50462 | Loss: 2.4974 | LR: 3.45e-06 | 5155.1ms/step | 11.6 samples/s +Step 47840/50462 | Loss: 2.5197 | LR: 3.42e-06 | 5158.4ms/step | 11.6 samples/s +Step 47850/50462 | Loss: 2.5102 | LR: 3.40e-06 | 5159.5ms/step | 11.6 samples/s +Step 47860/50462 | Loss: 2.5130 | LR: 3.37e-06 | 5156.5ms/step | 11.6 samples/s +Step 47870/50462 | Loss: 2.5065 | LR: 3.35e-06 | 5173.1ms/step | 11.6 samples/s +Step 47880/50462 | Loss: 2.4612 | LR: 3.32e-06 | 5151.7ms/step | 11.6 samples/s +Step 47890/50462 | Loss: 2.5190 | LR: 3.30e-06 | 5155.2ms/step | 11.6 samples/s +Step 47900/50462 | Loss: 2.5069 | LR: 3.27e-06 | 5176.8ms/step | 11.6 samples/s +Step 47910/50462 | Loss: 2.5068 | LR: 3.24e-06 | 5174.2ms/step | 11.6 samples/s +Step 47920/50462 | Loss: 2.4809 | LR: 3.22e-06 | 5166.4ms/step | 11.6 samples/s +Step 47930/50462 | Loss: 2.5179 | LR: 3.19e-06 | 5172.6ms/step | 11.6 samples/s +Step 47940/50462 | Loss: 2.4952 | LR: 3.17e-06 | 5176.6ms/step | 11.6 samples/s +Step 47950/50462 | Loss: 2.5012 | LR: 3.14e-06 | 5172.3ms/step | 11.6 samples/s +Step 47960/50462 | Loss: 2.4683 | LR: 3.12e-06 | 5201.4ms/step | 11.5 samples/s +Step 47970/50462 | Loss: 2.5050 | LR: 3.09e-06 | 5183.8ms/step | 11.6 samples/s +Step 47980/50462 | Loss: 2.5016 | LR: 3.07e-06 | 5172.0ms/step | 11.6 samples/s +Step 47990/50462 | Loss: 2.4848 | LR: 3.04e-06 | 5170.3ms/step | 11.6 samples/s +Step 48000/50462 | Loss: 2.4946 | LR: 3.02e-06 | 5178.2ms/step | 11.6 samples/s +Validation at step 48000: Loss = 2.5121, Perplexity = 12.33 +Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-48000 +Step 48010/50462 | Loss: 2.5078 | LR: 3.00e-06 | 8664.5ms/step | 6.9 samples/s +Step 48020/50462 | Loss: 2.5358 | LR: 2.97e-06 | 5166.7ms/step | 11.6 samples/s +Step 48030/50462 | Loss: 2.4779 | LR: 2.95e-06 | 5196.9ms/step | 11.5 samples/s +Step 48040/50462 | Loss: 2.4916 | LR: 2.92e-06 | 5163.9ms/step | 11.6 samples/s +Step 48050/50462 | Loss: 2.4842 | LR: 2.90e-06 | 5166.4ms/step | 11.6 samples/s +Step 48060/50462 | Loss: 2.5014 | LR: 2.87e-06 | 5165.8ms/step | 11.6 samples/s +Step 48070/50462 | Loss: 2.5037 | LR: 2.85e-06 | 5159.4ms/step | 11.6 samples/s +Step 48080/50462 | Loss: 2.5276 | LR: 2.83e-06 | 5160.8ms/step | 11.6 samples/s +Step 48090/50462 | Loss: 2.5037 | LR: 2.80e-06 | 5167.1ms/step | 11.6 samples/s +Step 48100/50462 | Loss: 2.5066 | LR: 2.78e-06 | 5155.8ms/step | 11.6 samples/s +Step 48110/50462 | Loss: 2.4964 | LR: 2.76e-06 | 5158.6ms/step | 11.6 samples/s +Step 48120/50462 | Loss: 2.4944 | LR: 2.73e-06 | 5166.9ms/step | 11.6 samples/s +Step 48130/50462 | Loss: 2.4955 | LR: 2.71e-06 | 5162.1ms/step | 11.6 samples/s +Step 48140/50462 | Loss: 2.4773 | LR: 2.69e-06 | 5148.5ms/step | 11.7 samples/s +Step 48150/50462 | Loss: 2.4907 | LR: 2.66e-06 | 5151.3ms/step | 11.6 samples/s +Step 48160/50462 | Loss: 2.5102 | LR: 2.64e-06 | 5146.5ms/step | 11.7 samples/s +Step 48170/50462 | Loss: 2.4898 | LR: 2.62e-06 | 5152.4ms/step | 11.6 samples/s +Step 48180/50462 | Loss: 2.5025 | LR: 2.60e-06 | 5207.4ms/step | 11.5 samples/s +Step 48190/50462 | Loss: 2.4987 | LR: 2.57e-06 | 5148.8ms/step | 11.7 samples/s +Step 48200/50462 | Loss: 2.5081 | LR: 2.55e-06 | 5141.7ms/step | 11.7 samples/s +Step 48210/50462 | Loss: 2.4840 | LR: 2.53e-06 | 5151.3ms/step | 11.6 samples/s +Step 48220/50462 | Loss: 2.4900 | LR: 2.51e-06 | 5145.2ms/step | 11.7 samples/s +Step 48230/50462 | Loss: 2.4973 | LR: 2.48e-06 | 5148.1ms/step | 11.7 samples/s +Step 48240/50462 | Loss: 2.4952 | LR: 2.46e-06 | 5224.1ms/step | 11.5 samples/s +Step 48250/50462 | Loss: 2.5342 | LR: 2.44e-06 | 5171.5ms/step | 11.6 samples/s +Validation at step 48250: Loss = 2.5121, Perplexity = 12.33 +Step 48260/50462 | Loss: 2.4716 | LR: 2.42e-06 | 7717.7ms/step | 7.8 samples/s +Step 48270/50462 | Loss: 2.4975 | LR: 2.39e-06 | 5150.2ms/step | 11.6 samples/s +Step 48280/50462 | Loss: 2.5058 | LR: 2.37e-06 | 5152.5ms/step | 11.6 samples/s +Step 48290/50462 | Loss: 2.4933 | LR: 2.35e-06 | 5161.0ms/step | 11.6 samples/s +Step 48300/50462 | Loss: 2.4867 | LR: 2.33e-06 | 5275.4ms/step | 11.4 samples/s diff --git a/epfl_final/log_files/cont_pretrain_8k.log b/epfl_final/log_files/cont_pretrain_8k.log new file mode 100644 index 0000000000000000000000000000000000000000..1f0707a73cacca1f5e2e9f73f99eb3223c551a0c --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_8k.log @@ -0,0 +1,325 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000", + "num_epochs": 1, + "max_steps": 20005, + "per_device_batch_size": 3, + "gradient_accumulation_steps": 8, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 250, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 200, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 8k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260513_164613-40cb81kv +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense continued 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/40cb81kv + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1024 + num_hidden_layers: 28 + num_attention_heads: 16 + num_key_value_heads: 8 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 596,049,920 + Trainable parameters: 596,049,920 + Non-trainable parameters: 0 + Model size (BF16): 1.19 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [3, 8192] -- -- True +├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True +│ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True +├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True +================================================================================================================================================================ +Total params: 751,632,384 +Trainable params: 751,632,384 +Non-trainable params: 0 +Total mult-adds (G): 2.25 +================================================================================================================================================================ +Input size (MB): 0.20 +Forward/backward pass size (MB): 63052.97 +Params size (MB): 1503.26 +Estimated Total Size (MB): 64556.43 +================================================================================================================================================================ +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003) +Dataset split: 1017602 train, 254401 validation +Training examples (packed, train split): 1017602 +Validation examples (packed, val split): 254401 +Per-device batch size: 3 +World size (GPUs): 1 +Gradient accumulation steps: 8 +Effective batch size (seqs/step): 24 +Sequence length: 8192 +Tokens per optimizer step: 196,608 +Micro-batches per epoch (per rank): 339200 +Optimizer steps per epoch: 42400 +Total optimizer steps: 20005 (capped by max_steps; 0.47 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000... +Warning: no RNG state file found for rank 0; RNG will not be restored. +Resumed from checkpoint at step 7999 (epoch 0, micro_step 0) +Resuming training from step 7999 +Starting training... +Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run. +Step 8000/20005 | Loss: 0.3005 | LR: 3.33e-04 | 60738.9ms/step | 0.4 samples/s +Validation at step 8000: Loss = 3.0463, Perplexity = 21.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000 +Step 8010/20005 | Loss: 3.0209 | LR: 3.33e-04 | 43173.7ms/step | 0.6 samples/s +Step 8020/20005 | Loss: 3.0490 | LR: 3.32e-04 | 5895.2ms/step | 4.1 samples/s +Step 8030/20005 | Loss: 3.0187 | LR: 3.32e-04 | 5845.3ms/step | 4.1 samples/s +Step 8040/20005 | Loss: 3.0369 | LR: 3.31e-04 | 5844.8ms/step | 4.1 samples/s +Step 8050/20005 | Loss: 3.0430 | LR: 3.31e-04 | 5844.2ms/step | 4.1 samples/s +Step 8060/20005 | Loss: 3.0399 | LR: 3.31e-04 | 5844.5ms/step | 4.1 samples/s +Step 8070/20005 | Loss: 3.0496 | LR: 3.30e-04 | 5845.8ms/step | 4.1 samples/s +Step 8080/20005 | Loss: 2.9954 | LR: 3.30e-04 | 5844.4ms/step | 4.1 samples/s +Step 8090/20005 | Loss: 3.0354 | LR: 3.30e-04 | 5846.4ms/step | 4.1 samples/s +Step 8100/20005 | Loss: 3.0208 | LR: 3.29e-04 | 5849.0ms/step | 4.1 samples/s +Step 8110/20005 | Loss: 3.0160 | LR: 3.29e-04 | 5844.7ms/step | 4.1 samples/s +Step 8120/20005 | Loss: 3.0308 | LR: 3.28e-04 | 5849.4ms/step | 4.1 samples/s +Step 8130/20005 | Loss: 3.0105 | LR: 3.28e-04 | 5851.7ms/step | 4.1 samples/s +Step 8140/20005 | Loss: 3.0254 | LR: 3.28e-04 | 5846.8ms/step | 4.1 samples/s +Step 8150/20005 | Loss: 3.0212 | LR: 3.27e-04 | 5848.3ms/step | 4.1 samples/s +Step 8160/20005 | Loss: 2.9973 | LR: 3.27e-04 | 5845.1ms/step | 4.1 samples/s +Step 8170/20005 | Loss: 2.9866 | LR: 3.27e-04 | 5844.8ms/step | 4.1 samples/s +Step 8180/20005 | Loss: 3.0179 | LR: 3.26e-04 | 5844.1ms/step | 4.1 samples/s +Step 8190/20005 | Loss: 3.0313 | LR: 3.26e-04 | 5845.2ms/step | 4.1 samples/s +Step 8200/20005 | Loss: 3.0137 | LR: 3.25e-04 | 5844.7ms/step | 4.1 samples/s +Step 8210/20005 | Loss: 3.0414 | LR: 3.25e-04 | 5844.0ms/step | 4.1 samples/s +Step 8220/20005 | Loss: 3.0272 | LR: 3.25e-04 | 5846.0ms/step | 4.1 samples/s +Step 8230/20005 | Loss: 3.0355 | LR: 3.24e-04 | 5844.2ms/step | 4.1 samples/s +Step 8240/20005 | Loss: 3.0317 | LR: 3.24e-04 | 5846.5ms/step | 4.1 samples/s +Step 8250/20005 | Loss: 2.9886 | LR: 3.24e-04 | 5846.2ms/step | 4.1 samples/s +Validation at step 8250: Loss = 3.0269, Perplexity = 20.63 +Step 8260/20005 | Loss: 3.0043 | LR: 3.23e-04 | 40512.8ms/step | 0.6 samples/s +Step 8270/20005 | Loss: 3.0274 | LR: 3.23e-04 | 5844.8ms/step | 4.1 samples/s +Step 8280/20005 | Loss: 3.0327 | LR: 3.22e-04 | 5845.8ms/step | 4.1 samples/s +Step 8290/20005 | Loss: 3.0293 | LR: 3.22e-04 | 5848.5ms/step | 4.1 samples/s +Step 8300/20005 | Loss: 3.0040 | LR: 3.22e-04 | 5846.7ms/step | 4.1 samples/s +Step 8310/20005 | Loss: 3.0314 | LR: 3.21e-04 | 5845.0ms/step | 4.1 samples/s +Step 8320/20005 | Loss: 3.0232 | LR: 3.21e-04 | 5846.6ms/step | 4.1 samples/s +Step 8330/20005 | Loss: 3.0138 | LR: 3.20e-04 | 5849.4ms/step | 4.1 samples/s +Step 8340/20005 | Loss: 2.9915 | LR: 3.20e-04 | 5850.4ms/step | 4.1 samples/s +Step 8350/20005 | Loss: 3.0238 | LR: 3.20e-04 | 5873.6ms/step | 4.1 samples/s +Step 8360/20005 | Loss: 3.0174 | LR: 3.19e-04 | 5877.6ms/step | 4.1 samples/s +Step 8370/20005 | Loss: 3.0191 | LR: 3.19e-04 | 5881.5ms/step | 4.1 samples/s +Step 8380/20005 | Loss: 3.0175 | LR: 3.19e-04 | 5886.7ms/step | 4.1 samples/s +Step 8390/20005 | Loss: 2.9975 | LR: 3.18e-04 | 5889.4ms/step | 4.1 samples/s +Step 8400/20005 | Loss: 3.0001 | LR: 3.18e-04 | 5890.0ms/step | 4.1 samples/s +Step 8410/20005 | Loss: 3.0340 | LR: 3.17e-04 | 5891.4ms/step | 4.1 samples/s +Step 8420/20005 | Loss: 2.9922 | LR: 3.17e-04 | 5892.5ms/step | 4.1 samples/s +Step 8430/20005 | Loss: 2.9943 | LR: 3.17e-04 | 5896.1ms/step | 4.1 samples/s +Step 8440/20005 | Loss: 2.9912 | LR: 3.16e-04 | 5888.7ms/step | 4.1 samples/s +Step 8450/20005 | Loss: 3.0143 | LR: 3.16e-04 | 5892.4ms/step | 4.1 samples/s +Step 8460/20005 | Loss: 3.0145 | LR: 3.15e-04 | 5890.0ms/step | 4.1 samples/s +Step 8470/20005 | Loss: 3.0116 | LR: 3.15e-04 | 5887.8ms/step | 4.1 samples/s +Step 8480/20005 | Loss: 2.9898 | LR: 3.15e-04 | 5894.9ms/step | 4.1 samples/s +Step 8490/20005 | Loss: 3.0156 | LR: 3.14e-04 | 5895.3ms/step | 4.1 samples/s +Step 8500/20005 | Loss: 3.0061 | LR: 3.14e-04 | 5891.2ms/step | 4.1 samples/s +Validation at step 8500: Loss = 3.0173, Perplexity = 20.44 +Step 8510/20005 | Loss: 3.0116 | LR: 3.14e-04 | 40904.0ms/step | 0.6 samples/s +Step 8520/20005 | Loss: 2.9885 | LR: 3.13e-04 | 5926.6ms/step | 4.0 samples/s +Step 8530/20005 | Loss: 2.9842 | LR: 3.13e-04 | 5884.8ms/step | 4.1 samples/s +Step 8540/20005 | Loss: 2.9798 | LR: 3.12e-04 | 5889.1ms/step | 4.1 samples/s +Step 8550/20005 | Loss: 3.0528 | LR: 3.12e-04 | 5889.0ms/step | 4.1 samples/s +Step 8560/20005 | Loss: 3.0133 | LR: 3.12e-04 | 5890.7ms/step | 4.1 samples/s +Step 8570/20005 | Loss: 3.0188 | LR: 3.11e-04 | 5886.1ms/step | 4.1 samples/s +Step 8580/20005 | Loss: 2.9773 | LR: 3.11e-04 | 5889.0ms/step | 4.1 samples/s +Step 8590/20005 | Loss: 2.9922 | LR: 3.10e-04 | 5892.9ms/step | 4.1 samples/s +Step 8600/20005 | Loss: 3.0110 | LR: 3.10e-04 | 5891.2ms/step | 4.1 samples/s +Step 8610/20005 | Loss: 3.0045 | LR: 3.10e-04 | 5892.3ms/step | 4.1 samples/s +Step 8620/20005 | Loss: 3.0010 | LR: 3.09e-04 | 5896.6ms/step | 4.1 samples/s +Step 8630/20005 | Loss: 3.0084 | LR: 3.09e-04 | 5891.8ms/step | 4.1 samples/s +Step 8640/20005 | Loss: 3.0234 | LR: 3.09e-04 | 5885.1ms/step | 4.1 samples/s +Step 8650/20005 | Loss: 3.0314 | LR: 3.08e-04 | 5891.4ms/step | 4.1 samples/s +Step 8660/20005 | Loss: 2.9966 | LR: 3.08e-04 | 5881.4ms/step | 4.1 samples/s +Step 8670/20005 | Loss: 2.9927 | LR: 3.07e-04 | 5885.8ms/step | 4.1 samples/s +Step 8680/20005 | Loss: 2.9941 | LR: 3.07e-04 | 5893.6ms/step | 4.1 samples/s +Step 8690/20005 | Loss: 2.9849 | LR: 3.07e-04 | 5890.2ms/step | 4.1 samples/s +Step 8700/20005 | Loss: 2.9973 | LR: 3.06e-04 | 5872.2ms/step | 4.1 samples/s +Step 8710/20005 | Loss: 2.9828 | LR: 3.06e-04 | 5873.4ms/step | 4.1 samples/s +Step 8720/20005 | Loss: 2.9969 | LR: 3.05e-04 | 5872.7ms/step | 4.1 samples/s +Step 8730/20005 | Loss: 2.9769 | LR: 3.05e-04 | 5878.6ms/step | 4.1 samples/s +Step 8740/20005 | Loss: 2.9891 | LR: 3.05e-04 | 5876.6ms/step | 4.1 samples/s +Step 8750/20005 | Loss: 2.9813 | LR: 3.04e-04 | 5881.4ms/step | 4.1 samples/s +Validation at step 8750: Loss = 3.0081, Perplexity = 20.25 +Step 8760/20005 | Loss: 2.9837 | LR: 3.04e-04 | 40797.1ms/step | 0.6 samples/s +Step 8770/20005 | Loss: 3.0124 | LR: 3.04e-04 | 5866.5ms/step | 4.1 samples/s +Step 8780/20005 | Loss: 3.0050 | LR: 3.03e-04 | 5867.2ms/step | 4.1 samples/s +Step 8790/20005 | Loss: 2.9934 | LR: 3.03e-04 | 5865.9ms/step | 4.1 samples/s +Step 8800/20005 | Loss: 2.9902 | LR: 3.02e-04 | 5868.6ms/step | 4.1 samples/s +Step 8810/20005 | Loss: 2.9924 | LR: 3.02e-04 | 5866.2ms/step | 4.1 samples/s +Step 8820/20005 | Loss: 2.9850 | LR: 3.02e-04 | 5866.9ms/step | 4.1 samples/s +Step 8830/20005 | Loss: 2.9816 | LR: 3.01e-04 | 5864.8ms/step | 4.1 samples/s +Step 8840/20005 | Loss: 2.9830 | LR: 3.01e-04 | 5866.6ms/step | 4.1 samples/s +Step 8850/20005 | Loss: 2.9927 | LR: 3.00e-04 | 5869.9ms/step | 4.1 samples/s +Step 8860/20005 | Loss: 2.9707 | LR: 3.00e-04 | 5868.1ms/step | 4.1 samples/s +Step 8870/20005 | Loss: 3.0131 | LR: 3.00e-04 | 5869.0ms/step | 4.1 samples/s +Step 8880/20005 | Loss: 2.9928 | LR: 2.99e-04 | 5867.9ms/step | 4.1 samples/s +Step 8890/20005 | Loss: 2.9964 | LR: 2.99e-04 | 5870.2ms/step | 4.1 samples/s +Step 8900/20005 | Loss: 2.9714 | LR: 2.98e-04 | 5865.4ms/step | 4.1 samples/s +Step 8910/20005 | Loss: 3.0026 | LR: 2.98e-04 | 5866.9ms/step | 4.1 samples/s +Step 8920/20005 | Loss: 2.9990 | LR: 2.98e-04 | 5865.3ms/step | 4.1 samples/s +Step 8930/20005 | Loss: 2.9676 | LR: 2.97e-04 | 5866.0ms/step | 4.1 samples/s +Step 8940/20005 | Loss: 2.9855 | LR: 2.97e-04 | 5864.0ms/step | 4.1 samples/s +Step 8950/20005 | Loss: 2.9960 | LR: 2.97e-04 | 5866.3ms/step | 4.1 samples/s +Step 8960/20005 | Loss: 3.0052 | LR: 2.96e-04 | 5866.2ms/step | 4.1 samples/s +Step 8970/20005 | Loss: 3.0131 | LR: 2.96e-04 | 5867.7ms/step | 4.1 samples/s +Step 8980/20005 | Loss: 3.0088 | LR: 2.95e-04 | 5862.7ms/step | 4.1 samples/s +Step 8990/20005 | Loss: 2.9720 | LR: 2.95e-04 | 5861.5ms/step | 4.1 samples/s +Step 9000/20005 | Loss: 2.9688 | LR: 2.95e-04 | 5865.9ms/step | 4.1 samples/s +Validation at step 9000: Loss = 3.0002, Perplexity = 20.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-9000 +Step 9010/20005 | Loss: 3.0051 | LR: 2.94e-04 | 41291.0ms/step | 0.6 samples/s +Step 9020/20005 | Loss: 2.9831 | LR: 2.94e-04 | 5909.6ms/step | 4.1 samples/s +Step 9030/20005 | Loss: 2.9727 | LR: 2.93e-04 | 5873.1ms/step | 4.1 samples/s +Step 9040/20005 | Loss: 2.9759 | LR: 2.93e-04 | 5877.0ms/step | 4.1 samples/s +Step 9050/20005 | Loss: 2.9739 | LR: 2.93e-04 | 5874.8ms/step | 4.1 samples/s +Step 9060/20005 | Loss: 3.0044 | LR: 2.92e-04 | 5868.5ms/step | 4.1 samples/s +Step 9070/20005 | Loss: 2.9791 | LR: 2.92e-04 | 5869.0ms/step | 4.1 samples/s +Step 9080/20005 | Loss: 3.0096 | LR: 2.91e-04 | 5868.7ms/step | 4.1 samples/s +Step 9090/20005 | Loss: 2.9563 | LR: 2.91e-04 | 5870.9ms/step | 4.1 samples/s +Step 9100/20005 | Loss: 3.0059 | LR: 2.91e-04 | 5871.2ms/step | 4.1 samples/s +Step 9110/20005 | Loss: 3.0081 | LR: 2.90e-04 | 5875.1ms/step | 4.1 samples/s +Step 9120/20005 | Loss: 2.9921 | LR: 2.90e-04 | 5876.5ms/step | 4.1 samples/s +Step 9130/20005 | Loss: 2.9977 | LR: 2.89e-04 | 5876.8ms/step | 4.1 samples/s +Step 9140/20005 | Loss: 2.9906 | LR: 2.89e-04 | 5872.4ms/step | 4.1 samples/s +Step 9150/20005 | Loss: 2.9799 | LR: 2.89e-04 | 5871.3ms/step | 4.1 samples/s +Step 9160/20005 | Loss: 2.9954 | LR: 2.88e-04 | 5874.4ms/step | 4.1 samples/s +Step 9170/20005 | Loss: 2.9937 | LR: 2.88e-04 | 5873.1ms/step | 4.1 samples/s +Step 9180/20005 | Loss: 2.9965 | LR: 2.87e-04 | 5873.8ms/step | 4.1 samples/s +Step 9190/20005 | Loss: 2.9725 | LR: 2.87e-04 | 5873.2ms/step | 4.1 samples/s +Step 9200/20005 | Loss: 2.9793 | LR: 2.87e-04 | 5871.9ms/step | 4.1 samples/s +Step 9210/20005 | Loss: 2.9903 | LR: 2.86e-04 | 5870.8ms/step | 4.1 samples/s +Step 9220/20005 | Loss: 2.9914 | LR: 2.86e-04 | 5874.5ms/step | 4.1 samples/s +Step 9230/20005 | Loss: 3.0041 | LR: 2.86e-04 | 5877.4ms/step | 4.1 samples/s +Step 9240/20005 | Loss: 2.9725 | LR: 2.85e-04 | 5876.6ms/step | 4.1 samples/s +Step 9250/20005 | Loss: 2.9899 | LR: 2.85e-04 | 5873.9ms/step | 4.1 samples/s +Validation at step 9250: Loss = 2.9921, Perplexity = 19.93 +Step 9260/20005 | Loss: 2.9607 | LR: 2.84e-04 | 40847.6ms/step | 0.6 samples/s +Step 9270/20005 | Loss: 2.9754 | LR: 2.84e-04 | 5880.0ms/step | 4.1 samples/s +Step 9280/20005 | Loss: 2.9653 | LR: 2.84e-04 | 5879.0ms/step | 4.1 samples/s +Step 9290/20005 | Loss: 2.9909 | LR: 2.83e-04 | 5880.4ms/step | 4.1 samples/s +Step 9300/20005 | Loss: 2.9755 | LR: 2.83e-04 | 5877.0ms/step | 4.1 samples/s +Step 9310/20005 | Loss: 2.9887 | LR: 2.82e-04 | 5877.2ms/step | 4.1 samples/s +Step 9320/20005 | Loss: 3.0098 | LR: 2.82e-04 | 5876.3ms/step | 4.1 samples/s +Step 9330/20005 | Loss: 2.9954 | LR: 2.82e-04 | 5875.9ms/step | 4.1 samples/s +Step 9340/20005 | Loss: 2.9673 | LR: 2.81e-04 | 5874.9ms/step | 4.1 samples/s +Step 9350/20005 | Loss: 2.9928 | LR: 2.81e-04 | 5874.9ms/step | 4.1 samples/s +Step 9360/20005 | Loss: 2.9668 | LR: 2.80e-04 | 5875.5ms/step | 4.1 samples/s +Step 9370/20005 | Loss: 2.9698 | LR: 2.80e-04 | 5874.3ms/step | 4.1 samples/s +Step 9380/20005 | Loss: 2.9749 | LR: 2.80e-04 | 5875.7ms/step | 4.1 samples/s +Step 9390/20005 | Loss: 2.9713 | LR: 2.79e-04 | 5875.9ms/step | 4.1 samples/s +Step 9400/20005 | Loss: 2.9855 | LR: 2.79e-04 | 5878.1ms/step | 4.1 samples/s +Step 9410/20005 | Loss: 2.9873 | LR: 2.78e-04 | 5874.1ms/step | 4.1 samples/s +Step 9420/20005 | Loss: 2.9552 | LR: 2.78e-04 | 5876.0ms/step | 4.1 samples/s +Step 9430/20005 | Loss: 3.0031 | LR: 2.78e-04 | 5875.3ms/step | 4.1 samples/s +Step 9440/20005 | Loss: 3.0040 | LR: 2.77e-04 | 5874.4ms/step | 4.1 samples/s +Step 9450/20005 | Loss: 2.9816 | LR: 2.77e-04 | 5876.0ms/step | 4.1 samples/s +Step 9460/20005 | Loss: 2.9469 | LR: 2.76e-04 | 5875.0ms/step | 4.1 samples/s +Step 9470/20005 | Loss: 2.9661 | LR: 2.76e-04 | 5877.1ms/step | 4.1 samples/s +Step 9480/20005 | Loss: 2.9778 | LR: 2.76e-04 | 5876.0ms/step | 4.1 samples/s +Step 9490/20005 | Loss: 2.9720 | LR: 2.75e-04 | 5877.4ms/step | 4.1 samples/s +Step 9500/20005 | Loss: 2.9646 | LR: 2.75e-04 | 5874.0ms/step | 4.1 samples/s +Validation at step 9500: Loss = 2.9845, Perplexity = 19.78 +Step 9510/20005 | Loss: 2.9689 | LR: 2.74e-04 | 40859.1ms/step | 0.6 samples/s +Step 9520/20005 | Loss: 2.9505 | LR: 2.74e-04 | 5914.9ms/step | 4.1 samples/s +Step 9530/20005 | Loss: 2.9465 | LR: 2.74e-04 | 5875.1ms/step | 4.1 samples/s +Step 9540/20005 | Loss: 2.9960 | LR: 2.73e-04 | 5875.3ms/step | 4.1 samples/s diff --git a/epfl_final/log_files/cont_pretrain_diff_lr.log b/epfl_final/log_files/cont_pretrain_diff_lr.log new file mode 100644 index 0000000000000000000000000000000000000000..691c82c71ef87164ad6d67d1e5e3adf5bcbc373e --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_diff_lr.log @@ -0,0 +1,364 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260530_032950 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260530_032950", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0009, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 46k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000 +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Overriding checkpoint LR with config LR: 0.0009 +Resumed from checkpoint at step 46000 (epoch 0, micro_step 460000) +Resuming training from step 46000 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260530_034853-10196kbl +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense continued 46k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/10196kbl +Starting training... +Validation at step 46000: Loss = 2.5221, Perplexity = 12.46 +Step 46010/50462 | Loss: 2.8364 | LR: 1.77e-05 | 56608.8ms/step | 1.1 samples/s +Step 46020/50462 | Loss: 2.6596 | LR: 1.76e-05 | 5187.9ms/step | 11.6 samples/s +Step 46030/50462 | Loss: 2.6007 | LR: 1.75e-05 | 5205.0ms/step | 11.5 samples/s +Step 46040/50462 | Loss: 2.5616 | LR: 1.75e-05 | 5240.7ms/step | 11.4 samples/s +Step 46050/50462 | Loss: 2.5284 | LR: 1.74e-05 | 5239.8ms/step | 11.5 samples/s +Step 46060/50462 | Loss: 2.5156 | LR: 1.73e-05 | 5244.0ms/step | 11.4 samples/s +Step 46070/50462 | Loss: 2.5310 | LR: 1.72e-05 | 5219.4ms/step | 11.5 samples/s +Step 46080/50462 | Loss: 2.5231 | LR: 1.71e-05 | 5235.6ms/step | 11.5 samples/s +Step 46090/50462 | Loss: 2.4899 | LR: 1.71e-05 | 5236.2ms/step | 11.5 samples/s +Step 46100/50462 | Loss: 2.5151 | LR: 1.70e-05 | 5255.0ms/step | 11.4 samples/s +Step 46110/50462 | Loss: 2.5095 | LR: 1.69e-05 | 5254.0ms/step | 11.4 samples/s +Step 46120/50462 | Loss: 2.5042 | LR: 1.68e-05 | 5246.2ms/step | 11.4 samples/s +Step 46130/50462 | Loss: 2.5049 | LR: 1.68e-05 | 5250.9ms/step | 11.4 samples/s +Step 46140/50462 | Loss: 2.5045 | LR: 1.67e-05 | 5239.2ms/step | 11.5 samples/s +Step 46150/50462 | Loss: 2.5024 | LR: 1.66e-05 | 5238.1ms/step | 11.5 samples/s +Step 46160/50462 | Loss: 2.5194 | LR: 1.65e-05 | 5241.8ms/step | 11.4 samples/s +Step 46170/50462 | Loss: 2.5199 | LR: 1.65e-05 | 5240.5ms/step | 11.4 samples/s +Step 46180/50462 | Loss: 2.4892 | LR: 1.64e-05 | 5284.6ms/step | 11.4 samples/s +Step 46190/50462 | Loss: 2.4895 | LR: 1.63e-05 | 5285.2ms/step | 11.4 samples/s +Step 46200/50462 | Loss: 2.4967 | LR: 1.62e-05 | 5258.0ms/step | 11.4 samples/s +Step 46210/50462 | Loss: 2.5132 | LR: 1.61e-05 | 5253.4ms/step | 11.4 samples/s +Step 46220/50462 | Loss: 2.5161 | LR: 1.61e-05 | 5265.6ms/step | 11.4 samples/s +Step 46230/50462 | Loss: 2.5219 | LR: 1.60e-05 | 5297.7ms/step | 11.3 samples/s +Step 46240/50462 | Loss: 2.5304 | LR: 1.59e-05 | 5229.3ms/step | 11.5 samples/s +Step 46250/50462 | Loss: 2.5282 | LR: 1.58e-05 | 5252.8ms/step | 11.4 samples/s +Validation at step 46250: Loss = 2.5195, Perplexity = 12.42 +Step 46260/50462 | Loss: 2.5277 | LR: 1.58e-05 | 7824.6ms/step | 7.7 samples/s +Step 46270/50462 | Loss: 2.5143 | LR: 1.57e-05 | 5250.4ms/step | 11.4 samples/s +Step 46280/50462 | Loss: 2.5458 | LR: 1.56e-05 | 5252.6ms/step | 11.4 samples/s +Step 46290/50462 | Loss: 2.5324 | LR: 1.56e-05 | 5248.0ms/step | 11.4 samples/s +Step 46300/50462 | Loss: 2.5027 | LR: 1.55e-05 | 5247.2ms/step | 11.4 samples/s +Step 46310/50462 | Loss: 2.5101 | LR: 1.54e-05 | 5244.6ms/step | 11.4 samples/s +Step 46320/50462 | Loss: 2.5358 | LR: 1.53e-05 | 5243.2ms/step | 11.4 samples/s +Step 46330/50462 | Loss: 2.5152 | LR: 1.53e-05 | 5250.3ms/step | 11.4 samples/s +Step 46340/50462 | Loss: 2.5148 | LR: 1.52e-05 | 5236.8ms/step | 11.5 samples/s +Step 46350/50462 | Loss: 2.5356 | LR: 1.51e-05 | 5245.1ms/step | 11.4 samples/s +Step 46360/50462 | Loss: 2.4985 | LR: 1.50e-05 | 5249.4ms/step | 11.4 samples/s +Step 46370/50462 | Loss: 2.4840 | LR: 1.50e-05 | 5249.6ms/step | 11.4 samples/s +Step 46380/50462 | Loss: 2.5088 | LR: 1.49e-05 | 5245.3ms/step | 11.4 samples/s +Step 46390/50462 | Loss: 2.4979 | LR: 1.48e-05 | 5240.0ms/step | 11.5 samples/s +Step 46400/50462 | Loss: 2.4894 | LR: 1.47e-05 | 5238.0ms/step | 11.5 samples/s +Step 46410/50462 | Loss: 2.5021 | LR: 1.47e-05 | 5247.4ms/step | 11.4 samples/s +Step 46420/50462 | Loss: 2.5016 | LR: 1.46e-05 | 5245.3ms/step | 11.4 samples/s +Step 46430/50462 | Loss: 2.5224 | LR: 1.45e-05 | 5241.4ms/step | 11.4 samples/s +Step 46440/50462 | Loss: 2.5048 | LR: 1.45e-05 | 5243.5ms/step | 11.4 samples/s +Step 46450/50462 | Loss: 2.5308 | LR: 1.44e-05 | 5319.2ms/step | 11.3 samples/s +Step 46460/50462 | Loss: 2.5089 | LR: 1.43e-05 | 5251.3ms/step | 11.4 samples/s +Step 46470/50462 | Loss: 2.5002 | LR: 1.42e-05 | 5250.7ms/step | 11.4 samples/s +Step 46480/50462 | Loss: 2.4954 | LR: 1.42e-05 | 5248.2ms/step | 11.4 samples/s +Step 46490/50462 | Loss: 2.4914 | LR: 1.41e-05 | 5243.4ms/step | 11.4 samples/s +Step 46500/50462 | Loss: 2.5083 | LR: 1.40e-05 | 5276.5ms/step | 11.4 samples/s +Validation at step 46500: Loss = 2.5178, Perplexity = 12.40 +Step 46510/50462 | Loss: 2.5026 | LR: 1.40e-05 | 7814.7ms/step | 7.7 samples/s +Step 46520/50462 | Loss: 2.5260 | LR: 1.39e-05 | 5246.6ms/step | 11.4 samples/s +Step 46530/50462 | Loss: 2.5060 | LR: 1.38e-05 | 5254.3ms/step | 11.4 samples/s +Step 46540/50462 | Loss: 2.5187 | LR: 1.38e-05 | 5255.4ms/step | 11.4 samples/s +Step 46550/50462 | Loss: 2.4805 | LR: 1.37e-05 | 5237.3ms/step | 11.5 samples/s +Step 46560/50462 | Loss: 2.4973 | LR: 1.36e-05 | 5247.2ms/step | 11.4 samples/s +Step 46570/50462 | Loss: 2.5079 | LR: 1.35e-05 | 5241.1ms/step | 11.4 samples/s +Step 46580/50462 | Loss: 2.4997 | LR: 1.35e-05 | 5234.1ms/step | 11.5 samples/s +Step 46590/50462 | Loss: 2.4922 | LR: 1.34e-05 | 5245.1ms/step | 11.4 samples/s +Step 46600/50462 | Loss: 2.5074 | LR: 1.33e-05 | 5249.5ms/step | 11.4 samples/s +Step 46610/50462 | Loss: 2.5054 | LR: 1.33e-05 | 5244.4ms/step | 11.4 samples/s +Step 46620/50462 | Loss: 2.4882 | LR: 1.32e-05 | 5242.8ms/step | 11.4 samples/s +Step 46630/50462 | Loss: 2.5024 | LR: 1.31e-05 | 5253.4ms/step | 11.4 samples/s +Step 46640/50462 | Loss: 2.5145 | LR: 1.31e-05 | 5244.6ms/step | 11.4 samples/s +Step 46650/50462 | Loss: 2.5065 | LR: 1.30e-05 | 5250.5ms/step | 11.4 samples/s +Step 46660/50462 | Loss: 2.4965 | LR: 1.29e-05 | 5254.2ms/step | 11.4 samples/s +Step 46670/50462 | Loss: 2.5113 | LR: 1.29e-05 | 5250.0ms/step | 11.4 samples/s +Step 46680/50462 | Loss: 2.5199 | LR: 1.28e-05 | 5246.2ms/step | 11.4 samples/s +Step 46690/50462 | Loss: 2.5366 | LR: 1.27e-05 | 5248.3ms/step | 11.4 samples/s +Step 46700/50462 | Loss: 2.4968 | LR: 1.27e-05 | 5249.6ms/step | 11.4 samples/s +Step 46710/50462 | Loss: 2.5298 | LR: 1.26e-05 | 5273.4ms/step | 11.4 samples/s +Step 46720/50462 | Loss: 2.5134 | LR: 1.25e-05 | 5279.3ms/step | 11.4 samples/s +Step 46730/50462 | Loss: 2.4932 | LR: 1.25e-05 | 5237.0ms/step | 11.5 samples/s +Step 46740/50462 | Loss: 2.5198 | LR: 1.24e-05 | 5227.1ms/step | 11.5 samples/s +Step 46750/50462 | Loss: 2.5026 | LR: 1.23e-05 | 5243.4ms/step | 11.4 samples/s +Validation at step 46750: Loss = 2.5172, Perplexity = 12.39 +Step 46760/50462 | Loss: 2.5171 | LR: 1.23e-05 | 7815.1ms/step | 7.7 samples/s +Step 46770/50462 | Loss: 2.5067 | LR: 1.22e-05 | 5292.7ms/step | 11.3 samples/s +Step 46780/50462 | Loss: 2.5321 | LR: 1.21e-05 | 5237.2ms/step | 11.5 samples/s +Step 46790/50462 | Loss: 2.4894 | LR: 1.21e-05 | 5240.5ms/step | 11.4 samples/s +Step 46800/50462 | Loss: 2.5274 | LR: 1.20e-05 | 5245.3ms/step | 11.4 samples/s +Step 46810/50462 | Loss: 2.5233 | LR: 1.19e-05 | 5250.8ms/step | 11.4 samples/s +Step 46820/50462 | Loss: 2.4867 | LR: 1.19e-05 | 5246.2ms/step | 11.4 samples/s +Step 46830/50462 | Loss: 2.4849 | LR: 1.18e-05 | 5246.5ms/step | 11.4 samples/s +Step 46840/50462 | Loss: 2.5104 | LR: 1.17e-05 | 5244.4ms/step | 11.4 samples/s +Step 46850/50462 | Loss: 2.5064 | LR: 1.17e-05 | 5237.2ms/step | 11.5 samples/s +Step 46860/50462 | Loss: 2.5059 | LR: 1.16e-05 | 5243.0ms/step | 11.4 samples/s +Step 46870/50462 | Loss: 2.4642 | LR: 1.15e-05 | 5248.1ms/step | 11.4 samples/s +Step 46880/50462 | Loss: 2.4996 | LR: 1.15e-05 | 5249.1ms/step | 11.4 samples/s +Step 46890/50462 | Loss: 2.5057 | LR: 1.14e-05 | 5249.4ms/step | 11.4 samples/s +Step 46900/50462 | Loss: 2.4978 | LR: 1.14e-05 | 5254.3ms/step | 11.4 samples/s +Step 46910/50462 | Loss: 2.4909 | LR: 1.13e-05 | 5249.4ms/step | 11.4 samples/s +Step 46920/50462 | Loss: 2.5288 | LR: 1.12e-05 | 5249.4ms/step | 11.4 samples/s +Step 46930/50462 | Loss: 2.5051 | LR: 1.12e-05 | 5250.7ms/step | 11.4 samples/s +Step 46940/50462 | Loss: 2.5071 | LR: 1.11e-05 | 5240.0ms/step | 11.5 samples/s +Step 46950/50462 | Loss: 2.4901 | LR: 1.10e-05 | 5248.1ms/step | 11.4 samples/s +Step 46960/50462 | Loss: 2.5134 | LR: 1.10e-05 | 5287.1ms/step | 11.3 samples/s +Step 46970/50462 | Loss: 2.5171 | LR: 1.09e-05 | 5229.2ms/step | 11.5 samples/s +Step 46980/50462 | Loss: 2.5163 | LR: 1.09e-05 | 5229.3ms/step | 11.5 samples/s +Step 46990/50462 | Loss: 2.5371 | LR: 1.08e-05 | 5270.8ms/step | 11.4 samples/s +Step 47000/50462 | Loss: 2.5167 | LR: 1.07e-05 | 5239.0ms/step | 11.5 samples/s +Validation at step 47000: Loss = 2.5168, Perplexity = 12.39 +Saved checkpoint to checkpoints/run_20260530_032950/checkpoint-47000 +Step 47010/50462 | Loss: 2.5287 | LR: 1.07e-05 | 8566.5ms/step | 7.0 samples/s +Step 47020/50462 | Loss: 2.5053 | LR: 1.06e-05 | 5215.9ms/step | 11.5 samples/s +Step 47030/50462 | Loss: 2.5252 | LR: 1.05e-05 | 5225.3ms/step | 11.5 samples/s +Step 47040/50462 | Loss: 2.4702 | LR: 1.05e-05 | 5266.7ms/step | 11.4 samples/s +Step 47050/50462 | Loss: 2.5070 | LR: 1.04e-05 | 5235.6ms/step | 11.5 samples/s +Step 47060/50462 | Loss: 2.5053 | LR: 1.04e-05 | 5236.8ms/step | 11.5 samples/s +Step 47070/50462 | Loss: 2.4982 | LR: 1.03e-05 | 5239.0ms/step | 11.5 samples/s +Step 47080/50462 | Loss: 2.5060 | LR: 1.02e-05 | 5227.9ms/step | 11.5 samples/s +Step 47090/50462 | Loss: 2.5009 | LR: 1.02e-05 | 5233.3ms/step | 11.5 samples/s +Step 47100/50462 | Loss: 2.5048 | LR: 1.01e-05 | 5237.2ms/step | 11.5 samples/s +Step 47110/50462 | Loss: 2.5085 | LR: 1.01e-05 | 5237.8ms/step | 11.5 samples/s +Step 47120/50462 | Loss: 2.4590 | LR: 1.00e-05 | 5242.8ms/step | 11.4 samples/s +Step 47130/50462 | Loss: 2.5058 | LR: 9.94e-06 | 5245.2ms/step | 11.4 samples/s +Step 47140/50462 | Loss: 2.5182 | LR: 9.88e-06 | 5239.9ms/step | 11.5 samples/s +Step 47150/50462 | Loss: 2.4744 | LR: 9.82e-06 | 5240.0ms/step | 11.5 samples/s +Step 47160/50462 | Loss: 2.5162 | LR: 9.76e-06 | 5237.6ms/step | 11.5 samples/s +Step 47170/50462 | Loss: 2.4992 | LR: 9.70e-06 | 5234.2ms/step | 11.5 samples/s +Step 47180/50462 | Loss: 2.5086 | LR: 9.64e-06 | 5250.4ms/step | 11.4 samples/s +Step 47190/50462 | Loss: 2.5033 | LR: 9.59e-06 | 5250.6ms/step | 11.4 samples/s +Step 47200/50462 | Loss: 2.5014 | LR: 9.53e-06 | 5235.9ms/step | 11.5 samples/s +Step 47210/50462 | Loss: 2.4751 | LR: 9.47e-06 | 5239.2ms/step | 11.5 samples/s +Step 47220/50462 | Loss: 2.5158 | LR: 9.41e-06 | 5266.9ms/step | 11.4 samples/s +Step 47230/50462 | Loss: 2.5139 | LR: 9.35e-06 | 5243.6ms/step | 11.4 samples/s +Step 47240/50462 | Loss: 2.4825 | LR: 9.30e-06 | 5242.1ms/step | 11.4 samples/s +Step 47250/50462 | Loss: 2.5128 | LR: 9.24e-06 | 5241.4ms/step | 11.4 samples/s +Validation at step 47250: Loss = 2.5166, Perplexity = 12.39 +Step 47260/50462 | Loss: 2.5073 | LR: 9.18e-06 | 7860.4ms/step | 7.6 samples/s +Step 47270/50462 | Loss: 2.5191 | LR: 9.12e-06 | 5244.8ms/step | 11.4 samples/s +Step 47280/50462 | Loss: 2.4928 | LR: 9.07e-06 | 5242.3ms/step | 11.4 samples/s +Step 47290/50462 | Loss: 2.5207 | LR: 9.01e-06 | 5247.2ms/step | 11.4 samples/s +Step 47300/50462 | Loss: 2.5191 | LR: 8.95e-06 | 5241.7ms/step | 11.4 samples/s +Step 47310/50462 | Loss: 2.4810 | LR: 8.90e-06 | 5283.8ms/step | 11.4 samples/s +Step 47320/50462 | Loss: 2.4981 | LR: 8.84e-06 | 5244.8ms/step | 11.4 samples/s +Step 47330/50462 | Loss: 2.5013 | LR: 8.79e-06 | 5250.6ms/step | 11.4 samples/s +Step 47340/50462 | Loss: 2.4662 | LR: 8.73e-06 | 5257.9ms/step | 11.4 samples/s +Step 47350/50462 | Loss: 2.5137 | LR: 8.67e-06 | 5252.8ms/step | 11.4 samples/s +Step 47360/50462 | Loss: 2.4842 | LR: 8.62e-06 | 5239.6ms/step | 11.5 samples/s +Step 47370/50462 | Loss: 2.5223 | LR: 8.56e-06 | 5234.3ms/step | 11.5 samples/s +Step 47380/50462 | Loss: 2.5149 | LR: 8.51e-06 | 5236.9ms/step | 11.5 samples/s +Step 47390/50462 | Loss: 2.4977 | LR: 8.45e-06 | 5229.2ms/step | 11.5 samples/s +Step 47400/50462 | Loss: 2.5006 | LR: 8.40e-06 | 5240.3ms/step | 11.4 samples/s +Step 47410/50462 | Loss: 2.5119 | LR: 8.34e-06 | 5236.1ms/step | 11.5 samples/s +Step 47420/50462 | Loss: 2.4639 | LR: 8.29e-06 | 5236.2ms/step | 11.5 samples/s +Step 47430/50462 | Loss: 2.4916 | LR: 8.24e-06 | 5240.8ms/step | 11.4 samples/s +Step 47440/50462 | Loss: 2.5003 | LR: 8.18e-06 | 5233.6ms/step | 11.5 samples/s +Step 47450/50462 | Loss: 2.4752 | LR: 8.13e-06 | 5230.7ms/step | 11.5 samples/s +Step 47460/50462 | Loss: 2.5199 | LR: 8.07e-06 | 5228.0ms/step | 11.5 samples/s +Step 47470/50462 | Loss: 2.5176 | LR: 8.02e-06 | 5239.7ms/step | 11.5 samples/s +Step 47480/50462 | Loss: 2.5164 | LR: 7.97e-06 | 5273.9ms/step | 11.4 samples/s +Step 47490/50462 | Loss: 2.5091 | LR: 7.91e-06 | 5210.5ms/step | 11.5 samples/s +Step 47500/50462 | Loss: 2.5005 | LR: 7.86e-06 | 5232.8ms/step | 11.5 samples/s +Validation at step 47500: Loss = 2.5165, Perplexity = 12.39 +Step 47510/50462 | Loss: 2.4714 | LR: 7.81e-06 | 7812.7ms/step | 7.7 samples/s +Step 47520/50462 | Loss: 2.5191 | LR: 7.76e-06 | 5273.9ms/step | 11.4 samples/s +Step 47530/50462 | Loss: 2.5146 | LR: 7.70e-06 | 5236.1ms/step | 11.5 samples/s +Step 47540/50462 | Loss: 2.4830 | LR: 7.65e-06 | 5239.3ms/step | 11.5 samples/s +Step 47550/50462 | Loss: 2.5124 | LR: 7.60e-06 | 5247.5ms/step | 11.4 samples/s +Step 47560/50462 | Loss: 2.4978 | LR: 7.55e-06 | 5230.9ms/step | 11.5 samples/s +Step 47570/50462 | Loss: 2.5079 | LR: 7.49e-06 | 5235.6ms/step | 11.5 samples/s +Step 47580/50462 | Loss: 2.5432 | LR: 7.44e-06 | 5269.0ms/step | 11.4 samples/s +Step 47590/50462 | Loss: 2.5140 | LR: 7.39e-06 | 5233.3ms/step | 11.5 samples/s +Step 47600/50462 | Loss: 2.5048 | LR: 7.34e-06 | 5242.6ms/step | 11.4 samples/s +Step 47610/50462 | Loss: 2.4847 | LR: 7.29e-06 | 5245.5ms/step | 11.4 samples/s +Step 47620/50462 | Loss: 2.4984 | LR: 7.24e-06 | 5246.2ms/step | 11.4 samples/s +Step 47630/50462 | Loss: 2.5256 | LR: 7.19e-06 | 5236.9ms/step | 11.5 samples/s +Step 47640/50462 | Loss: 2.4806 | LR: 7.14e-06 | 5242.7ms/step | 11.4 samples/s +Step 47650/50462 | Loss: 2.5000 | LR: 7.09e-06 | 5250.1ms/step | 11.4 samples/s +Step 47660/50462 | Loss: 2.4990 | LR: 7.04e-06 | 5236.7ms/step | 11.5 samples/s +Step 47670/50462 | Loss: 2.4911 | LR: 6.99e-06 | 5233.8ms/step | 11.5 samples/s +Step 47680/50462 | Loss: 2.5093 | LR: 6.94e-06 | 5241.7ms/step | 11.4 samples/s +Step 47690/50462 | Loss: 2.5050 | LR: 6.89e-06 | 5245.5ms/step | 11.4 samples/s +Step 47700/50462 | Loss: 2.5131 | LR: 6.84e-06 | 5251.1ms/step | 11.4 samples/s +Step 47710/50462 | Loss: 2.4791 | LR: 6.79e-06 | 5245.5ms/step | 11.4 samples/s +9ms/step | 11.4 samples/s +Step 48030/50462 | Loss: 2.4860 | LR: 2.95e-06 | 5289.5ms/step | 11.3 samples/s +Step 48040/50462 | Loss: 2.4994 | LR: 2.92e-06 | 5251.1ms/step | 11.4 samples/s +Step 48050/50462 | Loss: 2.4927 | LR: 2.90e-06 | 5245.4ms/step | 11.4 samples/s +Step 48060/50462 | Loss: 2.5093 | LR: 2.87e-06 | 5250.2ms/step | 11.4 samples/s +Step 48070/50462 | Loss: 2.5118 | LR: 2.85e-06 | 5249.8ms/step | 11.4 samples/s diff --git a/epfl_final/log_files/cont_pretrain_lr.log b/epfl_final/log_files/cont_pretrain_lr.log new file mode 100644 index 0000000000000000000000000000000000000000..77d4e82c1518d6cd389cfabea237a13b5d217544 --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_lr.log @@ -0,0 +1,264 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260530_021500 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260530_021500", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-44000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 44k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-44000 +Checkpoint activations: False +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-44000... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Resumed from checkpoint at step 44000 (epoch 0, micro_step 440000) +Resuming training from step 44000 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 408mcc4e +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260530_023351-408mcc4e +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense continued 44k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/408mcc4e +Starting training... +Validation at step 44000: Loss = 2.5224, Perplexity = 12.46 +Step 44010/50462 | Loss: 2.5521 | LR: 2.05e-05 | 59535.6ms/step | 1.0 samples/s +Step 44020/50462 | Loss: 2.5189 | LR: 2.04e-05 | 5212.1ms/step | 11.5 samples/s +Step 44030/50462 | Loss: 2.5073 | LR: 2.04e-05 | 5229.4ms/step | 11.5 samples/s +Step 44040/50462 | Loss: 2.5005 | LR: 2.03e-05 | 5226.5ms/step | 11.5 samples/s +Step 44050/50462 | Loss: 2.5321 | LR: 2.02e-05 | 5230.2ms/step | 11.5 samples/s +Step 44060/50462 | Loss: 2.5302 | LR: 2.02e-05 | 5237.1ms/step | 11.5 samples/s +Step 44070/50462 | Loss: 2.5293 | LR: 2.01e-05 | 5242.2ms/step | 11.4 samples/s +Step 44080/50462 | Loss: 2.5117 | LR: 2.01e-05 | 5244.5ms/step | 11.4 samples/s +Step 44090/50462 | Loss: 2.4805 | LR: 2.00e-05 | 5289.2ms/step | 11.3 samples/s +Step 44100/50462 | Loss: 2.4990 | LR: 1.99e-05 | 5233.3ms/step | 11.5 samples/s +Step 44110/50462 | Loss: 2.5007 | LR: 1.99e-05 | 5237.0ms/step | 11.5 samples/s +Step 44120/50462 | Loss: 2.5171 | LR: 1.98e-05 | 5239.0ms/step | 11.5 samples/s +Step 44130/50462 | Loss: 2.5227 | LR: 1.97e-05 | 5247.7ms/step | 11.4 samples/s +Step 44140/50462 | Loss: 2.4908 | LR: 1.97e-05 | 5256.8ms/step | 11.4 samples/s +Step 44150/50462 | Loss: 2.4862 | LR: 1.96e-05 | 5239.4ms/step | 11.5 samples/s +Step 44160/50462 | Loss: 2.5162 | LR: 1.96e-05 | 5235.0ms/step | 11.5 samples/s +Step 44170/50462 | Loss: 2.5198 | LR: 1.95e-05 | 5236.4ms/step | 11.5 samples/s +Step 44180/50462 | Loss: 2.4864 | LR: 1.94e-05 | 5249.8ms/step | 11.4 samples/s +Step 44190/50462 | Loss: 2.4929 | LR: 1.94e-05 | 5251.5ms/step | 11.4 samples/s +Step 44200/50462 | Loss: 2.5140 | LR: 1.93e-05 | 5252.4ms/step | 11.4 samples/s +Step 44210/50462 | Loss: 2.5021 | LR: 1.93e-05 | 5233.8ms/step | 11.5 samples/s +Step 44220/50462 | Loss: 2.5080 | LR: 1.92e-05 | 5226.6ms/step | 11.5 samples/s +Step 44230/50462 | Loss: 2.5198 | LR: 1.91e-05 | 5245.4ms/step | 11.4 samples/s +Step 44240/50462 | Loss: 2.5071 | LR: 1.91e-05 | 5253.7ms/step | 11.4 samples/s +Step 44250/50462 | Loss: 2.5205 | LR: 1.90e-05 | 5244.2ms/step | 11.4 samples/s +Validation at step 44250: Loss = 2.5158, Perplexity = 12.38 +Step 44260/50462 | Loss: 2.5148 | LR: 1.90e-05 | 7809.6ms/step | 7.7 samples/s +Step 44270/50462 | Loss: 2.4856 | LR: 1.89e-05 | 5244.5ms/step | 11.4 samples/s +Step 44280/50462 | Loss: 2.5090 | LR: 1.88e-05 | 5285.8ms/step | 11.4 samples/s +Step 44290/50462 | Loss: 2.5052 | LR: 1.88e-05 | 5234.3ms/step | 11.5 samples/s +Step 44300/50462 | Loss: 2.4564 | LR: 1.87e-05 | 5231.1ms/step | 11.5 samples/s +Step 44310/50462 | Loss: 2.4863 | LR: 1.87e-05 | 5198.5ms/step | 11.5 samples/s +Step 44320/50462 | Loss: 2.5187 | LR: 1.86e-05 | 5228.2ms/step | 11.5 samples/s +Step 44330/50462 | Loss: 2.5091 | LR: 1.85e-05 | 5238.5ms/step | 11.5 samples/s +Step 44340/50462 | Loss: 2.5053 | LR: 1.85e-05 | 5242.7ms/step | 11.4 samples/s +Step 44350/50462 | Loss: 2.5100 | LR: 1.84e-05 | 5237.8ms/step | 11.5 samples/s +Step 44360/50462 | Loss: 2.4832 | LR: 1.84e-05 | 5225.6ms/step | 11.5 samples/s +Step 44370/50462 | Loss: 2.5084 | LR: 1.83e-05 | 5218.8ms/step | 11.5 samples/s +Step 44380/50462 | Loss: 2.4995 | LR: 1.82e-05 | 5294.0ms/step | 11.3 samples/s +Step 44390/50462 | Loss: 2.5195 | LR: 1.82e-05 | 5229.2ms/step | 11.5 samples/s +Step 44400/50462 | Loss: 2.4860 | LR: 1.81e-05 | 5236.8ms/step | 11.5 samples/s +Step 44410/50462 | Loss: 2.4953 | LR: 1.81e-05 | 5246.0ms/step | 11.4 samples/s +Step 44420/50462 | Loss: 2.5053 | LR: 1.80e-05 | 5232.4ms/step | 11.5 samples/s +Step 44430/50462 | Loss: 2.5128 | LR: 1.79e-05 | 5233.2ms/step | 11.5 samples/s +Step 44440/50462 | Loss: 2.5019 | LR: 1.79e-05 | 5246.3ms/step | 11.4 samples/s +Step 44450/50462 | Loss: 2.4906 | LR: 1.78e-05 | 5247.1ms/step | 11.4 samples/s +Step 44460/50462 | Loss: 2.5063 | LR: 1.78e-05 | 5244.6ms/step | 11.4 samples/s +Step 44470/50462 | Loss: 2.5189 | LR: 1.77e-05 | 5233.5ms/step | 11.5 samples/s +Step 44480/50462 | Loss: 2.5330 | LR: 1.77e-05 | 5201.1ms/step | 11.5 samples/s +Step 44490/50462 | Loss: 2.5285 | LR: 1.76e-05 | 5230.3ms/step | 11.5 samples/s +Step 44500/50462 | Loss: 2.4912 | LR: 1.75e-05 | 5246.2ms/step | 11.4 samples/s +Validation at step 44500: Loss = 2.5150, Perplexity = 12.37 +Step 44510/50462 | Loss: 2.4808 | LR: 1.75e-05 | 7822.5ms/step | 7.7 samples/s +Step 44520/50462 | Loss: 2.4795 | LR: 1.74e-05 | 5244.6ms/step | 11.4 samples/s +W0530 03:29:22.958000 2737 torch/distributed/elastic/agent/server/api.py:753] Received Signals.SIGTERM death signal, shutting down workers +W0530 03:29:22.959000 2737 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 2901 closing signal SIGTERM +W0530 03:29:22.960000 2737 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 2902 closing signal SIGTERM +W0530 03:29:22.962000 2737 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 2903 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/usr/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1020, in + main() + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 367, in wrapper + return f(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1016, in main + run(args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1007, in run + elastic_launch( + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 191, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 362, in launch_agent + result = agent.run() + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 745, in run + result = self._invoke_run(role) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 923, in _invoke_run + time.sleep(monitor_interval) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 2737 got signal: 15 diff --git a/epfl_final/log_files/cont_pretrain_mixed.log b/epfl_final/log_files/cont_pretrain_mixed.log new file mode 100644 index 0000000000000000000000000000000000000000..4d7d6cc2f9fc65d87d6039614767237620d7573b --- /dev/null +++ b/epfl_final/log_files/cont_pretrain_mixed.log @@ -0,0 +1,3572 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260530_194538 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260530_194538", + "resume_from_checkpoint": "checkpoints/run_20260527_053212/checkpoint-19000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "sparse 1.2b", + "wandb_tags": [ + "qwen3", + "1.2b", + "sparse" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache", + "dense_mode": false, + "dense_step_intervals": [ + [ + 19750, + 20000 + ], + [ + 24750, + 25000 + ], + [ + 29750, + 30000 + ], + [ + 34750, + 35000 + ], + [ + 39750, + 40000 + ], + [ + 44750, + 45000 + ], + [ + 49750, + 50000 + ] + ] +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: checkpoints/run_20260527_053212/checkpoint-19000 +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... + + +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from checkpoints/run_20260527_053212/checkpoint-19000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 19000 (epoch 0, micro_step 190000) +Resuming training from step 19000 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ofounlqj +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260530_194904-ofounlqj +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run sparse 1.2b +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ofounlqj +Starting training... +Validation at step 19000: Loss = 2.6587, Perplexity = 14.28 +Switching attention mode for step 19001: sparse_stochastic (configured sparse attention) +Step 19010/50462 | Loss: 2.6506 | LR: 3.51e-04 | 46887.5ms/step | 1.3 samples/s +Step 19020/50462 | Loss: 2.6682 | LR: 3.51e-04 | 3081.3ms/step | 19.5 samples/s +Step 19030/50462 | Loss: 2.6665 | LR: 3.51e-04 | 3093.9ms/step | 19.4 samples/s +Step 19040/50462 | Loss: 2.6451 | LR: 3.51e-04 | 3103.5ms/step | 19.3 samples/s +Step 19050/50462 | Loss: 2.6491 | LR: 3.51e-04 | 3110.4ms/step | 19.3 samples/s +Step 19060/50462 | Loss: 2.6387 | LR: 3.50e-04 | 3115.2ms/step | 19.3 samples/s +Step 19070/50462 | Loss: 2.6315 | LR: 3.50e-04 | 3116.6ms/step | 19.3 samples/s +Step 19080/50462 | Loss: 2.6309 | LR: 3.50e-04 | 3115.1ms/step | 19.3 samples/s +Step 19090/50462 | Loss: 2.6450 | LR: 3.50e-04 | 3116.4ms/step | 19.3 samples/s +Step 19100/50462 | Loss: 2.6466 | LR: 3.50e-04 | 3115.6ms/step | 19.3 samples/s +Step 19110/50462 | Loss: 2.6276 | LR: 3.50e-04 | 3114.9ms/step | 19.3 samples/s +Step 19120/50462 | Loss: 2.6526 | LR: 3.50e-04 | 3116.0ms/step | 19.3 samples/s +Step 19130/50462 | Loss: 2.6860 | LR: 3.49e-04 | 3118.0ms/step | 19.2 samples/s +Step 19140/50462 | Loss: 2.6434 | LR: 3.49e-04 | 3117.7ms/step | 19.2 samples/s +Step 19150/50462 | Loss: 2.6581 | LR: 3.49e-04 | 3114.6ms/step | 19.3 samples/s +Step 19160/50462 | Loss: 2.6515 | LR: 3.49e-04 | 3115.0ms/step | 19.3 samples/s +Step 19170/50462 | Loss: 2.6350 | LR: 3.49e-04 | 3118.6ms/step | 19.2 samples/s +Step 19180/50462 | Loss: 2.6485 | LR: 3.49e-04 | 3114.8ms/step | 19.3 samples/s +Step 19190/50462 | Loss: 2.6267 | LR: 3.49e-04 | 3117.4ms/step | 19.2 samples/s +Step 19200/50462 | Loss: 2.6397 | LR: 3.48e-04 | 3111.6ms/step | 19.3 samples/s +Step 19210/50462 | Loss: 2.6345 | LR: 3.48e-04 | 3116.0ms/step | 19.3 samples/s +Step 19220/50462 | Loss: 2.6495 | LR: 3.48e-04 | 3117.4ms/step | 19.2 samples/s +Step 19230/50462 | Loss: 2.6367 | LR: 3.48e-04 | 3119.2ms/step | 19.2 samples/s +Step 19240/50462 | Loss: 2.6517 | LR: 3.48e-04 | 3122.5ms/step | 19.2 samples/s +Step 19250/50462 | Loss: 2.6384 | LR: 3.48e-04 | 3121.5ms/step | 19.2 samples/s +Validation at step 19250: Loss = 2.6574, Perplexity = 14.26 +Step 19260/50462 | Loss: 2.6254 | LR: 3.48e-04 | 4794.1ms/step | 12.5 samples/s +Step 19270/50462 | Loss: 2.6355 | LR: 3.47e-04 | 3121.5ms/step | 19.2 samples/s +Step 19280/50462 | Loss: 2.6372 | LR: 3.47e-04 | 3122.8ms/step | 19.2 samples/s +Step 19290/50462 | Loss: 2.6674 | LR: 3.47e-04 | 3123.7ms/step | 19.2 samples/s +Step 19300/50462 | Loss: 2.6342 | LR: 3.47e-04 | 3122.7ms/step | 19.2 samples/s +Step 19310/50462 | Loss: 2.6304 | LR: 3.47e-04 | 3123.5ms/step | 19.2 samples/s +Step 19320/50462 | Loss: 2.6045 | LR: 3.47e-04 | 3122.1ms/step | 19.2 samples/s +Step 19330/50462 | Loss: 2.6357 | LR: 3.47e-04 | 3123.6ms/step | 19.2 samples/s +Step 19340/50462 | Loss: 2.6224 | LR: 3.46e-04 | 3124.5ms/step | 19.2 samples/s +Step 19350/50462 | Loss: 2.6664 | LR: 3.46e-04 | 3123.6ms/step | 19.2 samples/s +Step 19360/50462 | Loss: 2.6587 | LR: 3.46e-04 | 3122.0ms/step | 19.2 samples/s +Step 19370/50462 | Loss: 2.6253 | LR: 3.46e-04 | 3121.7ms/step | 19.2 samples/s +Step 19380/50462 | Loss: 2.6490 | LR: 3.46e-04 | 3125.5ms/step | 19.2 samples/s +Step 19390/50462 | Loss: 2.6509 | LR: 3.46e-04 | 3120.2ms/step | 19.2 samples/s +Step 19400/50462 | Loss: 2.6261 | LR: 3.46e-04 | 3117.9ms/step | 19.2 samples/s +Step 19410/50462 | Loss: 2.6354 | LR: 3.45e-04 | 3120.6ms/step | 19.2 samples/s +Step 19420/50462 | Loss: 2.6206 | LR: 3.45e-04 | 3122.4ms/step | 19.2 samples/s +Step 19430/50462 | Loss: 2.6735 | LR: 3.45e-04 | 3121.8ms/step | 19.2 samples/s +Step 19440/50462 | Loss: 2.6413 | LR: 3.45e-04 | 3118.4ms/step | 19.2 samples/s +Step 19450/50462 | Loss: 2.6459 | LR: 3.45e-04 | 3123.4ms/step | 19.2 samples/s +Step 19460/50462 | Loss: 2.6378 | LR: 3.45e-04 | 3123.3ms/step | 19.2 samples/s +Step 19470/50462 | Loss: 2.6709 | LR: 3.45e-04 | 3121.1ms/step | 19.2 samples/s +Step 19480/50462 | Loss: 2.6304 | LR: 3.44e-04 | 3112.6ms/step | 19.3 samples/s +Step 19490/50462 | Loss: 2.6622 | LR: 3.44e-04 | 3115.8ms/step | 19.3 samples/s +Step 19500/50462 | Loss: 2.6468 | LR: 3.44e-04 | 3115.3ms/step | 19.3 samples/s +Validation at step 19500: Loss = 2.6552, Perplexity = 14.23 +Step 19510/50462 | Loss: 2.6467 | LR: 3.44e-04 | 4787.9ms/step | 12.5 samples/s +Step 19520/50462 | Loss: 2.6178 | LR: 3.44e-04 | 3113.1ms/step | 19.3 samples/s +Step 19530/50462 | Loss: 2.6450 | LR: 3.44e-04 | 3113.7ms/step | 19.3 samples/s +Step 19540/50462 | Loss: 2.6543 | LR: 3.44e-04 | 3115.4ms/step | 19.3 samples/s +Step 19550/50462 | Loss: 2.6475 | LR: 3.43e-04 | 3119.4ms/step | 19.2 samples/s +Step 19560/50462 | Loss: 2.6528 | LR: 3.43e-04 | 3118.7ms/step | 19.2 samples/s +Step 19570/50462 | Loss: 2.6009 | LR: 3.43e-04 | 3122.1ms/step | 19.2 samples/s +Step 19580/50462 | Loss: 2.6244 | LR: 3.43e-04 | 3122.2ms/step | 19.2 samples/s +Step 19590/50462 | Loss: 2.6557 | LR: 3.43e-04 | 3121.2ms/step | 19.2 samples/s +Step 19600/50462 | Loss: 2.6161 | LR: 3.43e-04 | 3121.7ms/step | 19.2 samples/s +Step 19610/50462 | Loss: 2.6700 | LR: 3.42e-04 | 3121.2ms/step | 19.2 samples/s +Step 19620/50462 | Loss: 2.6195 | LR: 3.42e-04 | 3593.3ms/step | 16.7 samples/s +Step 19630/50462 | Loss: 2.6214 | LR: 3.42e-04 | 3106.6ms/step | 19.3 samples/s +Step 19640/50462 | Loss: 2.6601 | LR: 3.42e-04 | 3113.2ms/step | 19.3 samples/s +Step 19650/50462 | Loss: 2.6283 | LR: 3.42e-04 | 3607.1ms/step | 16.6 samples/s +Step 19660/50462 | Loss: 2.6225 | LR: 3.42e-04 | 3097.9ms/step | 19.4 samples/s +Step 19670/50462 | Loss: 2.6658 | LR: 3.42e-04 | 3109.4ms/step | 19.3 samples/s +Step 19680/50462 | Loss: 2.6429 | LR: 3.41e-04 | 3117.9ms/step | 19.2 samples/s +Step 19690/50462 | Loss: 2.6501 | LR: 3.41e-04 | 3602.3ms/step | 16.7 samples/s +Step 19700/50462 | Loss: 2.6465 | LR: 3.41e-04 | 3110.5ms/step | 19.3 samples/s +Step 19710/50462 | Loss: 2.6505 | LR: 3.41e-04 | 3115.2ms/step | 19.3 samples/s +Step 19720/50462 | Loss: 2.6245 | LR: 3.41e-04 | 3118.0ms/step | 19.2 samples/s +Step 19730/50462 | Loss: 2.6288 | LR: 3.41e-04 | 3121.4ms/step | 19.2 samples/s +Step 19740/50462 | Loss: 2.6800 | LR: 3.41e-04 | 3122.4ms/step | 19.2 samples/s +Switching attention mode for step 19750: dense (dense step interval) +Step 19750/50462 | Loss: 2.6527 | LR: 3.40e-04 | 5154.5ms/step | 11.6 samples/s +Validation at step 19750: Loss = 2.6320, Perplexity = 13.90 +Step 19760/50462 | Loss: 2.6217 | LR: 3.40e-04 | 8932.4ms/step | 6.7 samples/s +Step 19770/50462 | Loss: 2.6479 | LR: 3.40e-04 | 5246.1ms/step | 11.4 samples/s +Step 19780/50462 | Loss: 2.6399 | LR: 3.40e-04 | 5255.0ms/step | 11.4 samples/s +Step 19790/50462 | Loss: 2.5700 | LR: 3.40e-04 | 5259.1ms/step | 11.4 samples/s +Step 19800/50462 | Loss: 2.6288 | LR: 3.40e-04 | 5252.3ms/step | 11.4 samples/s +Step 19810/50462 | Loss: 2.6127 | LR: 3.40e-04 | 5258.0ms/step | 11.4 samples/s +Step 19820/50462 | Loss: 2.6182 | LR: 3.39e-04 | 5255.9ms/step | 11.4 samples/s +Step 19830/50462 | Loss: 2.6318 | LR: 3.39e-04 | 5254.1ms/step | 11.4 samples/s +Step 19840/50462 | Loss: 2.6009 | LR: 3.39e-04 | 5253.6ms/step | 11.4 samples/s +Step 19850/50462 | Loss: 2.5985 | LR: 3.39e-04 | 5257.7ms/step | 11.4 samples/s +Step 19860/50462 | Loss: 2.6079 | LR: 3.39e-04 | 5256.7ms/step | 11.4 samples/s +Step 19870/50462 | Loss: 2.5992 | LR: 3.39e-04 | 5254.8ms/step | 11.4 samples/s +Step 19880/50462 | Loss: 2.6109 | LR: 3.39e-04 | 5260.4ms/step | 11.4 samples/s +Step 19890/50462 | Loss: 2.6028 | LR: 3.38e-04 | 5254.9ms/step | 11.4 samples/s +Step 19900/50462 | Loss: 2.6184 | LR: 3.38e-04 | 5254.8ms/step | 11.4 samples/s +Step 19910/50462 | Loss: 2.5813 | LR: 3.38e-04 | 5253.6ms/step | 11.4 samples/s +Step 19920/50462 | Loss: 2.6032 | LR: 3.38e-04 | 5262.1ms/step | 11.4 samples/s +Step 19930/50462 | Loss: 2.5982 | LR: 3.38e-04 | 5749.0ms/step | 10.4 samples/s +Step 19940/50462 | Loss: 2.5765 | LR: 3.38e-04 | 5244.0ms/step | 11.4 samples/s +Step 19950/50462 | Loss: 2.6264 | LR: 3.37e-04 | 5769.7ms/step | 10.4 samples/s +Step 19960/50462 | Loss: 2.6151 | LR: 3.37e-04 | 5251.7ms/step | 11.4 samples/s +Step 19970/50462 | Loss: 2.6187 | LR: 3.37e-04 | 5258.6ms/step | 11.4 samples/s +Step 19980/50462 | Loss: 2.5764 | LR: 3.37e-04 | 5259.5ms/step | 11.4 samples/s +Step 19990/50462 | Loss: 2.5966 | LR: 3.37e-04 | 5255.8ms/step | 11.4 samples/s +Step 20000/50462 | Loss: 2.5910 | LR: 3.37e-04 | 5260.7ms/step | 11.4 samples/s +Validation at step 20000: Loss = 2.6223, Perplexity = 13.77 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-20000 +Switching attention mode for step 20001: sparse_stochastic (configured sparse attention) +Step 20010/50462 | Loss: 2.6322 | LR: 3.37e-04 | 7175.1ms/step | 8.4 samples/s +Step 20020/50462 | Loss: 2.6345 | LR: 3.36e-04 | 3105.3ms/step | 19.3 samples/s +Step 20030/50462 | Loss: 2.6399 | LR: 3.36e-04 | 3116.2ms/step | 19.3 samples/s +Step 20040/50462 | Loss: 2.6618 | LR: 3.36e-04 | 3117.9ms/step | 19.2 samples/s +Step 20050/50462 | Loss: 2.6545 | LR: 3.36e-04 | 3119.8ms/step | 19.2 samples/s +Step 20060/50462 | Loss: 2.6164 | LR: 3.36e-04 | 3120.1ms/step | 19.2 samples/s +Step 20070/50462 | Loss: 2.6232 | LR: 3.36e-04 | 3120.6ms/step | 19.2 samples/s +Step 20080/50462 | Loss: 2.6221 | LR: 3.36e-04 | 3120.0ms/step | 19.2 samples/s +Step 20090/50462 | Loss: 2.6462 | LR: 3.35e-04 | 3120.9ms/step | 19.2 samples/s +Step 20100/50462 | Loss: 2.6626 | LR: 3.35e-04 | 3119.8ms/step | 19.2 samples/s +Step 20110/50462 | Loss: 2.6138 | LR: 3.35e-04 | 3121.8ms/step | 19.2 samples/s +Step 20120/50462 | Loss: 2.6397 | LR: 3.35e-04 | 3122.3ms/step | 19.2 samples/s +Step 20130/50462 | Loss: 2.6641 | LR: 3.35e-04 | 3119.3ms/step | 19.2 samples/s +Step 20140/50462 | Loss: 2.6037 | LR: 3.35e-04 | 3120.3ms/step | 19.2 samples/s +Step 20150/50462 | Loss: 2.6477 | LR: 3.35e-04 | 3118.8ms/step | 19.2 samples/s +Step 20160/50462 | Loss: 2.6280 | LR: 3.34e-04 | 3118.1ms/step | 19.2 samples/s +Step 20170/50462 | Loss: 2.6712 | LR: 3.34e-04 | 3116.5ms/step | 19.3 samples/s +Step 20180/50462 | Loss: 2.6034 | LR: 3.34e-04 | 3116.7ms/step | 19.3 samples/s +Step 20190/50462 | Loss: 2.6350 | LR: 3.34e-04 | 3121.9ms/step | 19.2 samples/s +Step 20200/50462 | Loss: 2.6403 | LR: 3.34e-04 | 3119.9ms/step | 19.2 samples/s +Step 20210/50462 | Loss: 2.6602 | LR: 3.34e-04 | 3121.7ms/step | 19.2 samples/s +Step 20220/50462 | Loss: 2.6533 | LR: 3.33e-04 | 3124.1ms/step | 19.2 samples/s +Step 20230/50462 | Loss: 2.6323 | LR: 3.33e-04 | 3122.1ms/step | 19.2 samples/s +Step 20240/50462 | Loss: 2.6286 | LR: 3.33e-04 | 3119.1ms/step | 19.2 samples/s +Step 20250/50462 | Loss: 2.6293 | LR: 3.33e-04 | 3120.9ms/step | 19.2 samples/s +Validation at step 20250: Loss = 2.6469, Perplexity = 14.11 +Step 20260/50462 | Loss: 2.6549 | LR: 3.33e-04 | 4786.5ms/step | 12.5 samples/s +Step 20270/50462 | Loss: 2.6805 | LR: 3.33e-04 | 3120.8ms/step | 19.2 samples/s +Step 20280/50462 | Loss: 2.5977 | LR: 3.33e-04 | 3118.0ms/step | 19.2 samples/s +Step 20290/50462 | Loss: 2.6619 | LR: 3.32e-04 | 3122.8ms/step | 19.2 samples/s +Step 20300/50462 | Loss: 2.6389 | LR: 3.32e-04 | 3121.0ms/step | 19.2 samples/s +Step 20310/50462 | Loss: 2.6351 | LR: 3.32e-04 | 3121.5ms/step | 19.2 samples/s +Step 20320/50462 | Loss: 2.6289 | LR: 3.32e-04 | 3122.6ms/step | 19.2 samples/s +Step 20330/50462 | Loss: 2.6605 | LR: 3.32e-04 | 3123.5ms/step | 19.2 samples/s +Step 20340/50462 | Loss: 2.6220 | LR: 3.32e-04 | 3121.6ms/step | 19.2 samples/s +Step 20350/50462 | Loss: 2.6168 | LR: 3.32e-04 | 3119.6ms/step | 19.2 samples/s +Step 20360/50462 | Loss: 2.6581 | LR: 3.31e-04 | 3120.3ms/step | 19.2 samples/s +Step 20370/50462 | Loss: 2.6429 | LR: 3.31e-04 | 3121.8ms/step | 19.2 samples/s +Step 20380/50462 | Loss: 2.6594 | LR: 3.31e-04 | 3121.8ms/step | 19.2 samples/s +Step 20390/50462 | Loss: 2.6237 | LR: 3.31e-04 | 3119.4ms/step | 19.2 samples/s +Step 20400/50462 | Loss: 2.6294 | LR: 3.31e-04 | 3118.3ms/step | 19.2 samples/s +Step 20410/50462 | Loss: 2.6228 | LR: 3.31e-04 | 3118.9ms/step | 19.2 samples/s +Step 20420/50462 | Loss: 2.6419 | LR: 3.30e-04 | 3117.4ms/step | 19.2 samples/s +Step 20430/50462 | Loss: 2.6213 | LR: 3.30e-04 | 3120.7ms/step | 19.2 samples/s +Step 20440/50462 | Loss: 2.6330 | LR: 3.30e-04 | 3120.8ms/step | 19.2 samples/s +Step 20450/50462 | Loss: 2.6409 | LR: 3.30e-04 | 3120.7ms/step | 19.2 samples/s +Step 20460/50462 | Loss: 2.6386 | LR: 3.30e-04 | 3121.4ms/step | 19.2 samples/s +Step 20470/50462 | Loss: 2.6599 | LR: 3.30e-04 | 3123.2ms/step | 19.2 samples/s +Step 20480/50462 | Loss: 2.6534 | LR: 3.30e-04 | 3124.9ms/step | 19.2 samples/s +Step 20490/50462 | Loss: 2.6433 | LR: 3.29e-04 | 3120.8ms/step | 19.2 samples/s +Step 20500/50462 | Loss: 2.6141 | LR: 3.29e-04 | 3123.0ms/step | 19.2 samples/s +Validation at step 20500: Loss = 2.6443, Perplexity = 14.07 +Step 20510/50462 | Loss: 2.6311 | LR: 3.29e-04 | 4794.8ms/step | 12.5 samples/s +Step 20520/50462 | Loss: 2.6322 | LR: 3.29e-04 | 3121.8ms/step | 19.2 samples/s +Step 20530/50462 | Loss: 2.6033 | LR: 3.29e-04 | 3122.5ms/step | 19.2 samples/s +Step 20540/50462 | Loss: 2.6317 | LR: 3.29e-04 | 3118.7ms/step | 19.2 samples/s +Step 20550/50462 | Loss: 2.6504 | LR: 3.29e-04 | 3120.6ms/step | 19.2 samples/s +Step 20560/50462 | Loss: 2.6692 | LR: 3.28e-04 | 3122.5ms/step | 19.2 samples/s +Step 20570/50462 | Loss: 2.6606 | LR: 3.28e-04 | 3123.6ms/step | 19.2 samples/s +Step 20580/50462 | Loss: 2.6413 | LR: 3.28e-04 | 3120.7ms/step | 19.2 samples/s +Step 20590/50462 | Loss: 2.6621 | LR: 3.28e-04 | 3119.8ms/step | 19.2 samples/s +Step 20600/50462 | Loss: 2.6129 | LR: 3.28e-04 | 3117.9ms/step | 19.2 samples/s +Step 20610/50462 | Loss: 2.6156 | LR: 3.28e-04 | 3123.0ms/step | 19.2 samples/s +Step 20620/50462 | Loss: 2.6152 | LR: 3.27e-04 | 3122.3ms/step | 19.2 samples/s +Step 20630/50462 | Loss: 2.6311 | LR: 3.27e-04 | 3118.3ms/step | 19.2 samples/s +Step 20640/50462 | Loss: 2.6368 | LR: 3.27e-04 | 3118.2ms/step | 19.2 samples/s +Step 20650/50462 | Loss: 2.6138 | LR: 3.27e-04 | 3119.3ms/step | 19.2 samples/s +Step 20660/50462 | Loss: 2.6509 | LR: 3.27e-04 | 3123.3ms/step | 19.2 samples/s +Step 20670/50462 | Loss: 2.6676 | LR: 3.27e-04 | 3121.8ms/step | 19.2 samples/s +Step 20680/50462 | Loss: 2.6134 | LR: 3.27e-04 | 3124.8ms/step | 19.2 samples/s +Step 20690/50462 | Loss: 2.6282 | LR: 3.26e-04 | 3123.8ms/step | 19.2 samples/s +Step 20700/50462 | Loss: 2.5973 | LR: 3.26e-04 | 3118.8ms/step | 19.2 samples/s +Step 20710/50462 | Loss: 2.6429 | LR: 3.26e-04 | 3120.1ms/step | 19.2 samples/s +Step 20720/50462 | Loss: 2.6489 | LR: 3.26e-04 | 3120.4ms/step | 19.2 samples/s +Step 20730/50462 | Loss: 2.6061 | LR: 3.26e-04 | 3122.8ms/step | 19.2 samples/s +Step 20740/50462 | Loss: 2.6182 | LR: 3.26e-04 | 3124.6ms/step | 19.2 samples/s +Step 20750/50462 | Loss: 2.6309 | LR: 3.26e-04 | 3123.9ms/step | 19.2 samples/s +Validation at step 20750: Loss = 2.6422, Perplexity = 14.04 +Step 20760/50462 | Loss: 2.6420 | LR: 3.25e-04 | 4793.5ms/step | 12.5 samples/s +Step 20770/50462 | Loss: 2.6429 | LR: 3.25e-04 | 3121.0ms/step | 19.2 samples/s +Step 20780/50462 | Loss: 2.6522 | LR: 3.25e-04 | 3121.7ms/step | 19.2 samples/s +Step 20790/50462 | Loss: 2.6011 | LR: 3.25e-04 | 3124.5ms/step | 19.2 samples/s +Step 20800/50462 | Loss: 2.6431 | LR: 3.25e-04 | 3122.6ms/step | 19.2 samples/s +Step 20810/50462 | Loss: 2.6394 | LR: 3.25e-04 | 3122.8ms/step | 19.2 samples/s +Step 20820/50462 | Loss: 2.6468 | LR: 3.24e-04 | 3122.1ms/step | 19.2 samples/s +Step 20830/50462 | Loss: 2.6215 | LR: 3.24e-04 | 3120.4ms/step | 19.2 samples/s +Step 20840/50462 | Loss: 2.6500 | LR: 3.24e-04 | 3121.9ms/step | 19.2 samples/s +Step 20850/50462 | Loss: 2.6424 | LR: 3.24e-04 | 3122.0ms/step | 19.2 samples/s +Step 20860/50462 | Loss: 2.6035 | LR: 3.24e-04 | 3118.6ms/step | 19.2 samples/s +Step 20870/50462 | Loss: 2.6132 | LR: 3.24e-04 | 3116.7ms/step | 19.3 samples/s +Step 20880/50462 | Loss: 2.6353 | LR: 3.24e-04 | 3115.2ms/step | 19.3 samples/s +Step 20890/50462 | Loss: 2.6435 | LR: 3.23e-04 | 3115.6ms/step | 19.3 samples/s +Step 20900/50462 | Loss: 2.6130 | LR: 3.23e-04 | 3114.6ms/step | 19.3 samples/s +Step 20910/50462 | Loss: 2.6209 | LR: 3.23e-04 | 3120.8ms/step | 19.2 samples/s +Step 20920/50462 | Loss: 2.6421 | LR: 3.23e-04 | 3112.6ms/step | 19.3 samples/s +Step 20930/50462 | Loss: 2.6594 | LR: 3.23e-04 | 3114.1ms/step | 19.3 samples/s +Step 20940/50462 | Loss: 2.6325 | LR: 3.23e-04 | 3116.9ms/step | 19.2 samples/s +Step 20950/50462 | Loss: 2.6506 | LR: 3.23e-04 | 3111.9ms/step | 19.3 samples/s +Step 20960/50462 | Loss: 2.6448 | LR: 3.22e-04 | 3112.4ms/step | 19.3 samples/s +Step 20970/50462 | Loss: 2.6080 | LR: 3.22e-04 | 3112.4ms/step | 19.3 samples/s +Step 20980/50462 | Loss: 2.6201 | LR: 3.22e-04 | 3110.0ms/step | 19.3 samples/s +Step 20990/50462 | Loss: 2.6321 | LR: 3.22e-04 | 3110.8ms/step | 19.3 samples/s +Step 21000/50462 | Loss: 2.6471 | LR: 3.22e-04 | 3113.1ms/step | 19.3 samples/s +Validation at step 21000: Loss = 2.6396, Perplexity = 14.01 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-21000 +Step 21010/50462 | Loss: 2.6463 | LR: 3.22e-04 | 5719.0ms/step | 10.5 samples/s +Step 21020/50462 | Loss: 2.6538 | LR: 3.21e-04 | 3111.5ms/step | 19.3 samples/s +Step 21030/50462 | Loss: 2.6029 | LR: 3.21e-04 | 3110.9ms/step | 19.3 samples/s +Step 21040/50462 | Loss: 2.6258 | LR: 3.21e-04 | 3114.2ms/step | 19.3 samples/s +Step 21050/50462 | Loss: 2.6113 | LR: 3.21e-04 | 3117.3ms/step | 19.2 samples/s +Step 21060/50462 | Loss: 2.6207 | LR: 3.21e-04 | 3118.3ms/step | 19.2 samples/s +Step 21070/50462 | Loss: 2.6241 | LR: 3.21e-04 | 3116.4ms/step | 19.3 samples/s +Step 21080/50462 | Loss: 2.6273 | LR: 3.21e-04 | 3120.5ms/step | 19.2 samples/s +Step 21090/50462 | Loss: 2.6487 | LR: 3.20e-04 | 3114.5ms/step | 19.3 samples/s +Step 21100/50462 | Loss: 2.6636 | LR: 3.20e-04 | 3112.4ms/step | 19.3 samples/s +Step 21110/50462 | Loss: 2.6355 | LR: 3.20e-04 | 3117.0ms/step | 19.2 samples/s +Step 21120/50462 | Loss: 2.6076 | LR: 3.20e-04 | 3115.4ms/step | 19.3 samples/s +Step 21130/50462 | Loss: 2.6354 | LR: 3.20e-04 | 3116.7ms/step | 19.3 samples/s +Step 21140/50462 | Loss: 2.6357 | LR: 3.20e-04 | 3114.3ms/step | 19.3 samples/s +Step 21150/50462 | Loss: 2.6086 | LR: 3.19e-04 | 3117.8ms/step | 19.2 samples/s +Step 21160/50462 | Loss: 2.5979 | LR: 3.19e-04 | 3115.8ms/step | 19.3 samples/s +Step 21170/50462 | Loss: 2.6320 | LR: 3.19e-04 | 3118.0ms/step | 19.2 samples/s +Step 21180/50462 | Loss: 2.6469 | LR: 3.19e-04 | 3120.3ms/step | 19.2 samples/s +Step 21190/50462 | Loss: 2.6552 | LR: 3.19e-04 | 3118.6ms/step | 19.2 samples/s +Step 21200/50462 | Loss: 2.6408 | LR: 3.19e-04 | 3118.1ms/step | 19.2 samples/s +Step 21210/50462 | Loss: 2.6502 | LR: 3.19e-04 | 3116.2ms/step | 19.3 samples/s +Step 21220/50462 | Loss: 2.6467 | LR: 3.18e-04 | 3118.8ms/step | 19.2 samples/s +Step 21230/50462 | Loss: 2.6006 | LR: 3.18e-04 | 3115.9ms/step | 19.3 samples/s +Step 21240/50462 | Loss: 2.6454 | LR: 3.18e-04 | 3118.3ms/step | 19.2 samples/s +Step 21250/50462 | Loss: 2.6237 | LR: 3.18e-04 | 3118.8ms/step | 19.2 samples/s +Validation at step 21250: Loss = 2.6365, Perplexity = 13.96 +Step 21260/50462 | Loss: 2.6109 | LR: 3.18e-04 | 4781.3ms/step | 12.5 samples/s +Step 21270/50462 | Loss: 2.6150 | LR: 3.18e-04 | 3119.5ms/step | 19.2 samples/s +Step 21280/50462 | Loss: 2.6560 | LR: 3.17e-04 | 3114.6ms/step | 19.3 samples/s +Step 21290/50462 | Loss: 2.6401 | LR: 3.17e-04 | 3116.7ms/step | 19.3 samples/s +Step 21300/50462 | Loss: 2.6198 | LR: 3.17e-04 | 3118.4ms/step | 19.2 samples/s +Step 21310/50462 | Loss: 2.6337 | LR: 3.17e-04 | 3115.4ms/step | 19.3 samples/s +Step 21320/50462 | Loss: 2.6160 | LR: 3.17e-04 | 3115.0ms/step | 19.3 samples/s +Step 21330/50462 | Loss: 2.6283 | LR: 3.17e-04 | 3115.3ms/step | 19.3 samples/s +Step 21340/50462 | Loss: 2.6300 | LR: 3.17e-04 | 3117.6ms/step | 19.2 samples/s +Step 21350/50462 | Loss: 2.6013 | LR: 3.16e-04 | 3118.8ms/step | 19.2 samples/s +Step 21360/50462 | Loss: 2.6173 | LR: 3.16e-04 | 3112.9ms/step | 19.3 samples/s +Step 21370/50462 | Loss: 2.6392 | LR: 3.16e-04 | 3113.8ms/step | 19.3 samples/s +Step 21380/50462 | Loss: 2.6177 | LR: 3.16e-04 | 3113.9ms/step | 19.3 samples/s +Step 21390/50462 | Loss: 2.6554 | LR: 3.16e-04 | 3113.9ms/step | 19.3 samples/s +Step 21400/50462 | Loss: 2.6410 | LR: 3.16e-04 | 3115.8ms/step | 19.3 samples/s +Step 21410/50462 | Loss: 2.6446 | LR: 3.16e-04 | 3116.5ms/step | 19.3 samples/s +Step 21420/50462 | Loss: 2.6165 | LR: 3.15e-04 | 3117.9ms/step | 19.2 samples/s +Step 21430/50462 | Loss: 2.6278 | LR: 3.15e-04 | 3118.7ms/step | 19.2 samples/s +Step 21440/50462 | Loss: 2.6160 | LR: 3.15e-04 | 3115.5ms/step | 19.3 samples/s +Step 21450/50462 | Loss: 2.6180 | LR: 3.15e-04 | 3114.0ms/step | 19.3 samples/s +Step 21460/50462 | Loss: 2.6110 | LR: 3.15e-04 | 3114.4ms/step | 19.3 samples/s +Step 21470/50462 | Loss: 2.6217 | LR: 3.15e-04 | 3117.2ms/step | 19.2 samples/s +Step 21480/50462 | Loss: 2.6266 | LR: 3.14e-04 | 3115.9ms/step | 19.3 samples/s +Step 21490/50462 | Loss: 2.6277 | LR: 3.14e-04 | 3119.2ms/step | 19.2 samples/s +Step 21500/50462 | Loss: 2.6031 | LR: 3.14e-04 | 3116.1ms/step | 19.3 samples/s +Validation at step 21500: Loss = 2.6339, Perplexity = 13.93 +Step 21510/50462 | Loss: 2.6255 | LR: 3.14e-04 | 4782.2ms/step | 12.5 samples/s +Step 21520/50462 | Loss: 2.5900 | LR: 3.14e-04 | 3116.3ms/step | 19.3 samples/s +Step 21530/50462 | Loss: 2.6442 | LR: 3.14e-04 | 3115.0ms/step | 19.3 samples/s +Step 21540/50462 | Loss: 2.6040 | LR: 3.14e-04 | 3117.5ms/step | 19.2 samples/s +Step 21550/50462 | Loss: 2.6205 | LR: 3.13e-04 | 3116.1ms/step | 19.3 samples/s +Step 21560/50462 | Loss: 2.6037 | LR: 3.13e-04 | 3119.3ms/step | 19.2 samples/s +Step 21570/50462 | Loss: 2.6266 | LR: 3.13e-04 | 3119.1ms/step | 19.2 samples/s +Step 21580/50462 | Loss: 2.6019 | LR: 3.13e-04 | 3117.1ms/step | 19.2 samples/s +Step 21590/50462 | Loss: 2.6175 | LR: 3.13e-04 | 3120.4ms/step | 19.2 samples/s +Step 21600/50462 | Loss: 2.5921 | LR: 3.13e-04 | 3118.1ms/step | 19.2 samples/s +Step 21610/50462 | Loss: 2.6323 | LR: 3.12e-04 | 3119.1ms/step | 19.2 samples/s +Step 21620/50462 | Loss: 2.6334 | LR: 3.12e-04 | 3120.7ms/step | 19.2 samples/s +Step 21630/50462 | Loss: 2.6215 | LR: 3.12e-04 | 3120.1ms/step | 19.2 samples/s +Step 21640/50462 | Loss: 2.5859 | LR: 3.12e-04 | 3120.1ms/step | 19.2 samples/s +Step 21650/50462 | Loss: 2.6476 | LR: 3.12e-04 | 3117.3ms/step | 19.2 samples/s +Step 21660/50462 | Loss: 2.5893 | LR: 3.12e-04 | 3118.1ms/step | 19.2 samples/s +Step 21670/50462 | Loss: 2.5921 | LR: 3.12e-04 | 3117.0ms/step | 19.2 samples/s +Step 21680/50462 | Loss: 2.6232 | LR: 3.11e-04 | 3120.7ms/step | 19.2 samples/s +Step 21690/50462 | Loss: 2.6002 | LR: 3.11e-04 | 3118.3ms/step | 19.2 samples/s +Step 21700/50462 | Loss: 2.6449 | LR: 3.11e-04 | 3116.0ms/step | 19.3 samples/s +Step 21710/50462 | Loss: 2.6526 | LR: 3.11e-04 | 3120.5ms/step | 19.2 samples/s +Step 21720/50462 | Loss: 2.5985 | LR: 3.11e-04 | 3119.3ms/step | 19.2 samples/s +Step 21730/50462 | Loss: 2.6267 | LR: 3.11e-04 | 3122.2ms/step | 19.2 samples/s +Step 21740/50462 | Loss: 2.6011 | LR: 3.10e-04 | 3118.9ms/step | 19.2 samples/s +Step 21750/50462 | Loss: 2.5935 | LR: 3.10e-04 | 3117.4ms/step | 19.2 samples/s +Validation at step 21750: Loss = 2.6316, Perplexity = 13.90 +Step 21760/50462 | Loss: 2.6060 | LR: 3.10e-04 | 4782.1ms/step | 12.5 samples/s +Step 21770/50462 | Loss: 2.5869 | LR: 3.10e-04 | 3118.9ms/step | 19.2 samples/s +Step 21780/50462 | Loss: 2.5844 | LR: 3.10e-04 | 3119.6ms/step | 19.2 samples/s +Step 21790/50462 | Loss: 2.6039 | LR: 3.10e-04 | 3121.9ms/step | 19.2 samples/s +Step 21800/50462 | Loss: 2.6284 | LR: 3.10e-04 | 3120.4ms/step | 19.2 samples/s +Step 21810/50462 | Loss: 2.6490 | LR: 3.09e-04 | 3120.9ms/step | 19.2 samples/s +Step 21820/50462 | Loss: 2.6014 | LR: 3.09e-04 | 3118.3ms/step | 19.2 samples/s +Step 21830/50462 | Loss: 2.6235 | LR: 3.09e-04 | 3119.2ms/step | 19.2 samples/s +Step 21840/50462 | Loss: 2.6185 | LR: 3.09e-04 | 3118.5ms/step | 19.2 samples/s +Step 21850/50462 | Loss: 2.6055 | LR: 3.09e-04 | 3116.6ms/step | 19.3 samples/s +Step 21860/50462 | Loss: 2.6293 | LR: 3.09e-04 | 3117.4ms/step | 19.2 samples/s +Step 21870/50462 | Loss: 2.6126 | LR: 3.08e-04 | 3117.4ms/step | 19.2 samples/s +Step 21880/50462 | Loss: 2.6169 | LR: 3.08e-04 | 3119.4ms/step | 19.2 samples/s +Step 21890/50462 | Loss: 2.6273 | LR: 3.08e-04 | 3117.9ms/step | 19.2 samples/s +Step 21900/50462 | Loss: 2.6008 | LR: 3.08e-04 | 3118.7ms/step | 19.2 samples/s +Step 21910/50462 | Loss: 2.6020 | LR: 3.08e-04 | 3118.8ms/step | 19.2 samples/s +Step 21920/50462 | Loss: 2.6118 | LR: 3.08e-04 | 3116.7ms/step | 19.3 samples/s +Step 21930/50462 | Loss: 2.6319 | LR: 3.08e-04 | 3117.6ms/step | 19.2 samples/s +Step 21940/50462 | Loss: 2.6423 | LR: 3.07e-04 | 3121.3ms/step | 19.2 samples/s +Step 21950/50462 | Loss: 2.6195 | LR: 3.07e-04 | 3118.9ms/step | 19.2 samples/s +Step 21960/50462 | Loss: 2.6134 | LR: 3.07e-04 | 3122.1ms/step | 19.2 samples/s +Step 21970/50462 | Loss: 2.6114 | LR: 3.07e-04 | 3120.5ms/step | 19.2 samples/s +Step 21980/50462 | Loss: 2.5998 | LR: 3.07e-04 | 3121.5ms/step | 19.2 samples/s +Step 21990/50462 | Loss: 2.6130 | LR: 3.07e-04 | 3122.3ms/step | 19.2 samples/s +Step 22000/50462 | Loss: 2.6367 | LR: 3.06e-04 | 3119.1ms/step | 19.2 samples/s +Validation at step 22000: Loss = 2.6287, Perplexity = 13.86 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-22000 +Step 22010/50462 | Loss: 2.6005 | LR: 3.06e-04 | 5741.9ms/step | 10.4 samples/s +Step 22020/50462 | Loss: 2.6071 | LR: 3.06e-04 | 3114.8ms/step | 19.3 samples/s +Step 22030/50462 | Loss: 2.6111 | LR: 3.06e-04 | 3119.9ms/step | 19.2 samples/s +Step 22040/50462 | Loss: 2.6347 | LR: 3.06e-04 | 3120.6ms/step | 19.2 samples/s +Step 22050/50462 | Loss: 2.6207 | LR: 3.06e-04 | 3123.0ms/step | 19.2 samples/s +Step 22060/50462 | Loss: 2.5983 | LR: 3.06e-04 | 3119.9ms/step | 19.2 samples/s +Step 22070/50462 | Loss: 2.6361 | LR: 3.05e-04 | 3119.1ms/step | 19.2 samples/s +Step 22080/50462 | Loss: 2.6013 | LR: 3.05e-04 | 3638.4ms/step | 16.5 samples/s +Step 22090/50462 | Loss: 2.6200 | LR: 3.05e-04 | 3113.5ms/step | 19.3 samples/s +Step 22100/50462 | Loss: 2.6092 | LR: 3.05e-04 | 3117.2ms/step | 19.2 samples/s +Step 22110/50462 | Loss: 2.6268 | LR: 3.05e-04 | 3121.6ms/step | 19.2 samples/s +Step 22120/50462 | Loss: 2.6090 | LR: 3.05e-04 | 3122.1ms/step | 19.2 samples/s +Step 22130/50462 | Loss: 2.6214 | LR: 3.04e-04 | 3618.7ms/step | 16.6 samples/s +Step 22140/50462 | Loss: 2.5986 | LR: 3.04e-04 | 3104.0ms/step | 19.3 samples/s +Step 22150/50462 | Loss: 2.6242 | LR: 3.04e-04 | 3114.6ms/step | 19.3 samples/s +Step 22160/50462 | Loss: 2.6471 | LR: 3.04e-04 | 3116.7ms/step | 19.3 samples/s +Step 22170/50462 | Loss: 2.6044 | LR: 3.04e-04 | 3117.5ms/step | 19.2 samples/s +Step 22180/50462 | Loss: 2.6255 | LR: 3.04e-04 | 3117.1ms/step | 19.2 samples/s +Step 22190/50462 | Loss: 2.6329 | LR: 3.04e-04 | 3115.2ms/step | 19.3 samples/s +Step 22200/50462 | Loss: 2.6247 | LR: 3.03e-04 | 3117.0ms/step | 19.2 samples/s +Step 22210/50462 | Loss: 2.6423 | LR: 3.03e-04 | 3635.6ms/step | 16.5 samples/s +Step 22220/50462 | Loss: 2.6314 | LR: 3.03e-04 | 3106.4ms/step | 19.3 samples/s +Step 22230/50462 | Loss: 2.6032 | LR: 3.03e-04 | 3115.9ms/step | 19.3 samples/s +Step 22240/50462 | Loss: 2.6077 | LR: 3.03e-04 | 3116.2ms/step | 19.3 samples/s +Step 22250/50462 | Loss: 2.6049 | LR: 3.03e-04 | 3119.4ms/step | 19.2 samples/s +Validation at step 22250: Loss = 2.6265, Perplexity = 13.83 +Step 22260/50462 | Loss: 2.6126 | LR: 3.02e-04 | 4776.2ms/step | 12.6 samples/s +Step 22270/50462 | Loss: 2.6468 | LR: 3.02e-04 | 3118.2ms/step | 19.2 samples/s +Step 22280/50462 | Loss: 2.5845 | LR: 3.02e-04 | 3119.3ms/step | 19.2 samples/s +Step 22290/50462 | Loss: 2.6216 | LR: 3.02e-04 | 3114.8ms/step | 19.3 samples/s +Step 22300/50462 | Loss: 2.5934 | LR: 3.02e-04 | 3119.3ms/step | 19.2 samples/s +Step 22310/50462 | Loss: 2.6282 | LR: 3.02e-04 | 3115.8ms/step | 19.3 samples/s +Step 22320/50462 | Loss: 2.5945 | LR: 3.02e-04 | 3116.4ms/step | 19.3 samples/s +Step 22330/50462 | Loss: 2.6399 | LR: 3.01e-04 | 3118.2ms/step | 19.2 samples/s +Step 22340/50462 | Loss: 2.6124 | LR: 3.01e-04 | 3118.8ms/step | 19.2 samples/s +Step 22350/50462 | Loss: 2.6255 | LR: 3.01e-04 | 3118.7ms/step | 19.2 samples/s +Step 22360/50462 | Loss: 2.6145 | LR: 3.01e-04 | 3118.1ms/step | 19.2 samples/s +Step 22370/50462 | Loss: 2.6154 | LR: 3.01e-04 | 3117.1ms/step | 19.2 samples/s +Step 22380/50462 | Loss: 2.5903 | LR: 3.01e-04 | 3120.7ms/step | 19.2 samples/s +Step 22390/50462 | Loss: 2.6285 | LR: 3.00e-04 | 3119.2ms/step | 19.2 samples/s +Step 22400/50462 | Loss: 2.5989 | LR: 3.00e-04 | 3116.0ms/step | 19.3 samples/s +Step 22410/50462 | Loss: 2.6193 | LR: 3.00e-04 | 3115.2ms/step | 19.3 samples/s +Step 22420/50462 | Loss: 2.6037 | LR: 3.00e-04 | 3118.5ms/step | 19.2 samples/s +Step 22430/50462 | Loss: 2.6205 | LR: 3.00e-04 | 3118.8ms/step | 19.2 samples/s +Step 22440/50462 | Loss: 2.6307 | LR: 3.00e-04 | 3117.3ms/step | 19.2 samples/s +Step 22450/50462 | Loss: 2.6141 | LR: 3.00e-04 | 3119.3ms/step | 19.2 samples/s +Step 22460/50462 | Loss: 2.5826 | LR: 2.99e-04 | 3118.5ms/step | 19.2 samples/s +Step 22470/50462 | Loss: 2.6052 | LR: 2.99e-04 | 3118.1ms/step | 19.2 samples/s +Step 22480/50462 | Loss: 2.6127 | LR: 2.99e-04 | 3119.7ms/step | 19.2 samples/s +Step 22490/50462 | Loss: 2.5990 | LR: 2.99e-04 | 3119.8ms/step | 19.2 samples/s +Step 22500/50462 | Loss: 2.5997 | LR: 2.99e-04 | 3118.6ms/step | 19.2 samples/s +Validation at step 22500: Loss = 2.6245, Perplexity = 13.80 +Step 22510/50462 | Loss: 2.6190 | LR: 2.99e-04 | 4778.4ms/step | 12.6 samples/s +Step 22520/50462 | Loss: 2.6272 | LR: 2.98e-04 | 3115.7ms/step | 19.3 samples/s +Step 22530/50462 | Loss: 2.5997 | LR: 2.98e-04 | 3115.4ms/step | 19.3 samples/s +Step 22540/50462 | Loss: 2.6273 | LR: 2.98e-04 | 3112.8ms/step | 19.3 samples/s +Step 22550/50462 | Loss: 2.6018 | LR: 2.98e-04 | 3111.8ms/step | 19.3 samples/s +Step 22560/50462 | Loss: 2.6102 | LR: 2.98e-04 | 3112.6ms/step | 19.3 samples/s +Step 22570/50462 | Loss: 2.6205 | LR: 2.98e-04 | 3116.7ms/step | 19.3 samples/s +Step 22580/50462 | Loss: 2.5710 | LR: 2.98e-04 | 3113.0ms/step | 19.3 samples/s +Step 22590/50462 | Loss: 2.6423 | LR: 2.97e-04 | 3111.3ms/step | 19.3 samples/s +Step 22600/50462 | Loss: 2.6114 | LR: 2.97e-04 | 3115.1ms/step | 19.3 samples/s +Step 22610/50462 | Loss: 2.6303 | LR: 2.97e-04 | 3117.4ms/step | 19.2 samples/s +Step 22620/50462 | Loss: 2.5915 | LR: 2.97e-04 | 3115.7ms/step | 19.3 samples/s +Step 22630/50462 | Loss: 2.5867 | LR: 2.97e-04 | 3114.5ms/step | 19.3 samples/s +Step 22640/50462 | Loss: 2.5946 | LR: 2.97e-04 | 3118.6ms/step | 19.2 samples/s +Step 22650/50462 | Loss: 2.6539 | LR: 2.96e-04 | 3116.3ms/step | 19.3 samples/s +Step 22660/50462 | Loss: 2.6133 | LR: 2.96e-04 | 3115.8ms/step | 19.3 samples/s +Step 22670/50462 | Loss: 2.5960 | LR: 2.96e-04 | 3117.5ms/step | 19.2 samples/s +Step 22680/50462 | Loss: 2.6263 | LR: 2.96e-04 | 3109.5ms/step | 19.3 samples/s +Step 22690/50462 | Loss: 2.6042 | LR: 2.96e-04 | 3097.9ms/step | 19.4 samples/s +Step 22700/50462 | Loss: 2.6207 | LR: 2.96e-04 | 3098.1ms/step | 19.4 samples/s +Step 22710/50462 | Loss: 2.6009 | LR: 2.95e-04 | 3103.7ms/step | 19.3 samples/s +Step 22720/50462 | Loss: 2.6440 | LR: 2.95e-04 | 3100.9ms/step | 19.3 samples/s +Step 22730/50462 | Loss: 2.5991 | LR: 2.95e-04 | 3099.5ms/step | 19.4 samples/s +Step 22740/50462 | Loss: 2.6226 | LR: 2.95e-04 | 3095.5ms/step | 19.4 samples/s +Step 22750/50462 | Loss: 2.6332 | LR: 2.95e-04 | 3088.0ms/step | 19.4 samples/s +Validation at step 22750: Loss = 2.6227, Perplexity = 13.77 +Step 22760/50462 | Loss: 2.6255 | LR: 2.95e-04 | 4742.6ms/step | 12.7 samples/s +Step 22770/50462 | Loss: 2.6139 | LR: 2.95e-04 | 3095.2ms/step | 19.4 samples/s +Step 22780/50462 | Loss: 2.6093 | LR: 2.94e-04 | 3098.8ms/step | 19.4 samples/s +Step 22790/50462 | Loss: 2.6059 | LR: 2.94e-04 | 3102.3ms/step | 19.3 samples/s +Step 22800/50462 | Loss: 2.6310 | LR: 2.94e-04 | 3091.4ms/step | 19.4 samples/s +Step 22810/50462 | Loss: 2.6102 | LR: 2.94e-04 | 3095.8ms/step | 19.4 samples/s +Step 22820/50462 | Loss: 2.6029 | LR: 2.94e-04 | 3097.0ms/step | 19.4 samples/s +Step 22830/50462 | Loss: 2.6152 | LR: 2.94e-04 | 3099.3ms/step | 19.4 samples/s +Step 22840/50462 | Loss: 2.5928 | LR: 2.93e-04 | 3099.1ms/step | 19.4 samples/s +Step 22850/50462 | Loss: 2.5888 | LR: 2.93e-04 | 3094.7ms/step | 19.4 samples/s +Step 22860/50462 | Loss: 2.6156 | LR: 2.93e-04 | 3084.4ms/step | 19.5 samples/s +Step 22870/50462 | Loss: 2.6283 | LR: 2.93e-04 | 3095.5ms/step | 19.4 samples/s +Step 22880/50462 | Loss: 2.6327 | LR: 2.93e-04 | 3095.8ms/step | 19.4 samples/s +Step 22890/50462 | Loss: 2.5901 | LR: 2.93e-04 | 3095.9ms/step | 19.4 samples/s +Step 22900/50462 | Loss: 2.5936 | LR: 2.93e-04 | 3100.2ms/step | 19.4 samples/s +Step 22910/50462 | Loss: 2.6110 | LR: 2.92e-04 | 3102.5ms/step | 19.3 samples/s +Step 22920/50462 | Loss: 2.5741 | LR: 2.92e-04 | 3102.4ms/step | 19.3 samples/s +Step 22930/50462 | Loss: 2.6141 | LR: 2.92e-04 | 3102.8ms/step | 19.3 samples/s +Step 22940/50462 | Loss: 2.5894 | LR: 2.92e-04 | 3096.8ms/step | 19.4 samples/s +Step 22950/50462 | Loss: 2.5905 | LR: 2.92e-04 | 3103.8ms/step | 19.3 samples/s +Step 22960/50462 | Loss: 2.5930 | LR: 2.92e-04 | 3108.1ms/step | 19.3 samples/s +Step 22970/50462 | Loss: 2.6065 | LR: 2.91e-04 | 3104.4ms/step | 19.3 samples/s +Step 22980/50462 | Loss: 2.6082 | LR: 2.91e-04 | 3097.9ms/step | 19.4 samples/s +Step 22990/50462 | Loss: 2.6216 | LR: 2.91e-04 | 3106.1ms/step | 19.3 samples/s +Step 23000/50462 | Loss: 2.5928 | LR: 2.91e-04 | 3107.7ms/step | 19.3 samples/s +Validation at step 23000: Loss = 2.6198, Perplexity = 13.73 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-23000 +Step 23010/50462 | Loss: 2.6157 | LR: 2.91e-04 | 5690.6ms/step | 10.5 samples/s +Step 23020/50462 | Loss: 2.5758 | LR: 2.91e-04 | 3100.9ms/step | 19.3 samples/s +Step 23030/50462 | Loss: 2.6054 | LR: 2.91e-04 | 3104.4ms/step | 19.3 samples/s +Step 23040/50462 | Loss: 2.6070 | LR: 2.90e-04 | 3107.9ms/step | 19.3 samples/s +Step 23050/50462 | Loss: 2.6022 | LR: 2.90e-04 | 3106.5ms/step | 19.3 samples/s +Step 23060/50462 | Loss: 2.5637 | LR: 2.90e-04 | 3117.0ms/step | 19.2 samples/s +Step 23070/50462 | Loss: 2.5967 | LR: 2.90e-04 | 3108.5ms/step | 19.3 samples/s +Step 23080/50462 | Loss: 2.6107 | LR: 2.90e-04 | 3111.7ms/step | 19.3 samples/s +Step 23090/50462 | Loss: 2.5867 | LR: 2.90e-04 | 3109.9ms/step | 19.3 samples/s +Step 23100/50462 | Loss: 2.5875 | LR: 2.89e-04 | 3113.4ms/step | 19.3 samples/s +Step 23110/50462 | Loss: 2.5930 | LR: 2.89e-04 | 3110.3ms/step | 19.3 samples/s +Step 23120/50462 | Loss: 2.6430 | LR: 2.89e-04 | 3108.5ms/step | 19.3 samples/s +Step 23130/50462 | Loss: 2.6039 | LR: 2.89e-04 | 3107.0ms/step | 19.3 samples/s +Step 23140/50462 | Loss: 2.6070 | LR: 2.89e-04 | 3104.8ms/step | 19.3 samples/s +Step 23150/50462 | Loss: 2.6304 | LR: 2.89e-04 | 3108.1ms/step | 19.3 samples/s +Step 23160/50462 | Loss: 2.6014 | LR: 2.88e-04 | 3109.0ms/step | 19.3 samples/s +Step 23170/50462 | Loss: 2.6182 | LR: 2.88e-04 | 3106.0ms/step | 19.3 samples/s +Step 23180/50462 | Loss: 2.5949 | LR: 2.88e-04 | 3109.4ms/step | 19.3 samples/s +Step 23190/50462 | Loss: 2.6220 | LR: 2.88e-04 | 3108.9ms/step | 19.3 samples/s +Step 23200/50462 | Loss: 2.6145 | LR: 2.88e-04 | 3114.0ms/step | 19.3 samples/s +Step 23210/50462 | Loss: 2.6000 | LR: 2.88e-04 | 3109.0ms/step | 19.3 samples/s +Step 23220/50462 | Loss: 2.6133 | LR: 2.88e-04 | 3110.6ms/step | 19.3 samples/s +Step 23230/50462 | Loss: 2.6222 | LR: 2.87e-04 | 3112.7ms/step | 19.3 samples/s +Step 23240/50462 | Loss: 2.5776 | LR: 2.87e-04 | 3108.8ms/step | 19.3 samples/s +Step 23250/50462 | Loss: 2.6166 | LR: 2.87e-04 | 3109.6ms/step | 19.3 samples/s +Validation at step 23250: Loss = 2.6175, Perplexity = 13.70 +Step 23260/50462 | Loss: 2.6262 | LR: 2.87e-04 | 4751.1ms/step | 12.6 samples/s +Step 23270/50462 | Loss: 2.6031 | LR: 2.87e-04 | 3101.6ms/step | 19.3 samples/s +Step 23280/50462 | Loss: 2.5945 | LR: 2.87e-04 | 3105.7ms/step | 19.3 samples/s +Step 23290/50462 | Loss: 2.6169 | LR: 2.86e-04 | 3102.9ms/step | 19.3 samples/s +Step 23300/50462 | Loss: 2.6010 | LR: 2.86e-04 | 3102.4ms/step | 19.3 samples/s +Step 23310/50462 | Loss: 2.6249 | LR: 2.86e-04 | 3103.9ms/step | 19.3 samples/s +Step 23320/50462 | Loss: 2.6239 | LR: 2.86e-04 | 3108.0ms/step | 19.3 samples/s +Step 23330/50462 | Loss: 2.6105 | LR: 2.86e-04 | 3105.8ms/step | 19.3 samples/s +Step 23340/50462 | Loss: 2.6093 | LR: 2.86e-04 | 3107.4ms/step | 19.3 samples/s +Step 23350/50462 | Loss: 2.6049 | LR: 2.86e-04 | 3099.7ms/step | 19.4 samples/s +Step 23360/50462 | Loss: 2.5622 | LR: 2.85e-04 | 3098.1ms/step | 19.4 samples/s +Step 23370/50462 | Loss: 2.6141 | LR: 2.85e-04 | 3107.8ms/step | 19.3 samples/s +Step 23380/50462 | Loss: 2.5991 | LR: 2.85e-04 | 3107.6ms/step | 19.3 samples/s +Step 23390/50462 | Loss: 2.5907 | LR: 2.85e-04 | 3106.6ms/step | 19.3 samples/s +Step 23400/50462 | Loss: 2.6238 | LR: 2.85e-04 | 3107.4ms/step | 19.3 samples/s +Step 23410/50462 | Loss: 2.5877 | LR: 2.85e-04 | 3107.9ms/step | 19.3 samples/s +Step 23420/50462 | Loss: 2.6148 | LR: 2.84e-04 | 3108.7ms/step | 19.3 samples/s +Step 23430/50462 | Loss: 2.5966 | LR: 2.84e-04 | 3097.4ms/step | 19.4 samples/s +Step 23440/50462 | Loss: 2.5982 | LR: 2.84e-04 | 3103.0ms/step | 19.3 samples/s +Step 23450/50462 | Loss: 2.5901 | LR: 2.84e-04 | 3108.0ms/step | 19.3 samples/s +Step 23460/50462 | Loss: 2.6368 | LR: 2.84e-04 | 3111.3ms/step | 19.3 samples/s +Step 23470/50462 | Loss: 2.6432 | LR: 2.84e-04 | 3097.5ms/step | 19.4 samples/s +Step 23480/50462 | Loss: 2.6322 | LR: 2.83e-04 | 3100.5ms/step | 19.4 samples/s +Step 23490/50462 | Loss: 2.5822 | LR: 2.83e-04 | 3101.5ms/step | 19.3 samples/s +Step 23500/50462 | Loss: 2.6280 | LR: 2.83e-04 | 3100.4ms/step | 19.4 samples/s +Validation at step 23500: Loss = 2.6159, Perplexity = 13.68 +Step 23510/50462 | Loss: 2.5871 | LR: 2.83e-04 | 4748.4ms/step | 12.6 samples/s +Step 23520/50462 | Loss: 2.6055 | LR: 2.83e-04 | 3107.5ms/step | 19.3 samples/s +Step 23530/50462 | Loss: 2.6112 | LR: 2.83e-04 | 3110.2ms/step | 19.3 samples/s +Step 23540/50462 | Loss: 2.6213 | LR: 2.83e-04 | 3113.5ms/step | 19.3 samples/s +Step 23550/50462 | Loss: 2.5792 | LR: 2.82e-04 | 3109.2ms/step | 19.3 samples/s +Step 23560/50462 | Loss: 2.5887 | LR: 2.82e-04 | 3106.5ms/step | 19.3 samples/s +Step 23570/50462 | Loss: 2.6128 | LR: 2.82e-04 | 3102.7ms/step | 19.3 samples/s +Step 23580/50462 | Loss: 2.5735 | LR: 2.82e-04 | 3108.0ms/step | 19.3 samples/s +Step 23590/50462 | Loss: 2.5935 | LR: 2.82e-04 | 3108.0ms/step | 19.3 samples/s +Step 23600/50462 | Loss: 2.6240 | LR: 2.82e-04 | 3106.0ms/step | 19.3 samples/s +Step 23610/50462 | Loss: 2.6104 | LR: 2.81e-04 | 3102.6ms/step | 19.3 samples/s +Step 23620/50462 | Loss: 2.6237 | LR: 2.81e-04 | 3103.6ms/step | 19.3 samples/s +Step 23630/50462 | Loss: 2.5919 | LR: 2.81e-04 | 3101.8ms/step | 19.3 samples/s +Step 23640/50462 | Loss: 2.6146 | LR: 2.81e-04 | 3103.7ms/step | 19.3 samples/s +Step 23650/50462 | Loss: 2.6116 | LR: 2.81e-04 | 3105.7ms/step | 19.3 samples/s +Step 23660/50462 | Loss: 2.5829 | LR: 2.81e-04 | 3103.5ms/step | 19.3 samples/s +Step 23670/50462 | Loss: 2.6102 | LR: 2.81e-04 | 3106.2ms/step | 19.3 samples/s +Step 23680/50462 | Loss: 2.5877 | LR: 2.80e-04 | 3102.2ms/step | 19.3 samples/s +Step 23690/50462 | Loss: 2.5873 | LR: 2.80e-04 | 3104.5ms/step | 19.3 samples/s +Step 23700/50462 | Loss: 2.5977 | LR: 2.80e-04 | 3101.0ms/step | 19.3 samples/s +Step 23710/50462 | Loss: 2.6308 | LR: 2.80e-04 | 3102.0ms/step | 19.3 samples/s +Step 23720/50462 | Loss: 2.5543 | LR: 2.80e-04 | 3104.3ms/step | 19.3 samples/s +Step 23730/50462 | Loss: 2.5921 | LR: 2.80e-04 | 3105.7ms/step | 19.3 samples/s +Step 23740/50462 | Loss: 2.6166 | LR: 2.79e-04 | 3099.3ms/step | 19.4 samples/s +Step 23750/50462 | Loss: 2.5868 | LR: 2.79e-04 | 3101.9ms/step | 19.3 samples/s +Validation at step 23750: Loss = 2.6140, Perplexity = 13.65 +Step 23760/50462 | Loss: 2.6336 | LR: 2.79e-04 | 4753.7ms/step | 12.6 samples/s +Step 23770/50462 | Loss: 2.5836 | LR: 2.79e-04 | 3103.0ms/step | 19.3 samples/s +Step 23780/50462 | Loss: 2.5801 | LR: 2.79e-04 | 3101.7ms/step | 19.3 samples/s +Step 23790/50462 | Loss: 2.5657 | LR: 2.79e-04 | 3109.8ms/step | 19.3 samples/s +Step 23800/50462 | Loss: 2.5982 | LR: 2.78e-04 | 3104.3ms/step | 19.3 samples/s +Step 23810/50462 | Loss: 2.5944 | LR: 2.78e-04 | 3104.2ms/step | 19.3 samples/s +Step 23820/50462 | Loss: 2.6045 | LR: 2.78e-04 | 3103.2ms/step | 19.3 samples/s +Step 23830/50462 | Loss: 2.5706 | LR: 2.78e-04 | 3108.9ms/step | 19.3 samples/s +Step 23840/50462 | Loss: 2.6011 | LR: 2.78e-04 | 3102.7ms/step | 19.3 samples/s +Step 23850/50462 | Loss: 2.6147 | LR: 2.78e-04 | 3104.4ms/step | 19.3 samples/s +Step 23860/50462 | Loss: 2.6213 | LR: 2.78e-04 | 3106.5ms/step | 19.3 samples/s +Step 23870/50462 | Loss: 2.6158 | LR: 2.77e-04 | 3108.7ms/step | 19.3 samples/s +Step 23880/50462 | Loss: 2.6240 | LR: 2.77e-04 | 3106.4ms/step | 19.3 samples/s +Step 23890/50462 | Loss: 2.6004 | LR: 2.77e-04 | 3105.8ms/step | 19.3 samples/s +Step 23900/50462 | Loss: 2.6077 | LR: 2.77e-04 | 3108.7ms/step | 19.3 samples/s +Step 23910/50462 | Loss: 2.5922 | LR: 2.77e-04 | 3109.3ms/step | 19.3 samples/s +Step 23920/50462 | Loss: 2.5884 | LR: 2.77e-04 | 3110.1ms/step | 19.3 samples/s +Step 23930/50462 | Loss: 2.6044 | LR: 2.76e-04 | 3113.3ms/step | 19.3 samples/s +Step 23940/50462 | Loss: 2.5966 | LR: 2.76e-04 | 3110.8ms/step | 19.3 samples/s +Step 23950/50462 | Loss: 2.5988 | LR: 2.76e-04 | 3105.3ms/step | 19.3 samples/s +Step 23960/50462 | Loss: 2.5969 | LR: 2.76e-04 | 3109.7ms/step | 19.3 samples/s +Step 23970/50462 | Loss: 2.5676 | LR: 2.76e-04 | 3106.9ms/step | 19.3 samples/s +Step 23980/50462 | Loss: 2.6302 | LR: 2.76e-04 | 3110.9ms/step | 19.3 samples/s +Step 23990/50462 | Loss: 2.6071 | LR: 2.75e-04 | 3108.3ms/step | 19.3 samples/s +Step 24000/50462 | Loss: 2.6070 | LR: 2.75e-04 | 3109.8ms/step | 19.3 samples/s +Validation at step 24000: Loss = 2.6123, Perplexity = 13.63 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-24000 +Step 24010/50462 | Loss: 2.5964 | LR: 2.75e-04 | 5726.8ms/step | 10.5 samples/s +Step 24020/50462 | Loss: 2.5954 | LR: 2.75e-04 | 3097.8ms/step | 19.4 samples/s +Step 24030/50462 | Loss: 2.5720 | LR: 2.75e-04 | 3105.3ms/step | 19.3 samples/s +Step 24040/50462 | Loss: 2.6176 | LR: 2.75e-04 | 3107.4ms/step | 19.3 samples/s +Step 24050/50462 | Loss: 2.5789 | LR: 2.75e-04 | 3106.0ms/step | 19.3 samples/s +Step 24060/50462 | Loss: 2.6023 | LR: 2.74e-04 | 3098.8ms/step | 19.4 samples/s +Step 24070/50462 | Loss: 2.6158 | LR: 2.74e-04 | 3104.5ms/step | 19.3 samples/s +Step 24080/50462 | Loss: 2.6189 | LR: 2.74e-04 | 3107.1ms/step | 19.3 samples/s +Step 24090/50462 | Loss: 2.5969 | LR: 2.74e-04 | 3109.3ms/step | 19.3 samples/s +Step 24100/50462 | Loss: 2.5747 | LR: 2.74e-04 | 3111.3ms/step | 19.3 samples/s +Step 24110/50462 | Loss: 2.5985 | LR: 2.74e-04 | 3112.1ms/step | 19.3 samples/s +Step 24120/50462 | Loss: 2.6130 | LR: 2.73e-04 | 3109.9ms/step | 19.3 samples/s +Step 24130/50462 | Loss: 2.6056 | LR: 2.73e-04 | 3109.9ms/step | 19.3 samples/s +Step 24140/50462 | Loss: 2.6034 | LR: 2.73e-04 | 3116.8ms/step | 19.3 samples/s +Step 24150/50462 | Loss: 2.5972 | LR: 2.73e-04 | 3107.9ms/step | 19.3 samples/s +Step 24160/50462 | Loss: 2.6113 | LR: 2.73e-04 | 3102.1ms/step | 19.3 samples/s +Step 24170/50462 | Loss: 2.5726 | LR: 2.73e-04 | 3108.6ms/step | 19.3 samples/s +Step 24180/50462 | Loss: 2.5906 | LR: 2.72e-04 | 3104.4ms/step | 19.3 samples/s +Step 24190/50462 | Loss: 2.5977 | LR: 2.72e-04 | 3108.3ms/step | 19.3 samples/s +Step 24200/50462 | Loss: 2.5803 | LR: 2.72e-04 | 3116.0ms/step | 19.3 samples/s +Step 24210/50462 | Loss: 2.5957 | LR: 2.72e-04 | 3630.2ms/step | 16.5 samples/s +Step 24220/50462 | Loss: 2.5654 | LR: 2.72e-04 | 3099.4ms/step | 19.4 samples/s +Step 24230/50462 | Loss: 2.6161 | LR: 2.72e-04 | 3106.7ms/step | 19.3 samples/s +Step 24240/50462 | Loss: 2.6034 | LR: 2.72e-04 | 3108.4ms/step | 19.3 samples/s +Step 24250/50462 | Loss: 2.5904 | LR: 2.71e-04 | 3106.1ms/step | 19.3 samples/s +Validation at step 24250: Loss = 2.6096, Perplexity = 13.59 +Step 24260/50462 | Loss: 2.5972 | LR: 2.71e-04 | 4762.7ms/step | 12.6 samples/s +Step 24270/50462 | Loss: 2.5980 | LR: 2.71e-04 | 3108.7ms/step | 19.3 samples/s +Step 24280/50462 | Loss: 2.6086 | LR: 2.71e-04 | 3106.0ms/step | 19.3 samples/s +Step 24290/50462 | Loss: 2.5880 | LR: 2.71e-04 | 3107.6ms/step | 19.3 samples/s +Step 24300/50462 | Loss: 2.5808 | LR: 2.71e-04 | 3108.0ms/step | 19.3 samples/s +Step 24310/50462 | Loss: 2.5995 | LR: 2.70e-04 | 3104.0ms/step | 19.3 samples/s +Step 24320/50462 | Loss: 2.5982 | LR: 2.70e-04 | 3637.6ms/step | 16.5 samples/s +Step 24330/50462 | Loss: 2.6213 | LR: 2.70e-04 | 3104.1ms/step | 19.3 samples/s +Step 24340/50462 | Loss: 2.5655 | LR: 2.70e-04 | 3104.9ms/step | 19.3 samples/s +Step 24350/50462 | Loss: 2.6018 | LR: 2.70e-04 | 3100.3ms/step | 19.4 samples/s +Step 24360/50462 | Loss: 2.5909 | LR: 2.70e-04 | 3104.2ms/step | 19.3 samples/s +Step 24370/50462 | Loss: 2.5966 | LR: 2.70e-04 | 3104.4ms/step | 19.3 samples/s +Step 24380/50462 | Loss: 2.6195 | LR: 2.69e-04 | 3101.6ms/step | 19.3 samples/s +Step 24390/50462 | Loss: 2.5787 | LR: 2.69e-04 | 3104.7ms/step | 19.3 samples/s +Step 24400/50462 | Loss: 2.6009 | LR: 2.69e-04 | 3109.0ms/step | 19.3 samples/s +Step 24410/50462 | Loss: 2.5988 | LR: 2.69e-04 | 3634.5ms/step | 16.5 samples/s +Step 24420/50462 | Loss: 2.5958 | LR: 2.69e-04 | 3104.4ms/step | 19.3 samples/s +Step 24430/50462 | Loss: 2.5724 | LR: 2.69e-04 | 3107.5ms/step | 19.3 samples/s +Step 24440/50462 | Loss: 2.6356 | LR: 2.68e-04 | 3107.8ms/step | 19.3 samples/s +Step 24450/50462 | Loss: 2.6079 | LR: 2.68e-04 | 3110.7ms/step | 19.3 samples/s +Step 24460/50462 | Loss: 2.5826 | LR: 2.68e-04 | 3111.6ms/step | 19.3 samples/s +Step 24470/50462 | Loss: 2.5779 | LR: 2.68e-04 | 3106.9ms/step | 19.3 samples/s +Step 24480/50462 | Loss: 2.5907 | LR: 2.68e-04 | 3103.2ms/step | 19.3 samples/s +Step 24490/50462 | Loss: 2.6143 | LR: 2.68e-04 | 3105.7ms/step | 19.3 samples/s +Step 24500/50462 | Loss: 2.6083 | LR: 2.67e-04 | 3102.9ms/step | 19.3 samples/s +Validation at step 24500: Loss = 2.6075, Perplexity = 13.57 +Step 24510/50462 | Loss: 2.5933 | LR: 2.67e-04 | 4752.9ms/step | 12.6 samples/s +Step 24520/50462 | Loss: 2.6171 | LR: 2.67e-04 | 3101.3ms/step | 19.3 samples/s +Step 24530/50462 | Loss: 2.5954 | LR: 2.67e-04 | 3103.7ms/step | 19.3 samples/s +Step 24540/50462 | Loss: 2.6215 | LR: 2.67e-04 | 3102.7ms/step | 19.3 samples/s +Step 24550/50462 | Loss: 2.5953 | LR: 2.67e-04 | 3109.2ms/step | 19.3 samples/s +Step 24560/50462 | Loss: 2.6455 | LR: 2.67e-04 | 3106.8ms/step | 19.3 samples/s +Step 24570/50462 | Loss: 2.5964 | LR: 2.66e-04 | 3102.3ms/step | 19.3 samples/s +Step 24580/50462 | Loss: 2.6065 | LR: 2.66e-04 | 3109.1ms/step | 19.3 samples/s +Step 24590/50462 | Loss: 2.6106 | LR: 2.66e-04 | 3111.4ms/step | 19.3 samples/s +Step 24600/50462 | Loss: 2.6143 | LR: 2.66e-04 | 3101.2ms/step | 19.3 samples/s +Step 24610/50462 | Loss: 2.5799 | LR: 2.66e-04 | 3097.7ms/step | 19.4 samples/s +Step 24620/50462 | Loss: 2.6224 | LR: 2.66e-04 | 3096.1ms/step | 19.4 samples/s +Step 24630/50462 | Loss: 2.5687 | LR: 2.65e-04 | 3094.3ms/step | 19.4 samples/s +Step 24640/50462 | Loss: 2.6119 | LR: 2.65e-04 | 3101.5ms/step | 19.3 samples/s +Step 24650/50462 | Loss: 2.5707 | LR: 2.65e-04 | 3105.6ms/step | 19.3 samples/s +Step 24660/50462 | Loss: 2.6118 | LR: 2.65e-04 | 3106.8ms/step | 19.3 samples/s +Step 24670/50462 | Loss: 2.6011 | LR: 2.65e-04 | 3105.2ms/step | 19.3 samples/s +Step 24680/50462 | Loss: 2.6044 | LR: 2.65e-04 | 3106.5ms/step | 19.3 samples/s +Step 24690/50462 | Loss: 2.5968 | LR: 2.64e-04 | 3095.3ms/step | 19.4 samples/s +Step 24700/50462 | Loss: 2.6216 | LR: 2.64e-04 | 3098.1ms/step | 19.4 samples/s +Step 24710/50462 | Loss: 2.5942 | LR: 2.64e-04 | 3105.5ms/step | 19.3 samples/s +Step 24720/50462 | Loss: 2.5703 | LR: 2.64e-04 | 3109.4ms/step | 19.3 samples/s +Step 24730/50462 | Loss: 2.5855 | LR: 2.64e-04 | 3107.4ms/step | 19.3 samples/s +Step 24740/50462 | Loss: 2.6110 | LR: 2.64e-04 | 3104.9ms/step | 19.3 samples/s +Switching attention mode for step 24750: dense (dense step interval) +Step 24750/50462 | Loss: 2.5853 | LR: 2.64e-04 | 3317.8ms/step | 18.1 samples/s +Validation at step 24750: Loss = 2.5849, Perplexity = 13.26 +Step 24760/50462 | Loss: 2.5874 | LR: 2.63e-04 | 7765.3ms/step | 7.7 samples/s +Step 24770/50462 | Loss: 2.5513 | LR: 2.63e-04 | 5234.5ms/step | 11.5 samples/s +Step 24780/50462 | Loss: 2.5548 | LR: 2.63e-04 | 5246.0ms/step | 11.4 samples/s +Step 24790/50462 | Loss: 2.5657 | LR: 2.63e-04 | 5236.2ms/step | 11.5 samples/s +Step 24800/50462 | Loss: 2.5916 | LR: 2.63e-04 | 5235.1ms/step | 11.5 samples/s +Step 24810/50462 | Loss: 2.5833 | LR: 2.63e-04 | 5244.2ms/step | 11.4 samples/s +Step 24820/50462 | Loss: 2.5800 | LR: 2.62e-04 | 5244.7ms/step | 11.4 samples/s +Step 24830/50462 | Loss: 2.5728 | LR: 2.62e-04 | 5233.0ms/step | 11.5 samples/s +Step 24840/50462 | Loss: 2.5456 | LR: 2.62e-04 | 5244.3ms/step | 11.4 samples/s +Step 24850/50462 | Loss: 2.5597 | LR: 2.62e-04 | 5232.1ms/step | 11.5 samples/s +Step 24860/50462 | Loss: 2.5882 | LR: 2.62e-04 | 5238.5ms/step | 11.5 samples/s +Step 24870/50462 | Loss: 2.5760 | LR: 2.62e-04 | 5232.1ms/step | 11.5 samples/s +Step 24880/50462 | Loss: 2.5219 | LR: 2.61e-04 | 5227.6ms/step | 11.5 samples/s +Step 24890/50462 | Loss: 2.5373 | LR: 2.61e-04 | 5231.6ms/step | 11.5 samples/s +Step 24900/50462 | Loss: 2.5919 | LR: 2.61e-04 | 5238.0ms/step | 11.5 samples/s +Step 24910/50462 | Loss: 2.5321 | LR: 2.61e-04 | 5238.9ms/step | 11.5 samples/s +Step 24920/50462 | Loss: 2.5398 | LR: 2.61e-04 | 5237.1ms/step | 11.5 samples/s +Step 24930/50462 | Loss: 2.5529 | LR: 2.61e-04 | 5215.7ms/step | 11.5 samples/s +Step 24940/50462 | Loss: 2.5863 | LR: 2.61e-04 | 5226.3ms/step | 11.5 samples/s +Step 24950/50462 | Loss: 2.5793 | LR: 2.60e-04 | 5219.7ms/step | 11.5 samples/s +Step 24960/50462 | Loss: 2.5368 | LR: 2.60e-04 | 5242.8ms/step | 11.4 samples/s +Step 24970/50462 | Loss: 2.5382 | LR: 2.60e-04 | 5198.2ms/step | 11.5 samples/s +Step 24980/50462 | Loss: 2.5744 | LR: 2.60e-04 | 5202.9ms/step | 11.5 samples/s +Step 24990/50462 | Loss: 2.5535 | LR: 2.60e-04 | 5197.2ms/step | 11.5 samples/s +Step 25000/50462 | Loss: 2.5737 | LR: 2.60e-04 | 5213.1ms/step | 11.5 samples/s +Validation at step 25000: Loss = 2.5760, Perplexity = 13.14 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-25000 +Switching attention mode for step 25001: sparse_stochastic (configured sparse attention) +Step 25010/50462 | Loss: 2.5891 | LR: 2.59e-04 | 6621.4ms/step | 9.1 samples/s +Step 25020/50462 | Loss: 2.5920 | LR: 2.59e-04 | 3100.4ms/step | 19.4 samples/s +Step 25030/50462 | Loss: 2.5802 | LR: 2.59e-04 | 3110.9ms/step | 19.3 samples/s +Step 25040/50462 | Loss: 2.6011 | LR: 2.59e-04 | 3113.3ms/step | 19.3 samples/s +Step 25050/50462 | Loss: 2.6115 | LR: 2.59e-04 | 3110.9ms/step | 19.3 samples/s +Step 25060/50462 | Loss: 2.5916 | LR: 2.59e-04 | 3103.0ms/step | 19.3 samples/s +Step 25070/50462 | Loss: 2.5750 | LR: 2.58e-04 | 3108.3ms/step | 19.3 samples/s +Step 25080/50462 | Loss: 2.5650 | LR: 2.58e-04 | 3109.6ms/step | 19.3 samples/s +Step 25090/50462 | Loss: 2.5858 | LR: 2.58e-04 | 3104.2ms/step | 19.3 samples/s +Step 25100/50462 | Loss: 2.5931 | LR: 2.58e-04 | 3106.3ms/step | 19.3 samples/s +Step 25110/50462 | Loss: 2.5842 | LR: 2.58e-04 | 3108.2ms/step | 19.3 samples/s +Step 25120/50462 | Loss: 2.6087 | LR: 2.58e-04 | 3113.6ms/step | 19.3 samples/s +Step 25130/50462 | Loss: 2.6132 | LR: 2.58e-04 | 3117.6ms/step | 19.2 samples/s +Step 25140/50462 | Loss: 2.5984 | LR: 2.57e-04 | 3113.7ms/step | 19.3 samples/s +Step 25150/50462 | Loss: 2.5684 | LR: 2.57e-04 | 3119.1ms/step | 19.2 samples/s +Step 25160/50462 | Loss: 2.5684 | LR: 2.57e-04 | 3112.7ms/step | 19.3 samples/s +Step 25170/50462 | Loss: 2.5834 | LR: 2.57e-04 | 3116.9ms/step | 19.3 samples/s +Step 25180/50462 | Loss: 2.5866 | LR: 2.57e-04 | 3115.9ms/step | 19.3 samples/s +Step 25190/50462 | Loss: 2.5808 | LR: 2.57e-04 | 3117.4ms/step | 19.2 samples/s +Step 25200/50462 | Loss: 2.6141 | LR: 2.56e-04 | 3116.9ms/step | 19.2 samples/s +Step 25210/50462 | Loss: 2.5953 | LR: 2.56e-04 | 3114.0ms/step | 19.3 samples/s +Step 25220/50462 | Loss: 2.6323 | LR: 2.56e-04 | 3115.9ms/step | 19.3 samples/s +Step 25230/50462 | Loss: 2.6128 | LR: 2.56e-04 | 3115.0ms/step | 19.3 samples/s +Step 25240/50462 | Loss: 2.5598 | LR: 2.56e-04 | 3118.6ms/step | 19.2 samples/s +Step 25250/50462 | Loss: 2.5952 | LR: 2.56e-04 | 3115.7ms/step | 19.3 samples/s +Validation at step 25250: Loss = 2.6000, Perplexity = 13.46 +Step 25260/50462 | Loss: 2.5699 | LR: 2.55e-04 | 4780.2ms/step | 12.6 samples/s +Step 25270/50462 | Loss: 2.5940 | LR: 2.55e-04 | 3112.8ms/step | 19.3 samples/s +Step 25280/50462 | Loss: 2.6230 | LR: 2.55e-04 | 3113.5ms/step | 19.3 samples/s +Step 25290/50462 | Loss: 2.5919 | LR: 2.55e-04 | 3114.2ms/step | 19.3 samples/s +Step 25300/50462 | Loss: 2.5758 | LR: 2.55e-04 | 3116.1ms/step | 19.3 samples/s +Step 25310/50462 | Loss: 2.5645 | LR: 2.55e-04 | 3116.1ms/step | 19.3 samples/s +Step 25320/50462 | Loss: 2.5990 | LR: 2.55e-04 | 3116.0ms/step | 19.3 samples/s +Step 25330/50462 | Loss: 2.5730 | LR: 2.54e-04 | 3115.8ms/step | 19.3 samples/s +Step 25340/50462 | Loss: 2.6056 | LR: 2.54e-04 | 3115.0ms/step | 19.3 samples/s +Step 25350/50462 | Loss: 2.5455 | LR: 2.54e-04 | 3114.6ms/step | 19.3 samples/s +Step 25360/50462 | Loss: 2.5968 | LR: 2.54e-04 | 3114.8ms/step | 19.3 samples/s +Step 25370/50462 | Loss: 2.5980 | LR: 2.54e-04 | 3115.1ms/step | 19.3 samples/s +Step 25380/50462 | Loss: 2.6390 | LR: 2.54e-04 | 3115.2ms/step | 19.3 samples/s +Step 25390/50462 | Loss: 2.6009 | LR: 2.53e-04 | 3119.6ms/step | 19.2 samples/s +Step 25400/50462 | Loss: 2.5750 | LR: 2.53e-04 | 3118.0ms/step | 19.2 samples/s +Step 25410/50462 | Loss: 2.5688 | LR: 2.53e-04 | 3115.1ms/step | 19.3 samples/s +Step 25420/50462 | Loss: 2.5947 | LR: 2.53e-04 | 3115.4ms/step | 19.3 samples/s +Step 25430/50462 | Loss: 2.5841 | LR: 2.53e-04 | 3114.1ms/step | 19.3 samples/s +Step 25440/50462 | Loss: 2.5759 | LR: 2.53e-04 | 3112.4ms/step | 19.3 samples/s +Step 25450/50462 | Loss: 2.5803 | LR: 2.52e-04 | 3110.9ms/step | 19.3 samples/s +Step 25460/50462 | Loss: 2.5922 | LR: 2.52e-04 | 3115.0ms/step | 19.3 samples/s +Step 25470/50462 | Loss: 2.5916 | LR: 2.52e-04 | 3112.9ms/step | 19.3 samples/s +Step 25480/50462 | Loss: 2.5764 | LR: 2.52e-04 | 3116.5ms/step | 19.3 samples/s +Step 25490/50462 | Loss: 2.5791 | LR: 2.52e-04 | 3115.0ms/step | 19.3 samples/s +Step 25500/50462 | Loss: 2.5810 | LR: 2.52e-04 | 3118.7ms/step | 19.2 samples/s +Validation at step 25500: Loss = 2.5985, Perplexity = 13.44 +Step 25510/50462 | Loss: 2.5755 | LR: 2.52e-04 | 4774.5ms/step | 12.6 samples/s +Step 25520/50462 | Loss: 2.5830 | LR: 2.51e-04 | 3111.8ms/step | 19.3 samples/s +Step 25530/50462 | Loss: 2.6111 | LR: 2.51e-04 | 3118.1ms/step | 19.2 samples/s +Step 25540/50462 | Loss: 2.6062 | LR: 2.51e-04 | 3114.7ms/step | 19.3 samples/s +Step 25550/50462 | Loss: 2.5994 | LR: 2.51e-04 | 3115.6ms/step | 19.3 samples/s +Step 25560/50462 | Loss: 2.5976 | LR: 2.51e-04 | 3117.3ms/step | 19.2 samples/s +Step 25570/50462 | Loss: 2.6006 | LR: 2.51e-04 | 3117.4ms/step | 19.2 samples/s +Step 25580/50462 | Loss: 2.6237 | LR: 2.50e-04 | 3115.3ms/step | 19.3 samples/s +Step 25590/50462 | Loss: 2.6147 | LR: 2.50e-04 | 3115.3ms/step | 19.3 samples/s +Step 25600/50462 | Loss: 2.5814 | LR: 2.50e-04 | 3116.7ms/step | 19.3 samples/s +Step 25610/50462 | Loss: 2.5987 | LR: 2.50e-04 | 3113.2ms/step | 19.3 samples/s +Step 25620/50462 | Loss: 2.5849 | LR: 2.50e-04 | 3117.2ms/step | 19.2 samples/s +Step 25630/50462 | Loss: 2.5798 | LR: 2.50e-04 | 3120.7ms/step | 19.2 samples/s +Step 25640/50462 | Loss: 2.6129 | LR: 2.49e-04 | 3114.4ms/step | 19.3 samples/s +Step 25650/50462 | Loss: 2.5745 | LR: 2.49e-04 | 3116.1ms/step | 19.3 samples/s +Step 25660/50462 | Loss: 2.5521 | LR: 2.49e-04 | 3118.9ms/step | 19.2 samples/s +Step 25670/50462 | Loss: 2.5789 | LR: 2.49e-04 | 3116.1ms/step | 19.3 samples/s +Step 25680/50462 | Loss: 2.5968 | LR: 2.49e-04 | 3116.5ms/step | 19.3 samples/s +Step 25690/50462 | Loss: 2.5992 | LR: 2.49e-04 | 3118.3ms/step | 19.2 samples/s +Step 25700/50462 | Loss: 2.5891 | LR: 2.49e-04 | 3118.5ms/step | 19.2 samples/s +Step 25710/50462 | Loss: 2.5650 | LR: 2.48e-04 | 3114.3ms/step | 19.3 samples/s +Step 25720/50462 | Loss: 2.5672 | LR: 2.48e-04 | 3117.1ms/step | 19.2 samples/s +Step 25730/50462 | Loss: 2.5992 | LR: 2.48e-04 | 3114.6ms/step | 19.3 samples/s +Step 25740/50462 | Loss: 2.5948 | LR: 2.48e-04 | 3116.8ms/step | 19.3 samples/s +Step 25750/50462 | Loss: 2.6044 | LR: 2.48e-04 | 3118.3ms/step | 19.2 samples/s +Validation at step 25750: Loss = 2.5963, Perplexity = 13.41 +Step 25760/50462 | Loss: 2.5801 | LR: 2.48e-04 | 4783.1ms/step | 12.5 samples/s +Step 25770/50462 | Loss: 2.5853 | LR: 2.47e-04 | 3118.7ms/step | 19.2 samples/s +Step 25780/50462 | Loss: 2.6116 | LR: 2.47e-04 | 3115.3ms/step | 19.3 samples/s +Step 25790/50462 | Loss: 2.5822 | LR: 2.47e-04 | 3115.3ms/step | 19.3 samples/s +Step 25800/50462 | Loss: 2.5716 | LR: 2.47e-04 | 3113.3ms/step | 19.3 samples/s +Step 25810/50462 | Loss: 2.5825 | LR: 2.47e-04 | 3114.4ms/step | 19.3 samples/s +Step 25820/50462 | Loss: 2.5849 | LR: 2.47e-04 | 3113.6ms/step | 19.3 samples/s +Step 25830/50462 | Loss: 2.5977 | LR: 2.46e-04 | 3113.2ms/step | 19.3 samples/s +Step 25840/50462 | Loss: 2.6163 | LR: 2.46e-04 | 3114.0ms/step | 19.3 samples/s +Step 25850/50462 | Loss: 2.5867 | LR: 2.46e-04 | 3117.7ms/step | 19.2 samples/s +Step 25860/50462 | Loss: 2.5758 | LR: 2.46e-04 | 3115.5ms/step | 19.3 samples/s +Step 25870/50462 | Loss: 2.5926 | LR: 2.46e-04 | 3113.2ms/step | 19.3 samples/s +Step 25880/50462 | Loss: 2.5965 | LR: 2.46e-04 | 3116.3ms/step | 19.3 samples/s +Step 25890/50462 | Loss: 2.5678 | LR: 2.46e-04 | 3115.7ms/step | 19.3 samples/s +Step 25900/50462 | Loss: 2.5581 | LR: 2.45e-04 | 3118.1ms/step | 19.2 samples/s +Step 25910/50462 | Loss: 2.5608 | LR: 2.45e-04 | 3116.0ms/step | 19.3 samples/s +Step 25920/50462 | Loss: 2.5932 | LR: 2.45e-04 | 3113.5ms/step | 19.3 samples/s +Step 25930/50462 | Loss: 2.5840 | LR: 2.45e-04 | 3111.4ms/step | 19.3 samples/s +Step 25940/50462 | Loss: 2.5797 | LR: 2.45e-04 | 3111.5ms/step | 19.3 samples/s +Step 25950/50462 | Loss: 2.6057 | LR: 2.45e-04 | 3112.8ms/step | 19.3 samples/s +Step 25960/50462 | Loss: 2.5904 | LR: 2.44e-04 | 3110.3ms/step | 19.3 samples/s +Step 25970/50462 | Loss: 2.5852 | LR: 2.44e-04 | 3112.8ms/step | 19.3 samples/s +Step 25980/50462 | Loss: 2.5694 | LR: 2.44e-04 | 3114.2ms/step | 19.3 samples/s +Step 25990/50462 | Loss: 2.5732 | LR: 2.44e-04 | 3112.0ms/step | 19.3 samples/s +Step 26000/50462 | Loss: 2.5705 | LR: 2.44e-04 | 3112.6ms/step | 19.3 samples/s +Validation at step 26000: Loss = 2.5948, Perplexity = 13.39 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-26000 +Step 26010/50462 | Loss: 2.6052 | LR: 2.44e-04 | 5729.4ms/step | 10.5 samples/s +Step 26020/50462 | Loss: 2.6049 | LR: 2.43e-04 | 3102.1ms/step | 19.3 samples/s +Step 26030/50462 | Loss: 2.5898 | LR: 2.43e-04 | 3111.8ms/step | 19.3 samples/s +Step 26040/50462 | Loss: 2.5715 | LR: 2.43e-04 | 3113.4ms/step | 19.3 samples/s +Step 26050/50462 | Loss: 2.5692 | LR: 2.43e-04 | 3113.9ms/step | 19.3 samples/s +Step 26060/50462 | Loss: 2.5736 | LR: 2.43e-04 | 3114.4ms/step | 19.3 samples/s +Step 26070/50462 | Loss: 2.6063 | LR: 2.43e-04 | 3114.1ms/step | 19.3 samples/s +Step 26080/50462 | Loss: 2.5580 | LR: 2.43e-04 | 3114.0ms/step | 19.3 samples/s +Step 26090/50462 | Loss: 2.5971 | LR: 2.42e-04 | 3114.6ms/step | 19.3 samples/s +Step 26100/50462 | Loss: 2.5980 | LR: 2.42e-04 | 3112.5ms/step | 19.3 samples/s +Step 26110/50462 | Loss: 2.6001 | LR: 2.42e-04 | 3116.2ms/step | 19.3 samples/s +Step 26120/50462 | Loss: 2.6039 | LR: 2.42e-04 | 3115.4ms/step | 19.3 samples/s +Step 26130/50462 | Loss: 2.5755 | LR: 2.42e-04 | 3115.0ms/step | 19.3 samples/s +Step 26140/50462 | Loss: 2.5876 | LR: 2.42e-04 | 3114.3ms/step | 19.3 samples/s +Step 26150/50462 | Loss: 2.5830 | LR: 2.41e-04 | 3116.2ms/step | 19.3 samples/s +Step 26160/50462 | Loss: 2.5752 | LR: 2.41e-04 | 3117.5ms/step | 19.2 samples/s +Step 26170/50462 | Loss: 2.5920 | LR: 2.41e-04 | 3124.2ms/step | 19.2 samples/s +Step 26180/50462 | Loss: 2.5848 | LR: 2.41e-04 | 3116.0ms/step | 19.3 samples/s +Step 26190/50462 | Loss: 2.5718 | LR: 2.41e-04 | 3118.4ms/step | 19.2 samples/s +Step 26200/50462 | Loss: 2.5830 | LR: 2.41e-04 | 3112.9ms/step | 19.3 samples/s +Step 26210/50462 | Loss: 2.5666 | LR: 2.40e-04 | 3113.1ms/step | 19.3 samples/s +Step 26220/50462 | Loss: 2.5753 | LR: 2.40e-04 | 3113.1ms/step | 19.3 samples/s +Step 26230/50462 | Loss: 2.5803 | LR: 2.40e-04 | 3114.1ms/step | 19.3 samples/s +Step 26240/50462 | Loss: 2.5879 | LR: 2.40e-04 | 3117.3ms/step | 19.2 samples/s +Step 26250/50462 | Loss: 2.5996 | LR: 2.40e-04 | 3117.2ms/step | 19.2 samples/s +Validation at step 26250: Loss = 2.5925, Perplexity = 13.36 +Step 26260/50462 | Loss: 2.5816 | LR: 2.40e-04 | 4770.8ms/step | 12.6 samples/s +Step 26270/50462 | Loss: 2.5509 | LR: 2.40e-04 | 3109.7ms/step | 19.3 samples/s +Step 26280/50462 | Loss: 2.5742 | LR: 2.39e-04 | 3112.4ms/step | 19.3 samples/s +Step 26290/50462 | Loss: 2.5829 | LR: 2.39e-04 | 3114.1ms/step | 19.3 samples/s +Step 26300/50462 | Loss: 2.5842 | LR: 2.39e-04 | 3116.6ms/step | 19.3 samples/s +Step 26310/50462 | Loss: 2.5759 | LR: 2.39e-04 | 3114.8ms/step | 19.3 samples/s +Step 26320/50462 | Loss: 2.5866 | LR: 2.39e-04 | 3117.1ms/step | 19.2 samples/s +Step 26330/50462 | Loss: 2.5846 | LR: 2.39e-04 | 3113.4ms/step | 19.3 samples/s +Step 26340/50462 | Loss: 2.5997 | LR: 2.38e-04 | 3625.1ms/step | 16.6 samples/s +Step 26350/50462 | Loss: 2.5992 | LR: 2.38e-04 | 3103.7ms/step | 19.3 samples/s +Step 26360/50462 | Loss: 2.5928 | LR: 2.38e-04 | 3111.1ms/step | 19.3 samples/s +Step 26370/50462 | Loss: 2.5663 | LR: 2.38e-04 | 3115.1ms/step | 19.3 samples/s +Step 26380/50462 | Loss: 2.5394 | LR: 2.38e-04 | 3113.4ms/step | 19.3 samples/s +Step 26390/50462 | Loss: 2.5786 | LR: 2.38e-04 | 3114.0ms/step | 19.3 samples/s +Step 26400/50462 | Loss: 2.5813 | LR: 2.37e-04 | 3109.6ms/step | 19.3 samples/s +Step 26410/50462 | Loss: 2.5922 | LR: 2.37e-04 | 3115.4ms/step | 19.3 samples/s +Step 26420/50462 | Loss: 2.5792 | LR: 2.37e-04 | 3113.8ms/step | 19.3 samples/s +Step 26430/50462 | Loss: 2.5762 | LR: 2.37e-04 | 3115.6ms/step | 19.3 samples/s +Step 26440/50462 | Loss: 2.5977 | LR: 2.37e-04 | 3114.2ms/step | 19.3 samples/s +Step 26450/50462 | Loss: 2.5929 | LR: 2.37e-04 | 3113.1ms/step | 19.3 samples/s +Step 26460/50462 | Loss: 2.5805 | LR: 2.37e-04 | 3110.5ms/step | 19.3 samples/s +Step 26470/50462 | Loss: 2.6192 | LR: 2.36e-04 | 3110.7ms/step | 19.3 samples/s +Step 26480/50462 | Loss: 2.5659 | LR: 2.36e-04 | 3110.8ms/step | 19.3 samples/s +Step 26490/50462 | Loss: 2.5747 | LR: 2.36e-04 | 3114.2ms/step | 19.3 samples/s +Step 26500/50462 | Loss: 2.5811 | LR: 2.36e-04 | 3109.2ms/step | 19.3 samples/s +Validation at step 26500: Loss = 2.5903, Perplexity = 13.33 +Step 26510/50462 | Loss: 2.5897 | LR: 2.36e-04 | 4763.0ms/step | 12.6 samples/s +Step 26520/50462 | Loss: 2.5884 | LR: 2.36e-04 | 3645.2ms/step | 16.5 samples/s +Step 26530/50462 | Loss: 2.5743 | LR: 2.35e-04 | 3088.2ms/step | 19.4 samples/s +Step 26540/50462 | Loss: 2.5578 | LR: 2.35e-04 | 3102.2ms/step | 19.3 samples/s +Step 26550/50462 | Loss: 2.6060 | LR: 2.35e-04 | 3110.0ms/step | 19.3 samples/s +Step 26560/50462 | Loss: 2.5992 | LR: 2.35e-04 | 3111.4ms/step | 19.3 samples/s +Step 26570/50462 | Loss: 2.5811 | LR: 2.35e-04 | 3113.6ms/step | 19.3 samples/s +Step 26580/50462 | Loss: 2.5785 | LR: 2.35e-04 | 3112.4ms/step | 19.3 samples/s +Step 26590/50462 | Loss: 2.5627 | LR: 2.34e-04 | 3112.5ms/step | 19.3 samples/s +Step 26600/50462 | Loss: 2.5632 | LR: 2.34e-04 | 3647.4ms/step | 16.4 samples/s +Step 26610/50462 | Loss: 2.6091 | LR: 2.34e-04 | 3092.2ms/step | 19.4 samples/s +Step 26620/50462 | Loss: 2.5759 | LR: 2.34e-04 | 3105.9ms/step | 19.3 samples/s +Step 26630/50462 | Loss: 2.5819 | LR: 2.34e-04 | 3108.5ms/step | 19.3 samples/s +Step 26640/50462 | Loss: 2.5874 | LR: 2.34e-04 | 3109.9ms/step | 19.3 samples/s +Step 26650/50462 | Loss: 2.5727 | LR: 2.34e-04 | 3108.9ms/step | 19.3 samples/s +Step 26660/50462 | Loss: 2.5483 | LR: 2.33e-04 | 3112.3ms/step | 19.3 samples/s +Step 26670/50462 | Loss: 2.5466 | LR: 2.33e-04 | 3109.4ms/step | 19.3 samples/s +Step 26680/50462 | Loss: 2.5417 | LR: 2.33e-04 | 3110.8ms/step | 19.3 samples/s +Step 26690/50462 | Loss: 2.5644 | LR: 2.33e-04 | 3113.3ms/step | 19.3 samples/s +Step 26700/50462 | Loss: 2.5806 | LR: 2.33e-04 | 3112.8ms/step | 19.3 samples/s +Step 26710/50462 | Loss: 2.5857 | LR: 2.33e-04 | 3111.6ms/step | 19.3 samples/s +Step 26720/50462 | Loss: 2.5306 | LR: 2.32e-04 | 3110.5ms/step | 19.3 samples/s +Step 26730/50462 | Loss: 2.5625 | LR: 2.32e-04 | 3107.7ms/step | 19.3 samples/s +Step 26740/50462 | Loss: 2.5794 | LR: 2.32e-04 | 3112.9ms/step | 19.3 samples/s +Step 26750/50462 | Loss: 2.5906 | LR: 2.32e-04 | 3115.3ms/step | 19.3 samples/s +Validation at step 26750: Loss = 2.5889, Perplexity = 13.32 +Step 26760/50462 | Loss: 2.5861 | LR: 2.32e-04 | 4775.2ms/step | 12.6 samples/s +Step 26770/50462 | Loss: 2.5591 | LR: 2.32e-04 | 3108.9ms/step | 19.3 samples/s +Step 26780/50462 | Loss: 2.5495 | LR: 2.31e-04 | 3110.7ms/step | 19.3 samples/s +Step 26790/50462 | Loss: 2.6153 | LR: 2.31e-04 | 3110.6ms/step | 19.3 samples/s +Step 26800/50462 | Loss: 2.6015 | LR: 2.31e-04 | 3108.4ms/step | 19.3 samples/s +Step 26810/50462 | Loss: 2.6136 | LR: 2.31e-04 | 3106.9ms/step | 19.3 samples/s +Step 26820/50462 | Loss: 2.5676 | LR: 2.31e-04 | 3112.1ms/step | 19.3 samples/s +Step 26830/50462 | Loss: 2.6129 | LR: 2.31e-04 | 3115.7ms/step | 19.3 samples/s +Step 26840/50462 | Loss: 2.5391 | LR: 2.31e-04 | 3110.9ms/step | 19.3 samples/s +Step 26850/50462 | Loss: 2.6054 | LR: 2.30e-04 | 3111.5ms/step | 19.3 samples/s +Step 26860/50462 | Loss: 2.5857 | LR: 2.30e-04 | 3111.6ms/step | 19.3 samples/s +Step 26870/50462 | Loss: 2.5659 | LR: 2.30e-04 | 3110.1ms/step | 19.3 samples/s +Step 26880/50462 | Loss: 2.5583 | LR: 2.30e-04 | 3113.2ms/step | 19.3 samples/s +Step 26890/50462 | Loss: 2.5665 | LR: 2.30e-04 | 3114.3ms/step | 19.3 samples/s +Step 26900/50462 | Loss: 2.5730 | LR: 2.30e-04 | 3109.7ms/step | 19.3 samples/s +Step 26910/50462 | Loss: 2.5866 | LR: 2.29e-04 | 3103.9ms/step | 19.3 samples/s +Step 26920/50462 | Loss: 2.6137 | LR: 2.29e-04 | 3110.6ms/step | 19.3 samples/s +Step 26930/50462 | Loss: 2.5700 | LR: 2.29e-04 | 3112.1ms/step | 19.3 samples/s +Step 26940/50462 | Loss: 2.5726 | LR: 2.29e-04 | 3113.4ms/step | 19.3 samples/s +Step 26950/50462 | Loss: 2.5731 | LR: 2.29e-04 | 3113.8ms/step | 19.3 samples/s +Step 26960/50462 | Loss: 2.5750 | LR: 2.29e-04 | 3114.0ms/step | 19.3 samples/s +Step 26970/50462 | Loss: 2.5827 | LR: 2.28e-04 | 3114.8ms/step | 19.3 samples/s +Step 26980/50462 | Loss: 2.5473 | LR: 2.28e-04 | 3111.3ms/step | 19.3 samples/s +Step 26990/50462 | Loss: 2.5873 | LR: 2.28e-04 | 3112.5ms/step | 19.3 samples/s +Step 27000/50462 | Loss: 2.5930 | LR: 2.28e-04 | 3114.0ms/step | 19.3 samples/s +Validation at step 27000: Loss = 2.5870, Perplexity = 13.29 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-27000 +Step 27010/50462 | Loss: 2.5548 | LR: 2.28e-04 | 5699.4ms/step | 10.5 samples/s +Step 27020/50462 | Loss: 2.5906 | LR: 2.28e-04 | 3096.7ms/step | 19.4 samples/s +Step 27030/50462 | Loss: 2.6014 | LR: 2.28e-04 | 3106.7ms/step | 19.3 samples/s +Step 27040/50462 | Loss: 2.5877 | LR: 2.27e-04 | 3112.9ms/step | 19.3 samples/s +Step 27050/50462 | Loss: 2.5793 | LR: 2.27e-04 | 3114.4ms/step | 19.3 samples/s +Step 27060/50462 | Loss: 2.5719 | LR: 2.27e-04 | 3116.0ms/step | 19.3 samples/s +Step 27070/50462 | Loss: 2.5740 | LR: 2.27e-04 | 3115.5ms/step | 19.3 samples/s +Step 27080/50462 | Loss: 2.5821 | LR: 2.27e-04 | 3114.8ms/step | 19.3 samples/s +Step 27090/50462 | Loss: 2.5650 | LR: 2.27e-04 | 3115.4ms/step | 19.3 samples/s +Step 27100/50462 | Loss: 2.5818 | LR: 2.26e-04 | 3113.6ms/step | 19.3 samples/s +Step 27110/50462 | Loss: 2.5730 | LR: 2.26e-04 | 3111.2ms/step | 19.3 samples/s +Step 27120/50462 | Loss: 2.5839 | LR: 2.26e-04 | 3111.4ms/step | 19.3 samples/s +Step 27130/50462 | Loss: 2.5975 | LR: 2.26e-04 | 3112.7ms/step | 19.3 samples/s +Step 27140/50462 | Loss: 2.5580 | LR: 2.26e-04 | 3119.9ms/step | 19.2 samples/s +Step 27150/50462 | Loss: 2.5663 | LR: 2.26e-04 | 3115.6ms/step | 19.3 samples/s +Step 27160/50462 | Loss: 2.5731 | LR: 2.25e-04 | 3116.6ms/step | 19.3 samples/s +Step 27170/50462 | Loss: 2.6028 | LR: 2.25e-04 | 3116.6ms/step | 19.3 samples/s +Step 27180/50462 | Loss: 2.5867 | LR: 2.25e-04 | 3113.2ms/step | 19.3 samples/s +Step 27190/50462 | Loss: 2.5925 | LR: 2.25e-04 | 3115.7ms/step | 19.3 samples/s +Step 27200/50462 | Loss: 2.5380 | LR: 2.25e-04 | 3114.3ms/step | 19.3 samples/s +Step 27210/50462 | Loss: 2.5769 | LR: 2.25e-04 | 3113.6ms/step | 19.3 samples/s +Step 27220/50462 | Loss: 2.5630 | LR: 2.25e-04 | 3115.5ms/step | 19.3 samples/s +Step 27230/50462 | Loss: 2.6075 | LR: 2.24e-04 | 3114.6ms/step | 19.3 samples/s +Step 27240/50462 | Loss: 2.5775 | LR: 2.24e-04 | 3115.8ms/step | 19.3 samples/s +Step 27250/50462 | Loss: 2.5921 | LR: 2.24e-04 | 3115.2ms/step | 19.3 samples/s +Validation at step 27250: Loss = 2.5856, Perplexity = 13.27 +Step 27260/50462 | Loss: 2.5798 | LR: 2.24e-04 | 4771.2ms/step | 12.6 samples/s +Step 27270/50462 | Loss: 2.5855 | LR: 2.24e-04 | 3114.2ms/step | 19.3 samples/s +Step 27280/50462 | Loss: 2.5571 | LR: 2.24e-04 | 3111.9ms/step | 19.3 samples/s +Step 27290/50462 | Loss: 2.5789 | LR: 2.23e-04 | 3112.8ms/step | 19.3 samples/s +Step 27300/50462 | Loss: 2.5387 | LR: 2.23e-04 | 3112.9ms/step | 19.3 samples/s +Step 27310/50462 | Loss: 2.5331 | LR: 2.23e-04 | 3114.2ms/step | 19.3 samples/s +Step 27320/50462 | Loss: 2.5760 | LR: 2.23e-04 | 3115.0ms/step | 19.3 samples/s +Step 27330/50462 | Loss: 2.6037 | LR: 2.23e-04 | 3116.9ms/step | 19.2 samples/s +Step 27340/50462 | Loss: 2.5657 | LR: 2.23e-04 | 3117.1ms/step | 19.2 samples/s +Step 27350/50462 | Loss: 2.5548 | LR: 2.23e-04 | 3115.7ms/step | 19.3 samples/s +Step 27360/50462 | Loss: 2.5239 | LR: 2.22e-04 | 3119.4ms/step | 19.2 samples/s +Step 27370/50462 | Loss: 2.5655 | LR: 2.22e-04 | 3115.7ms/step | 19.3 samples/s +Step 27380/50462 | Loss: 2.5568 | LR: 2.22e-04 | 3116.5ms/step | 19.3 samples/s +Step 27390/50462 | Loss: 2.5720 | LR: 2.22e-04 | 3114.6ms/step | 19.3 samples/s +Step 27400/50462 | Loss: 2.5880 | LR: 2.22e-04 | 3117.6ms/step | 19.2 samples/s +Step 27410/50462 | Loss: 2.5543 | LR: 2.22e-04 | 3115.4ms/step | 19.3 samples/s +Step 27420/50462 | Loss: 2.5554 | LR: 2.21e-04 | 3115.5ms/step | 19.3 samples/s +Step 27430/50462 | Loss: 2.5789 | LR: 2.21e-04 | 3114.9ms/step | 19.3 samples/s +Step 27440/50462 | Loss: 2.5633 | LR: 2.21e-04 | 3113.8ms/step | 19.3 samples/s +Step 27450/50462 | Loss: 2.5610 | LR: 2.21e-04 | 3113.1ms/step | 19.3 samples/s +Step 27460/50462 | Loss: 2.5962 | LR: 2.21e-04 | 3112.7ms/step | 19.3 samples/s +Step 27470/50462 | Loss: 2.5521 | LR: 2.21e-04 | 3112.6ms/step | 19.3 samples/s +Step 27480/50462 | Loss: 2.5695 | LR: 2.20e-04 | 3112.5ms/step | 19.3 samples/s +Step 27490/50462 | Loss: 2.5407 | LR: 2.20e-04 | 3111.1ms/step | 19.3 samples/s +Step 27500/50462 | Loss: 2.5779 | LR: 2.20e-04 | 3113.9ms/step | 19.3 samples/s +Validation at step 27500: Loss = 2.5836, Perplexity = 13.24 +Step 27510/50462 | Loss: 2.5703 | LR: 2.20e-04 | 4772.1ms/step | 12.6 samples/s +Step 27520/50462 | Loss: 2.5735 | LR: 2.20e-04 | 3111.3ms/step | 19.3 samples/s +Step 27530/50462 | Loss: 2.5957 | LR: 2.20e-04 | 3114.0ms/step | 19.3 samples/s +Step 27540/50462 | Loss: 2.5906 | LR: 2.20e-04 | 3111.9ms/step | 19.3 samples/s +Step 27550/50462 | Loss: 2.5694 | LR: 2.19e-04 | 3114.1ms/step | 19.3 samples/s +Step 27560/50462 | Loss: 2.5803 | LR: 2.19e-04 | 3114.7ms/step | 19.3 samples/s +Step 27570/50462 | Loss: 2.5721 | LR: 2.19e-04 | 3111.7ms/step | 19.3 samples/s +Step 27580/50462 | Loss: 2.5562 | LR: 2.19e-04 | 3112.5ms/step | 19.3 samples/s +Step 27590/50462 | Loss: 2.5806 | LR: 2.19e-04 | 3109.3ms/step | 19.3 samples/s +Step 27600/50462 | Loss: 2.5759 | LR: 2.19e-04 | 3110.6ms/step | 19.3 samples/s +Step 27610/50462 | Loss: 2.5617 | LR: 2.18e-04 | 3111.7ms/step | 19.3 samples/s +Step 27620/50462 | Loss: 2.5837 | LR: 2.18e-04 | 3113.3ms/step | 19.3 samples/s +Step 27630/50462 | Loss: 2.5615 | LR: 2.18e-04 | 3112.6ms/step | 19.3 samples/s +Step 27640/50462 | Loss: 2.5639 | LR: 2.18e-04 | 3114.6ms/step | 19.3 samples/s +Step 27650/50462 | Loss: 2.5518 | LR: 2.18e-04 | 3116.6ms/step | 19.3 samples/s +Step 27660/50462 | Loss: 2.5654 | LR: 2.18e-04 | 3114.1ms/step | 19.3 samples/s +Step 27670/50462 | Loss: 2.5714 | LR: 2.17e-04 | 3114.6ms/step | 19.3 samples/s +Step 27680/50462 | Loss: 2.5645 | LR: 2.17e-04 | 3116.0ms/step | 19.3 samples/s +Step 27690/50462 | Loss: 2.5539 | LR: 2.17e-04 | 3114.4ms/step | 19.3 samples/s +Step 27700/50462 | Loss: 2.5678 | LR: 2.17e-04 | 3116.0ms/step | 19.3 samples/s +Step 27710/50462 | Loss: 2.5671 | LR: 2.17e-04 | 3112.2ms/step | 19.3 samples/s +Step 27720/50462 | Loss: 2.5842 | LR: 2.17e-04 | 3112.4ms/step | 19.3 samples/s +Step 27730/50462 | Loss: 2.5828 | LR: 2.17e-04 | 3113.4ms/step | 19.3 samples/s +Step 27740/50462 | Loss: 2.5683 | LR: 2.16e-04 | 3113.0ms/step | 19.3 samples/s +Step 27750/50462 | Loss: 2.5767 | LR: 2.16e-04 | 3115.4ms/step | 19.3 samples/s +Validation at step 27750: Loss = 2.5813, Perplexity = 13.21 +Step 27760/50462 | Loss: 2.5636 | LR: 2.16e-04 | 4770.8ms/step | 12.6 samples/s +Step 27770/50462 | Loss: 2.5483 | LR: 2.16e-04 | 3112.5ms/step | 19.3 samples/s +Step 27780/50462 | Loss: 2.5857 | LR: 2.16e-04 | 3112.2ms/step | 19.3 samples/s +Step 27790/50462 | Loss: 2.5375 | LR: 2.16e-04 | 3114.6ms/step | 19.3 samples/s +Step 27800/50462 | Loss: 2.5713 | LR: 2.15e-04 | 3111.3ms/step | 19.3 samples/s +Step 27810/50462 | Loss: 2.5779 | LR: 2.15e-04 | 3111.3ms/step | 19.3 samples/s +Step 27820/50462 | Loss: 2.5626 | LR: 2.15e-04 | 3115.0ms/step | 19.3 samples/s +Step 27830/50462 | Loss: 2.5767 | LR: 2.15e-04 | 3112.7ms/step | 19.3 samples/s +Step 27840/50462 | Loss: 2.5570 | LR: 2.15e-04 | 3114.0ms/step | 19.3 samples/s +Step 27850/50462 | Loss: 2.5623 | LR: 2.15e-04 | 3113.9ms/step | 19.3 samples/s +Step 27860/50462 | Loss: 2.5662 | LR: 2.15e-04 | 3113.1ms/step | 19.3 samples/s +Step 27870/50462 | Loss: 2.5228 | LR: 2.14e-04 | 3112.0ms/step | 19.3 samples/s +Step 27880/50462 | Loss: 2.5811 | LR: 2.14e-04 | 3111.9ms/step | 19.3 samples/s +Step 27890/50462 | Loss: 2.5518 | LR: 2.14e-04 | 3111.1ms/step | 19.3 samples/s +Step 27900/50462 | Loss: 2.5516 | LR: 2.14e-04 | 3108.7ms/step | 19.3 samples/s +Step 27910/50462 | Loss: 2.6221 | LR: 2.14e-04 | 3109.9ms/step | 19.3 samples/s +Step 27920/50462 | Loss: 2.5893 | LR: 2.14e-04 | 3107.4ms/step | 19.3 samples/s +Step 27930/50462 | Loss: 2.5597 | LR: 2.13e-04 | 3108.3ms/step | 19.3 samples/s +Step 27940/50462 | Loss: 2.5477 | LR: 2.13e-04 | 3108.0ms/step | 19.3 samples/s +Step 27950/50462 | Loss: 2.5934 | LR: 2.13e-04 | 3108.4ms/step | 19.3 samples/s +Step 27960/50462 | Loss: 2.5733 | LR: 2.13e-04 | 3109.8ms/step | 19.3 samples/s +Step 27970/50462 | Loss: 2.5780 | LR: 2.13e-04 | 3107.2ms/step | 19.3 samples/s +Step 27980/50462 | Loss: 2.5705 | LR: 2.13e-04 | 3107.2ms/step | 19.3 samples/s +Step 27990/50462 | Loss: 2.5775 | LR: 2.12e-04 | 3108.8ms/step | 19.3 samples/s +Step 28000/50462 | Loss: 2.5898 | LR: 2.12e-04 | 3111.8ms/step | 19.3 samples/s +Validation at step 28000: Loss = 2.5793, Perplexity = 13.19 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-28000 +Step 28010/50462 | Loss: 2.5812 | LR: 2.12e-04 | 5653.9ms/step | 10.6 samples/s +Step 28020/50462 | Loss: 2.5886 | LR: 2.12e-04 | 3099.3ms/step | 19.4 samples/s +Step 28030/50462 | Loss: 2.5922 | LR: 2.12e-04 | 3108.6ms/step | 19.3 samples/s +Step 28040/50462 | Loss: 2.5886 | LR: 2.12e-04 | 3110.6ms/step | 19.3 samples/s +Step 28050/50462 | Loss: 2.5520 | LR: 2.12e-04 | 3111.8ms/step | 19.3 samples/s +Step 28060/50462 | Loss: 2.5944 | LR: 2.11e-04 | 3109.3ms/step | 19.3 samples/s +Step 28070/50462 | Loss: 2.5682 | LR: 2.11e-04 | 3108.0ms/step | 19.3 samples/s +Step 28080/50462 | Loss: 2.5590 | LR: 2.11e-04 | 3111.0ms/step | 19.3 samples/s +Step 28090/50462 | Loss: 2.5744 | LR: 2.11e-04 | 3109.1ms/step | 19.3 samples/s +Step 28100/50462 | Loss: 2.5373 | LR: 2.11e-04 | 3110.3ms/step | 19.3 samples/s +Step 28110/50462 | Loss: 2.5732 | LR: 2.11e-04 | 3112.3ms/step | 19.3 samples/s +Step 28120/50462 | Loss: 2.5575 | LR: 2.10e-04 | 3113.3ms/step | 19.3 samples/s +Step 28130/50462 | Loss: 2.5901 | LR: 2.10e-04 | 3112.5ms/step | 19.3 samples/s +Step 28140/50462 | Loss: 2.5593 | LR: 2.10e-04 | 3111.5ms/step | 19.3 samples/s +Step 28150/50462 | Loss: 2.5812 | LR: 2.10e-04 | 3110.6ms/step | 19.3 samples/s +Step 28160/50462 | Loss: 2.5709 | LR: 2.10e-04 | 3107.8ms/step | 19.3 samples/s +Step 28170/50462 | Loss: 2.5483 | LR: 2.10e-04 | 3110.4ms/step | 19.3 samples/s +Step 28180/50462 | Loss: 2.5836 | LR: 2.10e-04 | 3106.7ms/step | 19.3 samples/s +Step 28190/50462 | Loss: 2.5702 | LR: 2.09e-04 | 3111.2ms/step | 19.3 samples/s +Step 28200/50462 | Loss: 2.5763 | LR: 2.09e-04 | 3112.6ms/step | 19.3 samples/s +Step 28210/50462 | Loss: 2.5641 | LR: 2.09e-04 | 3110.5ms/step | 19.3 samples/s +Step 28220/50462 | Loss: 2.5831 | LR: 2.09e-04 | 3110.2ms/step | 19.3 samples/s +Step 28230/50462 | Loss: 2.5527 | LR: 2.09e-04 | 3112.2ms/step | 19.3 samples/s +Step 28240/50462 | Loss: 2.5580 | LR: 2.09e-04 | 3112.9ms/step | 19.3 samples/s +Step 28250/50462 | Loss: 2.5774 | LR: 2.08e-04 | 3109.7ms/step | 19.3 samples/s +Validation at step 28250: Loss = 2.5789, Perplexity = 13.18 +Step 28260/50462 | Loss: 2.5517 | LR: 2.08e-04 | 4767.1ms/step | 12.6 samples/s +Step 28270/50462 | Loss: 2.5915 | LR: 2.08e-04 | 3110.9ms/step | 19.3 samples/s +Step 28280/50462 | Loss: 2.5428 | LR: 2.08e-04 | 3112.1ms/step | 19.3 samples/s +Step 28290/50462 | Loss: 2.5622 | LR: 2.08e-04 | 3112.5ms/step | 19.3 samples/s +Step 28300/50462 | Loss: 2.5519 | LR: 2.08e-04 | 3112.9ms/step | 19.3 samples/s +Step 28310/50462 | Loss: 2.5846 | LR: 2.07e-04 | 3113.0ms/step | 19.3 samples/s +Step 28320/50462 | Loss: 2.5756 | LR: 2.07e-04 | 3112.7ms/step | 19.3 samples/s +Step 28330/50462 | Loss: 2.5836 | LR: 2.07e-04 | 3118.3ms/step | 19.2 samples/s +Step 28340/50462 | Loss: 2.5664 | LR: 2.07e-04 | 3110.5ms/step | 19.3 samples/s +Step 28350/50462 | Loss: 2.5541 | LR: 2.07e-04 | 3112.3ms/step | 19.3 samples/s +Step 28360/50462 | Loss: 2.5564 | LR: 2.07e-04 | 3112.3ms/step | 19.3 samples/s +Step 28370/50462 | Loss: 2.5506 | LR: 2.07e-04 | 3109.9ms/step | 19.3 samples/s +Step 28380/50462 | Loss: 2.5595 | LR: 2.06e-04 | 3109.9ms/step | 19.3 samples/s +Step 28390/50462 | Loss: 2.5553 | LR: 2.06e-04 | 3112.3ms/step | 19.3 samples/s +Step 28400/50462 | Loss: 2.5880 | LR: 2.06e-04 | 3109.4ms/step | 19.3 samples/s +Step 28410/50462 | Loss: 2.5647 | LR: 2.06e-04 | 3107.9ms/step | 19.3 samples/s +Step 28420/50462 | Loss: 2.5869 | LR: 2.06e-04 | 3109.8ms/step | 19.3 samples/s +Step 28430/50462 | Loss: 2.5324 | LR: 2.06e-04 | 3111.5ms/step | 19.3 samples/s +Step 28440/50462 | Loss: 2.5718 | LR: 2.05e-04 | 3110.1ms/step | 19.3 samples/s +Step 28450/50462 | Loss: 2.5402 | LR: 2.05e-04 | 3612.7ms/step | 16.6 samples/s +Step 28460/50462 | Loss: 2.5707 | LR: 2.05e-04 | 3092.8ms/step | 19.4 samples/s +Step 28470/50462 | Loss: 2.5716 | LR: 2.05e-04 | 3101.0ms/step | 19.3 samples/s +Step 28480/50462 | Loss: 2.5641 | LR: 2.05e-04 | 3108.7ms/step | 19.3 samples/s +Step 28490/50462 | Loss: 2.5572 | LR: 2.05e-04 | 3108.9ms/step | 19.3 samples/s +Step 28500/50462 | Loss: 2.5672 | LR: 2.05e-04 | 3107.2ms/step | 19.3 samples/s +Validation at step 28500: Loss = 2.5759, Perplexity = 13.14 +Step 28510/50462 | Loss: 2.5548 | LR: 2.04e-04 | 4764.1ms/step | 12.6 samples/s +Step 28520/50462 | Loss: 2.5679 | LR: 2.04e-04 | 3109.0ms/step | 19.3 samples/s +Step 28530/50462 | Loss: 2.5189 | LR: 2.04e-04 | 3110.1ms/step | 19.3 samples/s +Step 28540/50462 | Loss: 2.5609 | LR: 2.04e-04 | 3110.5ms/step | 19.3 samples/s +Step 28550/50462 | Loss: 2.6136 | LR: 2.04e-04 | 3111.4ms/step | 19.3 samples/s +Step 28560/50462 | Loss: 2.5532 | LR: 2.04e-04 | 3109.7ms/step | 19.3 samples/s +Step 28570/50462 | Loss: 2.5700 | LR: 2.03e-04 | 3111.2ms/step | 19.3 samples/s +Step 28580/50462 | Loss: 2.5639 | LR: 2.03e-04 | 3113.1ms/step | 19.3 samples/s +Step 28590/50462 | Loss: 2.5828 | LR: 2.03e-04 | 3109.7ms/step | 19.3 samples/s +Step 28600/50462 | Loss: 2.5504 | LR: 2.03e-04 | 3111.7ms/step | 19.3 samples/s +Step 28610/50462 | Loss: 2.5678 | LR: 2.03e-04 | 3111.8ms/step | 19.3 samples/s +Step 28620/50462 | Loss: 2.5461 | LR: 2.03e-04 | 3110.1ms/step | 19.3 samples/s +Step 28630/50462 | Loss: 2.5595 | LR: 2.03e-04 | 3111.0ms/step | 19.3 samples/s +Step 28640/50462 | Loss: 2.5739 | LR: 2.02e-04 | 3110.7ms/step | 19.3 samples/s +Step 28650/50462 | Loss: 2.5631 | LR: 2.02e-04 | 3110.1ms/step | 19.3 samples/s +Step 28660/50462 | Loss: 2.5651 | LR: 2.02e-04 | 3111.9ms/step | 19.3 samples/s +Step 28670/50462 | Loss: 2.5629 | LR: 2.02e-04 | 3111.9ms/step | 19.3 samples/s +Step 28680/50462 | Loss: 2.5776 | LR: 2.02e-04 | 3110.0ms/step | 19.3 samples/s +Step 28690/50462 | Loss: 2.5805 | LR: 2.02e-04 | 3112.1ms/step | 19.3 samples/s +Step 28700/50462 | Loss: 2.5616 | LR: 2.01e-04 | 3108.1ms/step | 19.3 samples/s +Step 28710/50462 | Loss: 2.5440 | LR: 2.01e-04 | 3107.1ms/step | 19.3 samples/s +Step 28720/50462 | Loss: 2.5783 | LR: 2.01e-04 | 3603.6ms/step | 16.6 samples/s +Step 28730/50462 | Loss: 2.5494 | LR: 2.01e-04 | 3102.5ms/step | 19.3 samples/s +Step 28740/50462 | Loss: 2.5678 | LR: 2.01e-04 | 3101.4ms/step | 19.3 samples/s +Step 28750/50462 | Loss: 2.5540 | LR: 2.01e-04 | 3106.0ms/step | 19.3 samples/s +Validation at step 28750: Loss = 2.5746, Perplexity = 13.13 +Step 28760/50462 | Loss: 2.6060 | LR: 2.00e-04 | 4754.5ms/step | 12.6 samples/s +Step 28770/50462 | Loss: 2.5953 | LR: 2.00e-04 | 3105.8ms/step | 19.3 samples/s +Step 28780/50462 | Loss: 2.5662 | LR: 2.00e-04 | 3107.0ms/step | 19.3 samples/s +Step 28790/50462 | Loss: 2.5391 | LR: 2.00e-04 | 3107.3ms/step | 19.3 samples/s +Step 28800/50462 | Loss: 2.6007 | LR: 2.00e-04 | 3633.8ms/step | 16.5 samples/s +Step 28810/50462 | Loss: 2.5367 | LR: 2.00e-04 | 3094.7ms/step | 19.4 samples/s +Step 28820/50462 | Loss: 2.5499 | LR: 2.00e-04 | 3108.0ms/step | 19.3 samples/s +Step 28830/50462 | Loss: 2.5584 | LR: 1.99e-04 | 3107.0ms/step | 19.3 samples/s +Step 28840/50462 | Loss: 2.5662 | LR: 1.99e-04 | 3110.7ms/step | 19.3 samples/s +Step 28850/50462 | Loss: 2.5542 | LR: 1.99e-04 | 3110.4ms/step | 19.3 samples/s +Step 28860/50462 | Loss: 2.5485 | LR: 1.99e-04 | 3112.0ms/step | 19.3 samples/s +Step 28870/50462 | Loss: 2.5695 | LR: 1.99e-04 | 3107.4ms/step | 19.3 samples/s +Step 28880/50462 | Loss: 2.5895 | LR: 1.99e-04 | 3111.1ms/step | 19.3 samples/s +Step 28890/50462 | Loss: 2.5426 | LR: 1.98e-04 | 3111.4ms/step | 19.3 samples/s +Step 28900/50462 | Loss: 2.5583 | LR: 1.98e-04 | 3109.3ms/step | 19.3 samples/s +Step 28910/50462 | Loss: 2.5644 | LR: 1.98e-04 | 3110.2ms/step | 19.3 samples/s +Step 28920/50462 | Loss: 2.5529 | LR: 1.98e-04 | 3107.4ms/step | 19.3 samples/s +Step 28930/50462 | Loss: 2.5794 | LR: 1.98e-04 | 3109.0ms/step | 19.3 samples/s +Step 28940/50462 | Loss: 2.5552 | LR: 1.98e-04 | 3107.3ms/step | 19.3 samples/s +Step 28950/50462 | Loss: 2.5648 | LR: 1.98e-04 | 3108.9ms/step | 19.3 samples/s +Step 28960/50462 | Loss: 2.5668 | LR: 1.97e-04 | 3108.1ms/step | 19.3 samples/s +Step 28970/50462 | Loss: 2.5701 | LR: 1.97e-04 | 3107.5ms/step | 19.3 samples/s +Step 28980/50462 | Loss: 2.5612 | LR: 1.97e-04 | 3106.4ms/step | 19.3 samples/s +Step 28990/50462 | Loss: 2.5557 | LR: 1.97e-04 | 3107.7ms/step | 19.3 samples/s +Step 29000/50462 | Loss: 2.5619 | LR: 1.97e-04 | 3108.0ms/step | 19.3 samples/s +Validation at step 29000: Loss = 2.5729, Perplexity = 13.10 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-29000 +Step 29010/50462 | Loss: 2.5678 | LR: 1.97e-04 | 5708.3ms/step | 10.5 samples/s +Step 29020/50462 | Loss: 2.5945 | LR: 1.96e-04 | 3095.4ms/step | 19.4 samples/s +Step 29030/50462 | Loss: 2.5663 | LR: 1.96e-04 | 3104.8ms/step | 19.3 samples/s +Step 29040/50462 | Loss: 2.5424 | LR: 1.96e-04 | 3106.6ms/step | 19.3 samples/s +Step 29050/50462 | Loss: 2.5522 | LR: 1.96e-04 | 3108.4ms/step | 19.3 samples/s +Step 29060/50462 | Loss: 2.5823 | LR: 1.96e-04 | 3110.4ms/step | 19.3 samples/s +Step 29070/50462 | Loss: 2.5637 | LR: 1.96e-04 | 3109.2ms/step | 19.3 samples/s +Step 29080/50462 | Loss: 2.5755 | LR: 1.96e-04 | 3108.7ms/step | 19.3 samples/s +Step 29090/50462 | Loss: 2.5702 | LR: 1.95e-04 | 3108.0ms/step | 19.3 samples/s +Step 29100/50462 | Loss: 2.5318 | LR: 1.95e-04 | 3105.3ms/step | 19.3 samples/s +Step 29110/50462 | Loss: 2.5714 | LR: 1.95e-04 | 3108.9ms/step | 19.3 samples/s +Step 29120/50462 | Loss: 2.5799 | LR: 1.95e-04 | 3108.6ms/step | 19.3 samples/s +Step 29130/50462 | Loss: 2.5605 | LR: 1.95e-04 | 3108.7ms/step | 19.3 samples/s +Step 29140/50462 | Loss: 2.5573 | LR: 1.95e-04 | 3109.8ms/step | 19.3 samples/s +Step 29150/50462 | Loss: 2.5898 | LR: 1.94e-04 | 3108.5ms/step | 19.3 samples/s +Step 29160/50462 | Loss: 2.5773 | LR: 1.94e-04 | 3111.1ms/step | 19.3 samples/s +Step 29170/50462 | Loss: 2.5506 | LR: 1.94e-04 | 3109.9ms/step | 19.3 samples/s +Step 29180/50462 | Loss: 2.5523 | LR: 1.94e-04 | 3110.2ms/step | 19.3 samples/s +Step 29190/50462 | Loss: 2.5656 | LR: 1.94e-04 | 3106.8ms/step | 19.3 samples/s +Step 29200/50462 | Loss: 2.5424 | LR: 1.94e-04 | 3108.2ms/step | 19.3 samples/s +Step 29210/50462 | Loss: 2.5743 | LR: 1.94e-04 | 3106.0ms/step | 19.3 samples/s +Step 29220/50462 | Loss: 2.5852 | LR: 1.93e-04 | 3108.0ms/step | 19.3 samples/s +Step 29230/50462 | Loss: 2.5512 | LR: 1.93e-04 | 3107.9ms/step | 19.3 samples/s +Step 29240/50462 | Loss: 2.5705 | LR: 1.93e-04 | 3109.5ms/step | 19.3 samples/s +Step 29250/50462 | Loss: 2.5943 | LR: 1.93e-04 | 3106.3ms/step | 19.3 samples/s +Validation at step 29250: Loss = 2.5714, Perplexity = 13.08 +Step 29260/50462 | Loss: 2.5634 | LR: 1.93e-04 | 4766.0ms/step | 12.6 samples/s +Step 29270/50462 | Loss: 2.5656 | LR: 1.93e-04 | 3105.8ms/step | 19.3 samples/s +Step 29280/50462 | Loss: 2.5562 | LR: 1.92e-04 | 3106.1ms/step | 19.3 samples/s +Step 29290/50462 | Loss: 2.5791 | LR: 1.92e-04 | 3106.7ms/step | 19.3 samples/s +Step 29300/50462 | Loss: 2.5642 | LR: 1.92e-04 | 3106.9ms/step | 19.3 samples/s +Step 29310/50462 | Loss: 2.5659 | LR: 1.92e-04 | 3109.3ms/step | 19.3 samples/s +Step 29320/50462 | Loss: 2.5643 | LR: 1.92e-04 | 3109.0ms/step | 19.3 samples/s +Step 29330/50462 | Loss: 2.5528 | LR: 1.92e-04 | 3108.8ms/step | 19.3 samples/s +Step 29340/50462 | Loss: 2.5743 | LR: 1.92e-04 | 3108.6ms/step | 19.3 samples/s +Step 29350/50462 | Loss: 2.5856 | LR: 1.91e-04 | 3107.7ms/step | 19.3 samples/s +Step 29360/50462 | Loss: 2.5379 | LR: 1.91e-04 | 3108.3ms/step | 19.3 samples/s +Step 29370/50462 | Loss: 2.5873 | LR: 1.91e-04 | 3106.8ms/step | 19.3 samples/s +Step 29380/50462 | Loss: 2.5622 | LR: 1.91e-04 | 3107.7ms/step | 19.3 samples/s +Step 29390/50462 | Loss: 2.5541 | LR: 1.91e-04 | 3106.8ms/step | 19.3 samples/s +Step 29400/50462 | Loss: 2.5847 | LR: 1.91e-04 | 3119.5ms/step | 19.2 samples/s +Step 29410/50462 | Loss: 2.5622 | LR: 1.90e-04 | 3108.2ms/step | 19.3 samples/s +Step 29420/50462 | Loss: 2.5596 | LR: 1.90e-04 | 3108.9ms/step | 19.3 samples/s +Step 29430/50462 | Loss: 2.5298 | LR: 1.90e-04 | 3109.0ms/step | 19.3 samples/s +Step 29440/50462 | Loss: 2.5541 | LR: 1.90e-04 | 3106.4ms/step | 19.3 samples/s +Step 29450/50462 | Loss: 2.5392 | LR: 1.90e-04 | 3107.7ms/step | 19.3 samples/s +Step 29460/50462 | Loss: 2.5797 | LR: 1.90e-04 | 3108.6ms/step | 19.3 samples/s +Step 29470/50462 | Loss: 2.5698 | LR: 1.90e-04 | 3107.3ms/step | 19.3 samples/s +Step 29480/50462 | Loss: 2.5507 | LR: 1.89e-04 | 3107.8ms/step | 19.3 samples/s +Step 29490/50462 | Loss: 2.5327 | LR: 1.89e-04 | 3108.9ms/step | 19.3 samples/s +Step 29500/50462 | Loss: 2.5477 | LR: 1.89e-04 | 3108.0ms/step | 19.3 samples/s +Validation at step 29500: Loss = 2.5703, Perplexity = 13.07 +Step 29510/50462 | Loss: 2.5485 | LR: 1.89e-04 | 4764.7ms/step | 12.6 samples/s +Step 29520/50462 | Loss: 2.5721 | LR: 1.89e-04 | 3108.7ms/step | 19.3 samples/s +Step 29530/50462 | Loss: 2.5782 | LR: 1.89e-04 | 3108.9ms/step | 19.3 samples/s +Step 29540/50462 | Loss: 2.5552 | LR: 1.88e-04 | 3108.8ms/step | 19.3 samples/s +Step 29550/50462 | Loss: 2.5271 | LR: 1.88e-04 | 3110.5ms/step | 19.3 samples/s +Step 29560/50462 | Loss: 2.5536 | LR: 1.88e-04 | 3109.8ms/step | 19.3 samples/s +Step 29570/50462 | Loss: 2.5445 | LR: 1.88e-04 | 3109.0ms/step | 19.3 samples/s +Step 29580/50462 | Loss: 2.6014 | LR: 1.88e-04 | 3109.7ms/step | 19.3 samples/s +Step 29590/50462 | Loss: 2.5474 | LR: 1.88e-04 | 3108.2ms/step | 19.3 samples/s +Step 29600/50462 | Loss: 2.5767 | LR: 1.88e-04 | 3108.0ms/step | 19.3 samples/s +Step 29610/50462 | Loss: 2.5810 | LR: 1.87e-04 | 3110.4ms/step | 19.3 samples/s +Step 29620/50462 | Loss: 2.5534 | LR: 1.87e-04 | 3110.9ms/step | 19.3 samples/s +Step 29630/50462 | Loss: 2.5456 | LR: 1.87e-04 | 3112.6ms/step | 19.3 samples/s +Step 29640/50462 | Loss: 2.5604 | LR: 1.87e-04 | 3112.5ms/step | 19.3 samples/s +Step 29650/50462 | Loss: 2.5744 | LR: 1.87e-04 | 3107.4ms/step | 19.3 samples/s +Step 29660/50462 | Loss: 2.5253 | LR: 1.87e-04 | 3106.4ms/step | 19.3 samples/s +Step 29670/50462 | Loss: 2.5628 | LR: 1.86e-04 | 3104.7ms/step | 19.3 samples/s +Step 29680/50462 | Loss: 2.5641 | LR: 1.86e-04 | 3106.2ms/step | 19.3 samples/s +Step 29690/50462 | Loss: 2.5413 | LR: 1.86e-04 | 3109.6ms/step | 19.3 samples/s +Step 29700/50462 | Loss: 2.5733 | LR: 1.86e-04 | 3110.0ms/step | 19.3 samples/s +Step 29710/50462 | Loss: 2.5866 | LR: 1.86e-04 | 3108.4ms/step | 19.3 samples/s +Step 29720/50462 | Loss: 2.5644 | LR: 1.86e-04 | 3105.3ms/step | 19.3 samples/s +Step 29730/50462 | Loss: 2.5592 | LR: 1.86e-04 | 3107.3ms/step | 19.3 samples/s +Step 29740/50462 | Loss: 2.5718 | LR: 1.85e-04 | 3106.3ms/step | 19.3 samples/s +Switching attention mode for step 29750: dense (dense step interval) +Step 29750/50462 | Loss: 2.5614 | LR: 1.85e-04 | 3320.8ms/step | 18.1 samples/s +Validation at step 29750: Loss = 2.5504, Perplexity = 12.81 +Step 29760/50462 | Loss: 2.5462 | LR: 1.85e-04 | 7800.4ms/step | 7.7 samples/s +Step 29770/50462 | Loss: 2.5403 | LR: 1.85e-04 | 5243.0ms/step | 11.4 samples/s +Step 29780/50462 | Loss: 2.5242 | LR: 1.85e-04 | 5239.2ms/step | 11.5 samples/s +Step 29790/50462 | Loss: 2.5359 | LR: 1.85e-04 | 5233.1ms/step | 11.5 samples/s +Step 29800/50462 | Loss: 2.5076 | LR: 1.85e-04 | 5242.5ms/step | 11.4 samples/s +Step 29810/50462 | Loss: 2.5035 | LR: 1.84e-04 | 5241.8ms/step | 11.4 samples/s +Step 29820/50462 | Loss: 2.5264 | LR: 1.84e-04 | 5242.6ms/step | 11.4 samples/s +Step 29830/50462 | Loss: 2.5419 | LR: 1.84e-04 | 5240.1ms/step | 11.5 samples/s +Step 29840/50462 | Loss: 2.5216 | LR: 1.84e-04 | 5237.2ms/step | 11.5 samples/s +Step 29850/50462 | Loss: 2.5229 | LR: 1.84e-04 | 5240.4ms/step | 11.4 samples/s +Step 29860/50462 | Loss: 2.5281 | LR: 1.84e-04 | 5239.5ms/step | 11.5 samples/s +Step 29870/50462 | Loss: 2.5346 | LR: 1.83e-04 | 5244.8ms/step | 11.4 samples/s +Step 29880/50462 | Loss: 2.5388 | LR: 1.83e-04 | 5235.0ms/step | 11.5 samples/s +Step 29890/50462 | Loss: 2.5365 | LR: 1.83e-04 | 5234.9ms/step | 11.5 samples/s +Step 29900/50462 | Loss: 2.5339 | LR: 1.83e-04 | 5237.7ms/step | 11.5 samples/s +Step 29910/50462 | Loss: 2.5176 | LR: 1.83e-04 | 5243.7ms/step | 11.4 samples/s +Step 29920/50462 | Loss: 2.5261 | LR: 1.83e-04 | 5236.9ms/step | 11.5 samples/s +Step 29930/50462 | Loss: 2.5505 | LR: 1.83e-04 | 5237.4ms/step | 11.5 samples/s +Step 29940/50462 | Loss: 2.5271 | LR: 1.82e-04 | 5239.8ms/step | 11.5 samples/s +Step 29950/50462 | Loss: 2.5673 | LR: 1.82e-04 | 5235.2ms/step | 11.5 samples/s +Step 29960/50462 | Loss: 2.5436 | LR: 1.82e-04 | 5234.8ms/step | 11.5 samples/s +Step 29970/50462 | Loss: 2.5616 | LR: 1.82e-04 | 5238.3ms/step | 11.5 samples/s +Step 29980/50462 | Loss: 2.5087 | LR: 1.82e-04 | 5238.7ms/step | 11.5 samples/s +Step 29990/50462 | Loss: 2.5046 | LR: 1.82e-04 | 5239.0ms/step | 11.5 samples/s +Step 30000/50462 | Loss: 2.4820 | LR: 1.81e-04 | 5235.9ms/step | 11.5 samples/s +Validation at step 30000: Loss = 2.5421, Perplexity = 12.71 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-30000 +Switching attention mode for step 30001: sparse_stochastic (configured sparse attention) +Step 30010/50462 | Loss: 2.5283 | LR: 1.81e-04 | 6611.6ms/step | 9.1 samples/s +Step 30020/50462 | Loss: 2.5412 | LR: 1.81e-04 | 3101.1ms/step | 19.3 samples/s +Step 30030/50462 | Loss: 2.5583 | LR: 1.81e-04 | 3107.4ms/step | 19.3 samples/s +Step 30040/50462 | Loss: 2.6153 | LR: 1.81e-04 | 3105.5ms/step | 19.3 samples/s +Step 30050/50462 | Loss: 2.5662 | LR: 1.81e-04 | 3106.4ms/step | 19.3 samples/s +Step 30060/50462 | Loss: 2.5659 | LR: 1.81e-04 | 3106.4ms/step | 19.3 samples/s +Step 30070/50462 | Loss: 2.5726 | LR: 1.80e-04 | 3108.9ms/step | 19.3 samples/s +Step 30080/50462 | Loss: 2.5640 | LR: 1.80e-04 | 3109.9ms/step | 19.3 samples/s +Step 30090/50462 | Loss: 2.5688 | LR: 1.80e-04 | 3107.5ms/step | 19.3 samples/s +Step 30100/50462 | Loss: 2.5944 | LR: 1.80e-04 | 3108.0ms/step | 19.3 samples/s +Step 30110/50462 | Loss: 2.5875 | LR: 1.80e-04 | 3107.2ms/step | 19.3 samples/s +Step 30120/50462 | Loss: 2.5628 | LR: 1.80e-04 | 3110.0ms/step | 19.3 samples/s +Step 30130/50462 | Loss: 2.5234 | LR: 1.79e-04 | 3108.8ms/step | 19.3 samples/s +Step 30140/50462 | Loss: 2.5771 | LR: 1.79e-04 | 3109.3ms/step | 19.3 samples/s +Step 30150/50462 | Loss: 2.5673 | LR: 1.79e-04 | 3106.8ms/step | 19.3 samples/s +Step 30160/50462 | Loss: 2.5670 | LR: 1.79e-04 | 3105.1ms/step | 19.3 samples/s +Step 30170/50462 | Loss: 2.5455 | LR: 1.79e-04 | 3106.1ms/step | 19.3 samples/s +Step 30180/50462 | Loss: 2.5342 | LR: 1.79e-04 | 3108.0ms/step | 19.3 samples/s +Step 30190/50462 | Loss: 2.5299 | LR: 1.79e-04 | 3106.6ms/step | 19.3 samples/s +Step 30200/50462 | Loss: 2.5575 | LR: 1.78e-04 | 3108.8ms/step | 19.3 samples/s +Step 30210/50462 | Loss: 2.5798 | LR: 1.78e-04 | 3107.9ms/step | 19.3 samples/s +Step 30220/50462 | Loss: 2.5409 | LR: 1.78e-04 | 3106.0ms/step | 19.3 samples/s +Step 30230/50462 | Loss: 2.5548 | LR: 1.78e-04 | 3107.4ms/step | 19.3 samples/s +Step 30240/50462 | Loss: 2.5732 | LR: 1.78e-04 | 3110.7ms/step | 19.3 samples/s +Step 30250/50462 | Loss: 2.5371 | LR: 1.78e-04 | 3108.1ms/step | 19.3 samples/s +Validation at step 30250: Loss = 2.5672, Perplexity = 13.03 +Step 30260/50462 | Loss: 2.5584 | LR: 1.78e-04 | 4758.9ms/step | 12.6 samples/s +Step 30270/50462 | Loss: 2.5728 | LR: 1.77e-04 | 3107.6ms/step | 19.3 samples/s +Step 30280/50462 | Loss: 2.5686 | LR: 1.77e-04 | 3105.8ms/step | 19.3 samples/s +Step 30290/50462 | Loss: 2.5604 | LR: 1.77e-04 | 3107.9ms/step | 19.3 samples/s +Step 30300/50462 | Loss: 2.5617 | LR: 1.77e-04 | 3107.0ms/step | 19.3 samples/s +Step 30310/50462 | Loss: 2.5426 | LR: 1.77e-04 | 3107.9ms/step | 19.3 samples/s +Step 30320/50462 | Loss: 2.5491 | LR: 1.77e-04 | 3106.3ms/step | 19.3 samples/s +Step 30330/50462 | Loss: 2.5649 | LR: 1.76e-04 | 3106.7ms/step | 19.3 samples/s +Step 30340/50462 | Loss: 2.5384 | LR: 1.76e-04 | 3109.0ms/step | 19.3 samples/s +Step 30350/50462 | Loss: 2.5872 | LR: 1.76e-04 | 3109.3ms/step | 19.3 samples/s +Step 30360/50462 | Loss: 2.5682 | LR: 1.76e-04 | 3120.7ms/step | 19.2 samples/s +Step 30370/50462 | Loss: 2.5609 | LR: 1.76e-04 | 3103.6ms/step | 19.3 samples/s +Step 30380/50462 | Loss: 2.5565 | LR: 1.76e-04 | 3107.7ms/step | 19.3 samples/s +Step 30390/50462 | Loss: 2.5426 | LR: 1.76e-04 | 3109.5ms/step | 19.3 samples/s +Step 30400/50462 | Loss: 2.5305 | LR: 1.75e-04 | 3108.7ms/step | 19.3 samples/s +Step 30410/50462 | Loss: 2.5502 | LR: 1.75e-04 | 3107.7ms/step | 19.3 samples/s +Step 30420/50462 | Loss: 2.5116 | LR: 1.75e-04 | 3106.0ms/step | 19.3 samples/s +Step 30430/50462 | Loss: 2.5657 | LR: 1.75e-04 | 3106.9ms/step | 19.3 samples/s +Step 30440/50462 | Loss: 2.5672 | LR: 1.75e-04 | 3108.7ms/step | 19.3 samples/s +Step 30450/50462 | Loss: 2.5773 | LR: 1.75e-04 | 3107.5ms/step | 19.3 samples/s +Step 30460/50462 | Loss: 2.5308 | LR: 1.75e-04 | 3106.7ms/step | 19.3 samples/s +Step 30470/50462 | Loss: 2.5557 | LR: 1.74e-04 | 3105.0ms/step | 19.3 samples/s +Step 30480/50462 | Loss: 2.5761 | LR: 1.74e-04 | 3109.0ms/step | 19.3 samples/s +Step 30490/50462 | Loss: 2.5701 | LR: 1.74e-04 | 3108.7ms/step | 19.3 samples/s +Step 30500/50462 | Loss: 2.5494 | LR: 1.74e-04 | 3105.7ms/step | 19.3 samples/s +Validation at step 30500: Loss = 2.5660, Perplexity = 13.01 +Step 30510/50462 | Loss: 2.5483 | LR: 1.74e-04 | 4759.5ms/step | 12.6 samples/s +Step 30520/50462 | Loss: 2.5503 | LR: 1.74e-04 | 3104.7ms/step | 19.3 samples/s +Step 30530/50462 | Loss: 2.5713 | LR: 1.73e-04 | 3110.3ms/step | 19.3 samples/s +Step 30540/50462 | Loss: 2.5283 | LR: 1.73e-04 | 3107.3ms/step | 19.3 samples/s +Step 30550/50462 | Loss: 2.5744 | LR: 1.73e-04 | 3107.1ms/step | 19.3 samples/s +Step 30560/50462 | Loss: 2.5287 | LR: 1.73e-04 | 3111.7ms/step | 19.3 samples/s +Step 30570/50462 | Loss: 2.5609 | LR: 1.73e-04 | 3107.2ms/step | 19.3 samples/s +Step 30580/50462 | Loss: 2.5618 | LR: 1.73e-04 | 3633.5ms/step | 16.5 samples/s +Step 30590/50462 | Loss: 2.5288 | LR: 1.73e-04 | 3096.5ms/step | 19.4 samples/s +Step 30600/50462 | Loss: 2.5330 | LR: 1.72e-04 | 3102.9ms/step | 19.3 samples/s +Step 30610/50462 | Loss: 2.5798 | LR: 1.72e-04 | 3106.0ms/step | 19.3 samples/s +Step 30620/50462 | Loss: 2.5420 | LR: 1.72e-04 | 3104.7ms/step | 19.3 samples/s +Step 30630/50462 | Loss: 2.5282 | LR: 1.72e-04 | 3107.7ms/step | 19.3 samples/s +Step 30640/50462 | Loss: 2.5325 | LR: 1.72e-04 | 3109.7ms/step | 19.3 samples/s +Step 30650/50462 | Loss: 2.5627 | LR: 1.72e-04 | 3107.8ms/step | 19.3 samples/s +Step 30660/50462 | Loss: 2.5195 | LR: 1.72e-04 | 3110.4ms/step | 19.3 samples/s +Step 30670/50462 | Loss: 2.5394 | LR: 1.71e-04 | 3109.3ms/step | 19.3 samples/s +Step 30680/50462 | Loss: 2.5410 | LR: 1.71e-04 | 3109.6ms/step | 19.3 samples/s +Step 30690/50462 | Loss: 2.5391 | LR: 1.71e-04 | 3109.7ms/step | 19.3 samples/s +Step 30700/50462 | Loss: 2.5680 | LR: 1.71e-04 | 3110.5ms/step | 19.3 samples/s +Step 30710/50462 | Loss: 2.5479 | LR: 1.71e-04 | 3114.4ms/step | 19.3 samples/s +Step 30720/50462 | Loss: 2.5795 | LR: 1.71e-04 | 3108.6ms/step | 19.3 samples/s +Step 30730/50462 | Loss: 2.5517 | LR: 1.70e-04 | 3110.5ms/step | 19.3 samples/s +Step 30740/50462 | Loss: 2.5496 | LR: 1.70e-04 | 3107.5ms/step | 19.3 samples/s +Step 30750/50462 | Loss: 2.5431 | LR: 1.70e-04 | 3107.3ms/step | 19.3 samples/s +Validation at step 30750: Loss = 2.5642, Perplexity = 12.99 +Step 30760/50462 | Loss: 2.5643 | LR: 1.70e-04 | 4760.5ms/step | 12.6 samples/s +Step 30770/50462 | Loss: 2.5566 | LR: 1.70e-04 | 3104.7ms/step | 19.3 samples/s +Step 30780/50462 | Loss: 2.5512 | LR: 1.70e-04 | 3105.4ms/step | 19.3 samples/s +Step 30790/50462 | Loss: 2.5643 | LR: 1.70e-04 | 3108.6ms/step | 19.3 samples/s +Step 30800/50462 | Loss: 2.5336 | LR: 1.69e-04 | 3109.8ms/step | 19.3 samples/s +Step 30810/50462 | Loss: 2.5309 | LR: 1.69e-04 | 3110.8ms/step | 19.3 samples/s +Step 30820/50462 | Loss: 2.5629 | LR: 1.69e-04 | 3110.3ms/step | 19.3 samples/s +Step 30830/50462 | Loss: 2.5586 | LR: 1.69e-04 | 3108.8ms/step | 19.3 samples/s +Step 30840/50462 | Loss: 2.5349 | LR: 1.69e-04 | 3111.4ms/step | 19.3 samples/s +Step 30850/50462 | Loss: 2.5565 | LR: 1.69e-04 | 3108.5ms/step | 19.3 samples/s +Step 30860/50462 | Loss: 2.5712 | LR: 1.69e-04 | 3109.5ms/step | 19.3 samples/s +Step 30870/50462 | Loss: 2.5404 | LR: 1.68e-04 | 3109.5ms/step | 19.3 samples/s +Step 30880/50462 | Loss: 2.5166 | LR: 1.68e-04 | 3109.8ms/step | 19.3 samples/s +Step 30890/50462 | Loss: 2.5402 | LR: 1.68e-04 | 3107.2ms/step | 19.3 samples/s +Step 30900/50462 | Loss: 2.5267 | LR: 1.68e-04 | 3109.3ms/step | 19.3 samples/s +Step 30910/50462 | Loss: 2.5287 | LR: 1.68e-04 | 3112.0ms/step | 19.3 samples/s +Step 30920/50462 | Loss: 2.5528 | LR: 1.68e-04 | 3615.5ms/step | 16.6 samples/s +Step 30930/50462 | Loss: 2.5328 | LR: 1.67e-04 | 3103.3ms/step | 19.3 samples/s +Step 30940/50462 | Loss: 2.5098 | LR: 1.67e-04 | 3109.4ms/step | 19.3 samples/s +Step 30950/50462 | Loss: 2.5732 | LR: 1.67e-04 | 3109.7ms/step | 19.3 samples/s +Step 30960/50462 | Loss: 2.5488 | LR: 1.67e-04 | 3107.7ms/step | 19.3 samples/s +Step 30970/50462 | Loss: 2.5709 | LR: 1.67e-04 | 3109.4ms/step | 19.3 samples/s +Step 30980/50462 | Loss: 2.5686 | LR: 1.67e-04 | 3109.4ms/step | 19.3 samples/s +Step 30990/50462 | Loss: 2.5482 | LR: 1.67e-04 | 3109.8ms/step | 19.3 samples/s +Step 31000/50462 | Loss: 2.5423 | LR: 1.66e-04 | 3620.6ms/step | 16.6 samples/s +Validation at step 31000: Loss = 2.5637, Perplexity = 12.98 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-31000 +Step 31010/50462 | Loss: 2.5839 | LR: 1.66e-04 | 5695.4ms/step | 10.5 samples/s +Step 31020/50462 | Loss: 2.5501 | LR: 1.66e-04 | 3097.8ms/step | 19.4 samples/s +Step 31030/50462 | Loss: 2.5421 | LR: 1.66e-04 | 3105.0ms/step | 19.3 samples/s +Step 31040/50462 | Loss: 2.5190 | LR: 1.66e-04 | 3105.9ms/step | 19.3 samples/s +Step 31050/50462 | Loss: 2.5456 | LR: 1.66e-04 | 3108.8ms/step | 19.3 samples/s +Step 31060/50462 | Loss: 2.5450 | LR: 1.66e-04 | 3108.9ms/step | 19.3 samples/s +Step 31070/50462 | Loss: 2.5345 | LR: 1.65e-04 | 3106.5ms/step | 19.3 samples/s +Step 31080/50462 | Loss: 2.5118 | LR: 1.65e-04 | 3106.4ms/step | 19.3 samples/s +Step 31090/50462 | Loss: 2.5365 | LR: 1.65e-04 | 3106.1ms/step | 19.3 samples/s +Step 31100/50462 | Loss: 2.5727 | LR: 1.65e-04 | 3107.2ms/step | 19.3 samples/s +Step 31110/50462 | Loss: 2.5372 | LR: 1.65e-04 | 3108.2ms/step | 19.3 samples/s +Step 31120/50462 | Loss: 2.5746 | LR: 1.65e-04 | 3111.2ms/step | 19.3 samples/s +Step 31130/50462 | Loss: 2.5668 | LR: 1.64e-04 | 3108.6ms/step | 19.3 samples/s +Step 31140/50462 | Loss: 2.5179 | LR: 1.64e-04 | 3106.3ms/step | 19.3 samples/s +Step 31150/50462 | Loss: 2.5772 | LR: 1.64e-04 | 3110.7ms/step | 19.3 samples/s +Step 31160/50462 | Loss: 2.5421 | LR: 1.64e-04 | 3106.9ms/step | 19.3 samples/s +Step 31170/50462 | Loss: 2.5618 | LR: 1.64e-04 | 3108.0ms/step | 19.3 samples/s +Step 31180/50462 | Loss: 2.5445 | LR: 1.64e-04 | 3104.3ms/step | 19.3 samples/s +Step 31190/50462 | Loss: 2.5543 | LR: 1.64e-04 | 3108.4ms/step | 19.3 samples/s +Step 31200/50462 | Loss: 2.5267 | LR: 1.63e-04 | 3107.6ms/step | 19.3 samples/s +Step 31210/50462 | Loss: 2.5469 | LR: 1.63e-04 | 3107.5ms/step | 19.3 samples/s +Step 31220/50462 | Loss: 2.5300 | LR: 1.63e-04 | 3107.3ms/step | 19.3 samples/s +Step 31230/50462 | Loss: 2.5458 | LR: 1.63e-04 | 3107.1ms/step | 19.3 samples/s +Step 31240/50462 | Loss: 2.5259 | LR: 1.63e-04 | 3106.8ms/step | 19.3 samples/s +Step 31250/50462 | Loss: 2.5392 | LR: 1.63e-04 | 3105.8ms/step | 19.3 samples/s +Validation at step 31250: Loss = 2.5620, Perplexity = 12.96 +Step 31260/50462 | Loss: 2.5295 | LR: 1.63e-04 | 4760.7ms/step | 12.6 samples/s +Step 31270/50462 | Loss: 2.5389 | LR: 1.62e-04 | 3110.3ms/step | 19.3 samples/s +Step 31280/50462 | Loss: 2.5747 | LR: 1.62e-04 | 3105.2ms/step | 19.3 samples/s +Step 31290/50462 | Loss: 2.5701 | LR: 1.62e-04 | 3107.5ms/step | 19.3 samples/s +Step 31300/50462 | Loss: 2.5528 | LR: 1.62e-04 | 3106.0ms/step | 19.3 samples/s +Step 31310/50462 | Loss: 2.5512 | LR: 1.62e-04 | 3107.4ms/step | 19.3 samples/s +Step 31320/50462 | Loss: 2.5651 | LR: 1.62e-04 | 3107.6ms/step | 19.3 samples/s +Step 31330/50462 | Loss: 2.5051 | LR: 1.62e-04 | 3108.1ms/step | 19.3 samples/s +Step 31340/50462 | Loss: 2.5535 | LR: 1.61e-04 | 3106.3ms/step | 19.3 samples/s +Step 31350/50462 | Loss: 2.5722 | LR: 1.61e-04 | 3105.2ms/step | 19.3 samples/s +Step 31360/50462 | Loss: 2.5650 | LR: 1.61e-04 | 3105.4ms/step | 19.3 samples/s +Step 31370/50462 | Loss: 2.5433 | LR: 1.61e-04 | 3103.9ms/step | 19.3 samples/s +Step 31380/50462 | Loss: 2.5246 | LR: 1.61e-04 | 3104.4ms/step | 19.3 samples/s +Step 31390/50462 | Loss: 2.5171 | LR: 1.61e-04 | 3107.9ms/step | 19.3 samples/s +Step 31400/50462 | Loss: 2.5432 | LR: 1.60e-04 | 3106.5ms/step | 19.3 samples/s +Step 31410/50462 | Loss: 2.5517 | LR: 1.60e-04 | 3107.2ms/step | 19.3 samples/s +Step 31420/50462 | Loss: 2.5563 | LR: 1.60e-04 | 3107.2ms/step | 19.3 samples/s +Step 31430/50462 | Loss: 2.5573 | LR: 1.60e-04 | 3107.5ms/step | 19.3 samples/s +Step 31440/50462 | Loss: 2.5759 | LR: 1.60e-04 | 3109.8ms/step | 19.3 samples/s +Step 31450/50462 | Loss: 2.5419 | LR: 1.60e-04 | 3107.0ms/step | 19.3 samples/s +Step 31460/50462 | Loss: 2.5557 | LR: 1.60e-04 | 3118.3ms/step | 19.2 samples/s +Step 31470/50462 | Loss: 2.5317 | LR: 1.59e-04 | 3105.8ms/step | 19.3 samples/s +Step 31480/50462 | Loss: 2.5721 | LR: 1.59e-04 | 3098.7ms/step | 19.4 samples/s +Step 31490/50462 | Loss: 2.5775 | LR: 1.59e-04 | 3103.6ms/step | 19.3 samples/s +Step 31500/50462 | Loss: 2.5487 | LR: 1.59e-04 | 3103.8ms/step | 19.3 samples/s +Validation at step 31500: Loss = 2.5606, Perplexity = 12.94 +Step 31510/50462 | Loss: 2.5642 | LR: 1.59e-04 | 4743.9ms/step | 12.6 samples/s +Step 31520/50462 | Loss: 2.5945 | LR: 1.59e-04 | 3092.9ms/step | 19.4 samples/s +Step 31530/50462 | Loss: 2.5205 | LR: 1.59e-04 | 3092.7ms/step | 19.4 samples/s +Step 31540/50462 | Loss: 2.5376 | LR: 1.58e-04 | 3101.7ms/step | 19.3 samples/s +Step 31550/50462 | Loss: 2.5335 | LR: 1.58e-04 | 3082.9ms/step | 19.5 samples/s +Step 31560/50462 | Loss: 2.5633 | LR: 1.58e-04 | 3086.2ms/step | 19.4 samples/s +Step 31570/50462 | Loss: 2.5384 | LR: 1.58e-04 | 3107.9ms/step | 19.3 samples/s +Step 31580/50462 | Loss: 2.5187 | LR: 1.58e-04 | 3088.1ms/step | 19.4 samples/s +Step 31590/50462 | Loss: 2.5481 | LR: 1.58e-04 | 3085.4ms/step | 19.4 samples/s +Step 31600/50462 | Loss: 2.5418 | LR: 1.58e-04 | 3101.5ms/step | 19.3 samples/s +Step 31610/50462 | Loss: 2.5221 | LR: 1.57e-04 | 3108.2ms/step | 19.3 samples/s +Step 31620/50462 | Loss: 2.5727 | LR: 1.57e-04 | 3081.3ms/step | 19.5 samples/s +Step 31630/50462 | Loss: 2.5422 | LR: 1.57e-04 | 3087.9ms/step | 19.4 samples/s +Step 31640/50462 | Loss: 2.5656 | LR: 1.57e-04 | 3096.4ms/step | 19.4 samples/s +Step 31650/50462 | Loss: 2.5533 | LR: 1.57e-04 | 3092.6ms/step | 19.4 samples/s +Step 31660/50462 | Loss: 2.5323 | LR: 1.57e-04 | 3090.7ms/step | 19.4 samples/s +Step 31670/50462 | Loss: 2.5540 | LR: 1.57e-04 | 3098.0ms/step | 19.4 samples/s +Step 31680/50462 | Loss: 2.5576 | LR: 1.56e-04 | 3102.2ms/step | 19.3 samples/s +Step 31690/50462 | Loss: 2.5547 | LR: 1.56e-04 | 3083.5ms/step | 19.5 samples/s +Step 31700/50462 | Loss: 2.5087 | LR: 1.56e-04 | 3089.3ms/step | 19.4 samples/s +Step 31710/50462 | Loss: 2.5166 | LR: 1.56e-04 | 3098.4ms/step | 19.4 samples/s +Step 31720/50462 | Loss: 2.5614 | LR: 1.56e-04 | 3094.7ms/step | 19.4 samples/s +Step 31730/50462 | Loss: 2.5673 | LR: 1.56e-04 | 3079.9ms/step | 19.5 samples/s +Step 31740/50462 | Loss: 2.5614 | LR: 1.55e-04 | 3087.3ms/step | 19.4 samples/s +Step 31750/50462 | Loss: 2.5536 | LR: 1.55e-04 | 3095.1ms/step | 19.4 samples/s +Validation at step 31750: Loss = 2.5596, Perplexity = 12.93 +Step 31760/50462 | Loss: 2.5441 | LR: 1.55e-04 | 4724.1ms/step | 12.7 samples/s +Step 31770/50462 | Loss: 2.5646 | LR: 1.55e-04 | 3087.6ms/step | 19.4 samples/s +Step 31780/50462 | Loss: 2.5357 | LR: 1.55e-04 | 3096.7ms/step | 19.4 samples/s +Step 31790/50462 | Loss: 2.5567 | LR: 1.55e-04 | 3088.6ms/step | 19.4 samples/s +Step 31800/50462 | Loss: 2.5482 | LR: 1.55e-04 | 3088.4ms/step | 19.4 samples/s +Step 31810/50462 | Loss: 2.5399 | LR: 1.54e-04 | 3094.0ms/step | 19.4 samples/s +Step 31820/50462 | Loss: 2.5549 | LR: 1.54e-04 | 3100.0ms/step | 19.4 samples/s +Step 31830/50462 | Loss: 2.5286 | LR: 1.54e-04 | 3087.2ms/step | 19.4 samples/s +Step 31840/50462 | Loss: 2.5469 | LR: 1.54e-04 | 3091.0ms/step | 19.4 samples/s +Step 31850/50462 | Loss: 2.5412 | LR: 1.54e-04 | 3096.3ms/step | 19.4 samples/s +Step 31860/50462 | Loss: 2.5342 | LR: 1.54e-04 | 3094.4ms/step | 19.4 samples/s +Step 31870/50462 | Loss: 2.5487 | LR: 1.54e-04 | 3090.5ms/step | 19.4 samples/s +Step 31880/50462 | Loss: 2.5484 | LR: 1.53e-04 | 3093.1ms/step | 19.4 samples/s +Step 31890/50462 | Loss: 2.5492 | LR: 1.53e-04 | 3099.8ms/step | 19.4 samples/s +Step 31900/50462 | Loss: 2.5687 | LR: 1.53e-04 | 3077.2ms/step | 19.5 samples/s +Step 31910/50462 | Loss: 2.5507 | LR: 1.53e-04 | 3086.7ms/step | 19.4 samples/s +Step 31920/50462 | Loss: 2.5126 | LR: 1.53e-04 | 3095.3ms/step | 19.4 samples/s +Step 31930/50462 | Loss: 2.5319 | LR: 1.53e-04 | 3096.9ms/step | 19.4 samples/s +Step 31940/50462 | Loss: 2.5615 | LR: 1.53e-04 | 3089.4ms/step | 19.4 samples/s +Step 31950/50462 | Loss: 2.5661 | LR: 1.52e-04 | 3091.2ms/step | 19.4 samples/s +Step 31960/50462 | Loss: 2.5576 | LR: 1.52e-04 | 3096.3ms/step | 19.4 samples/s +Step 31970/50462 | Loss: 2.5465 | LR: 1.52e-04 | 3091.6ms/step | 19.4 samples/s +Step 31980/50462 | Loss: 2.5702 | LR: 1.52e-04 | 3088.8ms/step | 19.4 samples/s +Step 31990/50462 | Loss: 2.5694 | LR: 1.52e-04 | 3093.1ms/step | 19.4 samples/s +Step 32000/50462 | Loss: 2.5946 | LR: 1.52e-04 | 3100.1ms/step | 19.4 samples/s +Validation at step 32000: Loss = 2.5585, Perplexity = 12.92 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-32000 +Step 32010/50462 | Loss: 2.5752 | LR: 1.52e-04 | 5713.2ms/step | 10.5 samples/s +Step 32020/50462 | Loss: 2.5289 | LR: 1.51e-04 | 3088.5ms/step | 19.4 samples/s +Step 32030/50462 | Loss: 2.5320 | LR: 1.51e-04 | 3097.4ms/step | 19.4 samples/s +Step 32040/50462 | Loss: 2.5840 | LR: 1.51e-04 | 3104.1ms/step | 19.3 samples/s +Step 32050/50462 | Loss: 2.5473 | LR: 1.51e-04 | 3108.4ms/step | 19.3 samples/s +Step 32060/50462 | Loss: 2.5327 | LR: 1.51e-04 | 3109.8ms/step | 19.3 samples/s +Step 32070/50462 | Loss: 2.5502 | LR: 1.51e-04 | 3110.0ms/step | 19.3 samples/s +Step 32080/50462 | Loss: 2.5582 | LR: 1.51e-04 | 3111.8ms/step | 19.3 samples/s +Step 32090/50462 | Loss: 2.5428 | LR: 1.50e-04 | 3111.5ms/step | 19.3 samples/s +Step 32100/50462 | Loss: 2.5393 | LR: 1.50e-04 | 3111.6ms/step | 19.3 samples/s +Step 32110/50462 | Loss: 2.5255 | LR: 1.50e-04 | 3112.4ms/step | 19.3 samples/s +Step 32120/50462 | Loss: 2.5531 | LR: 1.50e-04 | 3110.9ms/step | 19.3 samples/s +Step 32130/50462 | Loss: 2.5402 | LR: 1.50e-04 | 3106.0ms/step | 19.3 samples/s +Step 32140/50462 | Loss: 2.5608 | LR: 1.50e-04 | 3104.6ms/step | 19.3 samples/s +Step 32150/50462 | Loss: 2.5797 | LR: 1.50e-04 | 3107.9ms/step | 19.3 samples/s +Step 32160/50462 | Loss: 2.5436 | LR: 1.49e-04 | 3108.3ms/step | 19.3 samples/s +Step 32170/50462 | Loss: 2.5437 | LR: 1.49e-04 | 3109.0ms/step | 19.3 samples/s +Step 32180/50462 | Loss: 2.5300 | LR: 1.49e-04 | 3111.3ms/step | 19.3 samples/s +Step 32190/50462 | Loss: 2.5590 | LR: 1.49e-04 | 3109.0ms/step | 19.3 samples/s +Step 32200/50462 | Loss: 2.5391 | LR: 1.49e-04 | 3108.9ms/step | 19.3 samples/s +Step 32210/50462 | Loss: 2.5603 | LR: 1.49e-04 | 3108.1ms/step | 19.3 samples/s +Step 32220/50462 | Loss: 2.5550 | LR: 1.49e-04 | 3104.7ms/step | 19.3 samples/s +Step 32230/50462 | Loss: 2.5456 | LR: 1.48e-04 | 3108.3ms/step | 19.3 samples/s +Step 32240/50462 | Loss: 2.5335 | LR: 1.48e-04 | 3111.6ms/step | 19.3 samples/s +Step 32250/50462 | Loss: 2.5481 | LR: 1.48e-04 | 3111.8ms/step | 19.3 samples/s +Validation at step 32250: Loss = 2.5578, Perplexity = 12.91 +Step 32260/50462 | Loss: 2.5394 | LR: 1.48e-04 | 4754.4ms/step | 12.6 samples/s +Step 32270/50462 | Loss: 2.5565 | LR: 1.48e-04 | 3105.3ms/step | 19.3 samples/s +Step 32280/50462 | Loss: 2.5455 | LR: 1.48e-04 | 3106.5ms/step | 19.3 samples/s +Step 32290/50462 | Loss: 2.5587 | LR: 1.48e-04 | 3106.2ms/step | 19.3 samples/s +Step 32300/50462 | Loss: 2.5641 | LR: 1.47e-04 | 3107.9ms/step | 19.3 samples/s +Step 32310/50462 | Loss: 2.5856 | LR: 1.47e-04 | 3110.1ms/step | 19.3 samples/s +Step 32320/50462 | Loss: 2.5230 | LR: 1.47e-04 | 3110.6ms/step | 19.3 samples/s +Step 32330/50462 | Loss: 2.5671 | LR: 1.47e-04 | 3107.8ms/step | 19.3 samples/s +Step 32340/50462 | Loss: 2.5963 | LR: 1.47e-04 | 3109.1ms/step | 19.3 samples/s +Step 32350/50462 | Loss: 2.5326 | LR: 1.47e-04 | 3108.5ms/step | 19.3 samples/s +Step 32360/50462 | Loss: 2.5476 | LR: 1.47e-04 | 3110.1ms/step | 19.3 samples/s +Step 32370/50462 | Loss: 2.5561 | LR: 1.46e-04 | 3109.2ms/step | 19.3 samples/s +Step 32380/50462 | Loss: 2.5068 | LR: 1.46e-04 | 3112.0ms/step | 19.3 samples/s +Step 32390/50462 | Loss: 2.5368 | LR: 1.46e-04 | 3111.5ms/step | 19.3 samples/s +Step 32400/50462 | Loss: 2.5657 | LR: 1.46e-04 | 3107.6ms/step | 19.3 samples/s +Step 32410/50462 | Loss: 2.5613 | LR: 1.46e-04 | 3111.7ms/step | 19.3 samples/s +Step 32420/50462 | Loss: 2.5381 | LR: 1.46e-04 | 3109.5ms/step | 19.3 samples/s +Step 32430/50462 | Loss: 2.5328 | LR: 1.45e-04 | 3107.9ms/step | 19.3 samples/s +Step 32440/50462 | Loss: 2.5391 | LR: 1.45e-04 | 3111.3ms/step | 19.3 samples/s +Step 32450/50462 | Loss: 2.5571 | LR: 1.45e-04 | 3109.9ms/step | 19.3 samples/s +Step 32460/50462 | Loss: 2.5365 | LR: 1.45e-04 | 3108.6ms/step | 19.3 samples/s +Step 32470/50462 | Loss: 2.5608 | LR: 1.45e-04 | 3108.1ms/step | 19.3 samples/s +Step 32480/50462 | Loss: 2.5640 | LR: 1.45e-04 | 3110.7ms/step | 19.3 samples/s +Step 32490/50462 | Loss: 2.5409 | LR: 1.45e-04 | 3113.0ms/step | 19.3 samples/s +Step 32500/50462 | Loss: 2.5885 | LR: 1.44e-04 | 3109.4ms/step | 19.3 samples/s +Validation at step 32500: Loss = 2.5565, Perplexity = 12.89 +Step 32510/50462 | Loss: 2.5425 | LR: 1.44e-04 | 4767.9ms/step | 12.6 samples/s +Step 32520/50462 | Loss: 2.5424 | LR: 1.44e-04 | 3120.9ms/step | 19.2 samples/s +Step 32530/50462 | Loss: 2.5477 | LR: 1.44e-04 | 3108.8ms/step | 19.3 samples/s +Step 32540/50462 | Loss: 2.5429 | LR: 1.44e-04 | 3110.0ms/step | 19.3 samples/s +Step 32550/50462 | Loss: 2.5464 | LR: 1.44e-04 | 3109.8ms/step | 19.3 samples/s +Step 32560/50462 | Loss: 2.5090 | LR: 1.44e-04 | 3110.6ms/step | 19.3 samples/s +Step 32570/50462 | Loss: 2.5325 | LR: 1.43e-04 | 3110.1ms/step | 19.3 samples/s +Step 32580/50462 | Loss: 2.5295 | LR: 1.43e-04 | 3108.2ms/step | 19.3 samples/s +Step 32590/50462 | Loss: 2.5413 | LR: 1.43e-04 | 3109.5ms/step | 19.3 samples/s +Step 32600/50462 | Loss: 2.5442 | LR: 1.43e-04 | 3110.4ms/step | 19.3 samples/s +Step 32610/50462 | Loss: 2.5360 | LR: 1.43e-04 | 3107.4ms/step | 19.3 samples/s +Step 32620/50462 | Loss: 2.5509 | LR: 1.43e-04 | 3107.9ms/step | 19.3 samples/s +Step 32630/50462 | Loss: 2.5346 | LR: 1.43e-04 | 3110.1ms/step | 19.3 samples/s +Step 32640/50462 | Loss: 2.5415 | LR: 1.42e-04 | 3105.7ms/step | 19.3 samples/s +Step 32650/50462 | Loss: 2.5410 | LR: 1.42e-04 | 3105.5ms/step | 19.3 samples/s +Step 32660/50462 | Loss: 2.5576 | LR: 1.42e-04 | 3106.7ms/step | 19.3 samples/s +Step 32670/50462 | Loss: 2.5606 | LR: 1.42e-04 | 3107.7ms/step | 19.3 samples/s +Step 32680/50462 | Loss: 2.5509 | LR: 1.42e-04 | 3109.1ms/step | 19.3 samples/s +Step 32690/50462 | Loss: 2.5187 | LR: 1.42e-04 | 3107.7ms/step | 19.3 samples/s +Step 32700/50462 | Loss: 2.5175 | LR: 1.42e-04 | 3607.1ms/step | 16.6 samples/s +Step 32710/50462 | Loss: 2.5657 | LR: 1.41e-04 | 3100.9ms/step | 19.3 samples/s +Step 32720/50462 | Loss: 2.5543 | LR: 1.41e-04 | 3106.4ms/step | 19.3 samples/s +Step 32730/50462 | Loss: 2.5294 | LR: 1.41e-04 | 3109.9ms/step | 19.3 samples/s +Step 32740/50462 | Loss: 2.5636 | LR: 1.41e-04 | 3107.8ms/step | 19.3 samples/s +Step 32750/50462 | Loss: 2.5547 | LR: 1.41e-04 | 3109.8ms/step | 19.3 samples/s +Validation at step 32750: Loss = 2.5556, Perplexity = 12.88 +Step 32760/50462 | Loss: 2.5608 | LR: 1.41e-04 | 4771.4ms/step | 12.6 samples/s +Step 32770/50462 | Loss: 2.5427 | LR: 1.41e-04 | 3112.6ms/step | 19.3 samples/s +Step 32780/50462 | Loss: 2.5275 | LR: 1.41e-04 | 3110.9ms/step | 19.3 samples/s +Step 32790/50462 | Loss: 2.5546 | LR: 1.40e-04 | 3107.7ms/step | 19.3 samples/s +Step 32800/50462 | Loss: 2.5125 | LR: 1.40e-04 | 3113.0ms/step | 19.3 samples/s +Step 32810/50462 | Loss: 2.5403 | LR: 1.40e-04 | 3108.0ms/step | 19.3 samples/s +Step 32820/50462 | Loss: 2.5574 | LR: 1.40e-04 | 3111.6ms/step | 19.3 samples/s +Step 32830/50462 | Loss: 2.5570 | LR: 1.40e-04 | 3109.7ms/step | 19.3 samples/s +Step 32840/50462 | Loss: 2.5703 | LR: 1.40e-04 | 3109.2ms/step | 19.3 samples/s +Step 32850/50462 | Loss: 2.5861 | LR: 1.40e-04 | 3108.8ms/step | 19.3 samples/s +Step 32860/50462 | Loss: 2.5517 | LR: 1.39e-04 | 3107.4ms/step | 19.3 samples/s +Step 32870/50462 | Loss: 2.5305 | LR: 1.39e-04 | 3109.7ms/step | 19.3 samples/s +Step 32880/50462 | Loss: 2.5369 | LR: 1.39e-04 | 3112.2ms/step | 19.3 samples/s +Step 32890/50462 | Loss: 2.5630 | LR: 1.39e-04 | 3108.4ms/step | 19.3 samples/s +Step 32900/50462 | Loss: 2.5460 | LR: 1.39e-04 | 3108.9ms/step | 19.3 samples/s +Step 32910/50462 | Loss: 2.5342 | LR: 1.39e-04 | 3105.4ms/step | 19.3 samples/s +Step 32920/50462 | Loss: 2.5503 | LR: 1.39e-04 | 3109.9ms/step | 19.3 samples/s +Step 32930/50462 | Loss: 2.5420 | LR: 1.38e-04 | 3108.7ms/step | 19.3 samples/s +Step 32940/50462 | Loss: 2.5382 | LR: 1.38e-04 | 3113.3ms/step | 19.3 samples/s +Step 32950/50462 | Loss: 2.5274 | LR: 1.38e-04 | 3112.9ms/step | 19.3 samples/s +Step 32960/50462 | Loss: 2.5155 | LR: 1.38e-04 | 3108.0ms/step | 19.3 samples/s +Step 32970/50462 | Loss: 2.5626 | LR: 1.38e-04 | 3112.6ms/step | 19.3 samples/s +Step 32980/50462 | Loss: 2.5299 | LR: 1.38e-04 | 3114.2ms/step | 19.3 samples/s +Step 32990/50462 | Loss: 2.5247 | LR: 1.38e-04 | 3113.8ms/step | 19.3 samples/s +Step 33000/50462 | Loss: 2.5385 | LR: 1.37e-04 | 3110.8ms/step | 19.3 samples/s +Validation at step 33000: Loss = 2.5545, Perplexity = 12.87 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-33000 +Step 33010/50462 | Loss: 2.5560 | LR: 1.37e-04 | 5684.4ms/step | 10.6 samples/s +Step 33020/50462 | Loss: 2.5290 | LR: 1.37e-04 | 3099.4ms/step | 19.4 samples/s +Step 33030/50462 | Loss: 2.5595 | LR: 1.37e-04 | 3106.8ms/step | 19.3 samples/s +Step 33040/50462 | Loss: 2.5483 | LR: 1.37e-04 | 3107.2ms/step | 19.3 samples/s +Step 33050/50462 | Loss: 2.5128 | LR: 1.37e-04 | 3109.9ms/step | 19.3 samples/s +Step 33060/50462 | Loss: 2.5447 | LR: 1.37e-04 | 3111.4ms/step | 19.3 samples/s +Step 33070/50462 | Loss: 2.5375 | LR: 1.36e-04 | 3111.9ms/step | 19.3 samples/s +Step 33080/50462 | Loss: 2.5427 | LR: 1.36e-04 | 3113.1ms/step | 19.3 samples/s +Step 33090/50462 | Loss: 2.5402 | LR: 1.36e-04 | 3109.4ms/step | 19.3 samples/s +Step 33100/50462 | Loss: 2.5279 | LR: 1.36e-04 | 3111.4ms/step | 19.3 samples/s +Step 33110/50462 | Loss: 2.5263 | LR: 1.36e-04 | 3642.7ms/step | 16.5 samples/s +Step 33120/50462 | Loss: 2.5352 | LR: 1.36e-04 | 3100.5ms/step | 19.4 samples/s +Step 33130/50462 | Loss: 2.5315 | LR: 1.36e-04 | 3111.4ms/step | 19.3 samples/s +Step 33140/50462 | Loss: 2.5415 | LR: 1.35e-04 | 3111.1ms/step | 19.3 samples/s +Step 33150/50462 | Loss: 2.5493 | LR: 1.35e-04 | 3115.2ms/step | 19.3 samples/s +Step 33160/50462 | Loss: 2.5717 | LR: 1.35e-04 | 3113.0ms/step | 19.3 samples/s +Step 33170/50462 | Loss: 2.5499 | LR: 1.35e-04 | 3112.0ms/step | 19.3 samples/s +Step 33180/50462 | Loss: 2.5351 | LR: 1.35e-04 | 3113.2ms/step | 19.3 samples/s +Step 33190/50462 | Loss: 2.5230 | LR: 1.35e-04 | 3625.9ms/step | 16.5 samples/s +Step 33200/50462 | Loss: 2.5472 | LR: 1.35e-04 | 3099.6ms/step | 19.4 samples/s +Step 33210/50462 | Loss: 2.5751 | LR: 1.34e-04 | 3103.0ms/step | 19.3 samples/s +Step 33220/50462 | Loss: 2.5482 | LR: 1.34e-04 | 3109.1ms/step | 19.3 samples/s +Step 33230/50462 | Loss: 2.5446 | LR: 1.34e-04 | 3108.7ms/step | 19.3 samples/s +Step 33240/50462 | Loss: 2.4948 | LR: 1.34e-04 | 3115.6ms/step | 19.3 samples/s +Step 33250/50462 | Loss: 2.5592 | LR: 1.34e-04 | 3112.1ms/step | 19.3 samples/s +Validation at step 33250: Loss = 2.5538, Perplexity = 12.86 +Step 33260/50462 | Loss: 2.5098 | LR: 1.34e-04 | 4757.1ms/step | 12.6 samples/s +Step 33270/50462 | Loss: 2.5560 | LR: 1.34e-04 | 3112.1ms/step | 19.3 samples/s +Step 33280/50462 | Loss: 2.5547 | LR: 1.33e-04 | 3113.0ms/step | 19.3 samples/s +Step 33290/50462 | Loss: 2.5378 | LR: 1.33e-04 | 3112.7ms/step | 19.3 samples/s +Step 33300/50462 | Loss: 2.5499 | LR: 1.33e-04 | 3108.8ms/step | 19.3 samples/s +Step 33310/50462 | Loss: 2.5652 | LR: 1.33e-04 | 3113.2ms/step | 19.3 samples/s +Step 33320/50462 | Loss: 2.5503 | LR: 1.33e-04 | 3111.1ms/step | 19.3 samples/s +Step 33330/50462 | Loss: 2.5721 | LR: 1.33e-04 | 3108.9ms/step | 19.3 samples/s +Step 33340/50462 | Loss: 2.5943 | LR: 1.33e-04 | 3113.7ms/step | 19.3 samples/s +Step 33350/50462 | Loss: 2.5458 | LR: 1.32e-04 | 3112.9ms/step | 19.3 samples/s +Step 33360/50462 | Loss: 2.5235 | LR: 1.32e-04 | 3108.2ms/step | 19.3 samples/s +Step 33370/50462 | Loss: 2.5158 | LR: 1.32e-04 | 3112.8ms/step | 19.3 samples/s +Step 33380/50462 | Loss: 2.5512 | LR: 1.32e-04 | 3113.9ms/step | 19.3 samples/s +Step 33390/50462 | Loss: 2.5246 | LR: 1.32e-04 | 3115.7ms/step | 19.3 samples/s +Step 33400/50462 | Loss: 2.5300 | LR: 1.32e-04 | 3116.2ms/step | 19.3 samples/s +Step 33410/50462 | Loss: 2.5286 | LR: 1.32e-04 | 3114.1ms/step | 19.3 samples/s +Step 33420/50462 | Loss: 2.5341 | LR: 1.32e-04 | 3112.4ms/step | 19.3 samples/s +Step 33430/50462 | Loss: 2.5470 | LR: 1.31e-04 | 3112.9ms/step | 19.3 samples/s +Step 33440/50462 | Loss: 2.5886 | LR: 1.31e-04 | 3112.5ms/step | 19.3 samples/s +Step 33450/50462 | Loss: 2.5543 | LR: 1.31e-04 | 3114.5ms/step | 19.3 samples/s +Step 33460/50462 | Loss: 2.5319 | LR: 1.31e-04 | 3115.0ms/step | 19.3 samples/s +Step 33470/50462 | Loss: 2.5328 | LR: 1.31e-04 | 3111.9ms/step | 19.3 samples/s +Step 33480/50462 | Loss: 2.5172 | LR: 1.31e-04 | 3111.2ms/step | 19.3 samples/s +Step 33490/50462 | Loss: 2.5564 | LR: 1.31e-04 | 3107.3ms/step | 19.3 samples/s +Step 33500/50462 | Loss: 2.5335 | LR: 1.30e-04 | 3106.8ms/step | 19.3 samples/s +Validation at step 33500: Loss = 2.5530, Perplexity = 12.85 +Step 33510/50462 | Loss: 2.5558 | LR: 1.30e-04 | 4764.5ms/step | 12.6 samples/s +Step 33520/50462 | Loss: 2.5373 | LR: 1.30e-04 | 3109.8ms/step | 19.3 samples/s +Step 33530/50462 | Loss: 2.5083 | LR: 1.30e-04 | 3111.3ms/step | 19.3 samples/s +Step 33540/50462 | Loss: 2.5432 | LR: 1.30e-04 | 3111.3ms/step | 19.3 samples/s +Step 33550/50462 | Loss: 2.5447 | LR: 1.30e-04 | 3109.6ms/step | 19.3 samples/s +Step 33560/50462 | Loss: 2.5300 | LR: 1.30e-04 | 3111.4ms/step | 19.3 samples/s +Step 33570/50462 | Loss: 2.5021 | LR: 1.29e-04 | 3110.3ms/step | 19.3 samples/s +Step 33580/50462 | Loss: 2.5414 | LR: 1.29e-04 | 3110.3ms/step | 19.3 samples/s +Step 33590/50462 | Loss: 2.5409 | LR: 1.29e-04 | 3110.9ms/step | 19.3 samples/s +Step 33600/50462 | Loss: 2.5347 | LR: 1.29e-04 | 3114.1ms/step | 19.3 samples/s +Step 33610/50462 | Loss: 2.5585 | LR: 1.29e-04 | 3110.1ms/step | 19.3 samples/s +Step 33620/50462 | Loss: 2.5419 | LR: 1.29e-04 | 3107.5ms/step | 19.3 samples/s +Step 33630/50462 | Loss: 2.4952 | LR: 1.29e-04 | 3110.5ms/step | 19.3 samples/s +Step 33640/50462 | Loss: 2.5441 | LR: 1.28e-04 | 3112.2ms/step | 19.3 samples/s +Step 33650/50462 | Loss: 2.5368 | LR: 1.28e-04 | 3105.6ms/step | 19.3 samples/s +Step 33660/50462 | Loss: 2.5357 | LR: 1.28e-04 | 3107.5ms/step | 19.3 samples/s +Step 33670/50462 | Loss: 2.5262 | LR: 1.28e-04 | 3111.1ms/step | 19.3 samples/s +Step 33680/50462 | Loss: 2.5529 | LR: 1.28e-04 | 3110.3ms/step | 19.3 samples/s +Step 33690/50462 | Loss: 2.5377 | LR: 1.28e-04 | 3109.3ms/step | 19.3 samples/s +Step 33700/50462 | Loss: 2.5219 | LR: 1.28e-04 | 3111.3ms/step | 19.3 samples/s +Step 33710/50462 | Loss: 2.5638 | LR: 1.27e-04 | 3109.1ms/step | 19.3 samples/s +Step 33720/50462 | Loss: 2.5479 | LR: 1.27e-04 | 3113.3ms/step | 19.3 samples/s +Step 33730/50462 | Loss: 2.5152 | LR: 1.27e-04 | 3113.0ms/step | 19.3 samples/s +Step 33740/50462 | Loss: 2.5353 | LR: 1.27e-04 | 3113.1ms/step | 19.3 samples/s +Step 33750/50462 | Loss: 2.5505 | LR: 1.27e-04 | 3112.6ms/step | 19.3 samples/s +Validation at step 33750: Loss = 2.5523, Perplexity = 12.84 +Step 33760/50462 | Loss: 2.5111 | LR: 1.27e-04 | 4770.8ms/step | 12.6 samples/s +Step 33770/50462 | Loss: 2.5329 | LR: 1.27e-04 | 3114.2ms/step | 19.3 samples/s +Step 33780/50462 | Loss: 2.5515 | LR: 1.27e-04 | 3110.4ms/step | 19.3 samples/s +Step 33790/50462 | Loss: 2.5524 | LR: 1.26e-04 | 3111.5ms/step | 19.3 samples/s +Step 33800/50462 | Loss: 2.5418 | LR: 1.26e-04 | 3115.0ms/step | 19.3 samples/s +Step 33810/50462 | Loss: 2.5472 | LR: 1.26e-04 | 3109.6ms/step | 19.3 samples/s +Step 33820/50462 | Loss: 2.5119 | LR: 1.26e-04 | 3112.7ms/step | 19.3 samples/s +Step 33830/50462 | Loss: 2.5077 | LR: 1.26e-04 | 3114.4ms/step | 19.3 samples/s +Step 33840/50462 | Loss: 2.5482 | LR: 1.26e-04 | 3112.7ms/step | 19.3 samples/s +Step 33850/50462 | Loss: 2.5330 | LR: 1.26e-04 | 3112.2ms/step | 19.3 samples/s +Step 33860/50462 | Loss: 2.5364 | LR: 1.25e-04 | 3111.9ms/step | 19.3 samples/s +Step 33870/50462 | Loss: 2.5206 | LR: 1.25e-04 | 3115.7ms/step | 19.3 samples/s +Step 33880/50462 | Loss: 2.5799 | LR: 1.25e-04 | 3108.1ms/step | 19.3 samples/s +Step 33890/50462 | Loss: 2.5448 | LR: 1.25e-04 | 3108.4ms/step | 19.3 samples/s +Step 33900/50462 | Loss: 2.5630 | LR: 1.25e-04 | 3106.7ms/step | 19.3 samples/s +Step 33910/50462 | Loss: 2.5323 | LR: 1.25e-04 | 3094.4ms/step | 19.4 samples/s +Step 33920/50462 | Loss: 2.5688 | LR: 1.25e-04 | 3096.3ms/step | 19.4 samples/s +Step 33930/50462 | Loss: 2.5173 | LR: 1.24e-04 | 3101.8ms/step | 19.3 samples/s +Step 33940/50462 | Loss: 2.5296 | LR: 1.24e-04 | 3103.3ms/step | 19.3 samples/s +Step 33950/50462 | Loss: 2.5403 | LR: 1.24e-04 | 3106.9ms/step | 19.3 samples/s +Step 33960/50462 | Loss: 2.5716 | LR: 1.24e-04 | 3108.0ms/step | 19.3 samples/s +Step 33970/50462 | Loss: 2.5046 | LR: 1.24e-04 | 3108.1ms/step | 19.3 samples/s +Step 33980/50462 | Loss: 2.5515 | LR: 1.24e-04 | 3107.0ms/step | 19.3 samples/s +Step 33990/50462 | Loss: 2.5448 | LR: 1.24e-04 | 3109.7ms/step | 19.3 samples/s +Step 34000/50462 | Loss: 2.5060 | LR: 1.24e-04 | 3108.5ms/step | 19.3 samples/s +Validation at step 34000: Loss = 2.5512, Perplexity = 12.82 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-34000 +Step 34010/50462 | Loss: 2.5329 | LR: 1.23e-04 | 5704.0ms/step | 10.5 samples/s +Step 34020/50462 | Loss: 2.5429 | LR: 1.23e-04 | 3094.4ms/step | 19.4 samples/s +Step 34030/50462 | Loss: 2.5147 | LR: 1.23e-04 | 3102.7ms/step | 19.3 samples/s +Step 34040/50462 | Loss: 2.5476 | LR: 1.23e-04 | 3104.3ms/step | 19.3 samples/s +Step 34050/50462 | Loss: 2.5579 | LR: 1.23e-04 | 3105.0ms/step | 19.3 samples/s +Step 34060/50462 | Loss: 2.5455 | LR: 1.23e-04 | 3108.6ms/step | 19.3 samples/s +Step 34070/50462 | Loss: 2.5343 | LR: 1.23e-04 | 3110.9ms/step | 19.3 samples/s +Step 34080/50462 | Loss: 2.5592 | LR: 1.22e-04 | 3110.5ms/step | 19.3 samples/s +Step 34090/50462 | Loss: 2.5564 | LR: 1.22e-04 | 3112.2ms/step | 19.3 samples/s +Step 34100/50462 | Loss: 2.5591 | LR: 1.22e-04 | 3111.6ms/step | 19.3 samples/s +Step 34110/50462 | Loss: 2.5432 | LR: 1.22e-04 | 3112.7ms/step | 19.3 samples/s +Step 34120/50462 | Loss: 2.5356 | LR: 1.22e-04 | 3114.6ms/step | 19.3 samples/s +Step 34130/50462 | Loss: 2.5203 | LR: 1.22e-04 | 3113.6ms/step | 19.3 samples/s +Step 34140/50462 | Loss: 2.5639 | LR: 1.22e-04 | 3113.2ms/step | 19.3 samples/s +Step 34150/50462 | Loss: 2.5177 | LR: 1.21e-04 | 3113.6ms/step | 19.3 samples/s +Step 34160/50462 | Loss: 2.5496 | LR: 1.21e-04 | 3110.0ms/step | 19.3 samples/s +Step 34170/50462 | Loss: 2.5252 | LR: 1.21e-04 | 3109.5ms/step | 19.3 samples/s +Step 34180/50462 | Loss: 2.5385 | LR: 1.21e-04 | 3110.7ms/step | 19.3 samples/s +Step 34190/50462 | Loss: 2.5305 | LR: 1.21e-04 | 3109.6ms/step | 19.3 samples/s +Step 34200/50462 | Loss: 2.5268 | LR: 1.21e-04 | 3110.3ms/step | 19.3 samples/s +Step 34210/50462 | Loss: 2.5388 | LR: 1.21e-04 | 3113.2ms/step | 19.3 samples/s +Step 34220/50462 | Loss: 2.5397 | LR: 1.21e-04 | 3112.1ms/step | 19.3 samples/s +Step 34230/50462 | Loss: 2.5489 | LR: 1.20e-04 | 3107.5ms/step | 19.3 samples/s +Step 34240/50462 | Loss: 2.5206 | LR: 1.20e-04 | 3111.4ms/step | 19.3 samples/s +Step 34250/50462 | Loss: 2.5310 | LR: 1.20e-04 | 3114.1ms/step | 19.3 samples/s +Validation at step 34250: Loss = 2.5505, Perplexity = 12.81 +Step 34260/50462 | Loss: 2.4907 | LR: 1.20e-04 | 4766.4ms/step | 12.6 samples/s +Step 34270/50462 | Loss: 2.5532 | LR: 1.20e-04 | 3108.9ms/step | 19.3 samples/s +Step 34280/50462 | Loss: 2.5372 | LR: 1.20e-04 | 3111.0ms/step | 19.3 samples/s +Step 34290/50462 | Loss: 2.5595 | LR: 1.20e-04 | 3111.7ms/step | 19.3 samples/s +Step 34300/50462 | Loss: 2.5557 | LR: 1.19e-04 | 3107.0ms/step | 19.3 samples/s +Step 34310/50462 | Loss: 2.5696 | LR: 1.19e-04 | 3109.1ms/step | 19.3 samples/s +Step 34320/50462 | Loss: 2.5188 | LR: 1.19e-04 | 3112.4ms/step | 19.3 samples/s +Step 34330/50462 | Loss: 2.5274 | LR: 1.19e-04 | 3111.2ms/step | 19.3 samples/s +Step 34340/50462 | Loss: 2.5186 | LR: 1.19e-04 | 3112.3ms/step | 19.3 samples/s +Step 34350/50462 | Loss: 2.5326 | LR: 1.19e-04 | 3114.9ms/step | 19.3 samples/s +Step 34360/50462 | Loss: 2.5435 | LR: 1.19e-04 | 3112.4ms/step | 19.3 samples/s +Step 34370/50462 | Loss: 2.5294 | LR: 1.19e-04 | 3113.8ms/step | 19.3 samples/s +Step 34380/50462 | Loss: 2.5111 | LR: 1.18e-04 | 3112.1ms/step | 19.3 samples/s +Step 34390/50462 | Loss: 2.5376 | LR: 1.18e-04 | 3110.6ms/step | 19.3 samples/s +Step 34400/50462 | Loss: 2.5458 | LR: 1.18e-04 | 3110.5ms/step | 19.3 samples/s +Step 34410/50462 | Loss: 2.5329 | LR: 1.18e-04 | 3109.9ms/step | 19.3 samples/s +Step 34420/50462 | Loss: 2.5232 | LR: 1.18e-04 | 3109.7ms/step | 19.3 samples/s +Step 34430/50462 | Loss: 2.5432 | LR: 1.18e-04 | 3109.6ms/step | 19.3 samples/s +Step 34440/50462 | Loss: 2.5272 | LR: 1.18e-04 | 3111.4ms/step | 19.3 samples/s +Step 34450/50462 | Loss: 2.5611 | LR: 1.17e-04 | 3111.2ms/step | 19.3 samples/s +Step 34460/50462 | Loss: 2.5205 | LR: 1.17e-04 | 3113.8ms/step | 19.3 samples/s +Step 34470/50462 | Loss: 2.5591 | LR: 1.17e-04 | 3115.6ms/step | 19.3 samples/s +Step 34480/50462 | Loss: 2.5623 | LR: 1.17e-04 | 3113.3ms/step | 19.3 samples/s +Step 34490/50462 | Loss: 2.5328 | LR: 1.17e-04 | 3109.8ms/step | 19.3 samples/s +Step 34500/50462 | Loss: 2.5443 | LR: 1.17e-04 | 3110.6ms/step | 19.3 samples/s +Validation at step 34500: Loss = 2.5498, Perplexity = 12.80 +Step 34510/50462 | Loss: 2.5403 | LR: 1.17e-04 | 4766.6ms/step | 12.6 samples/s +Step 34520/50462 | Loss: 2.5238 | LR: 1.16e-04 | 3111.8ms/step | 19.3 samples/s +Step 34530/50462 | Loss: 2.5283 | LR: 1.16e-04 | 3111.9ms/step | 19.3 samples/s +Step 34540/50462 | Loss: 2.5320 | LR: 1.16e-04 | 3111.7ms/step | 19.3 samples/s +Step 34550/50462 | Loss: 2.5141 | LR: 1.16e-04 | 3114.8ms/step | 19.3 samples/s +Step 34560/50462 | Loss: 2.5515 | LR: 1.16e-04 | 3110.3ms/step | 19.3 samples/s +Step 34570/50462 | Loss: 2.5215 | LR: 1.16e-04 | 3112.1ms/step | 19.3 samples/s +Step 34580/50462 | Loss: 2.5360 | LR: 1.16e-04 | 3110.0ms/step | 19.3 samples/s +Step 34590/50462 | Loss: 2.5405 | LR: 1.16e-04 | 3116.1ms/step | 19.3 samples/s +Step 34600/50462 | Loss: 2.5286 | LR: 1.15e-04 | 3113.0ms/step | 19.3 samples/s +Step 34610/50462 | Loss: 2.5388 | LR: 1.15e-04 | 3112.3ms/step | 19.3 samples/s +Step 34620/50462 | Loss: 2.5444 | LR: 1.15e-04 | 3111.6ms/step | 19.3 samples/s +Step 34630/50462 | Loss: 2.5323 | LR: 1.15e-04 | 3111.3ms/step | 19.3 samples/s +Step 34640/50462 | Loss: 2.5621 | LR: 1.15e-04 | 3112.5ms/step | 19.3 samples/s +Step 34650/50462 | Loss: 2.5485 | LR: 1.15e-04 | 3113.1ms/step | 19.3 samples/s +Step 34660/50462 | Loss: 2.5472 | LR: 1.15e-04 | 3115.2ms/step | 19.3 samples/s +Step 34670/50462 | Loss: 2.5625 | LR: 1.15e-04 | 3111.2ms/step | 19.3 samples/s +Step 34680/50462 | Loss: 2.5523 | LR: 1.14e-04 | 3122.2ms/step | 19.2 samples/s +Step 34690/50462 | Loss: 2.5489 | LR: 1.14e-04 | 3110.8ms/step | 19.3 samples/s +Step 34700/50462 | Loss: 2.5463 | LR: 1.14e-04 | 3111.3ms/step | 19.3 samples/s +Step 34710/50462 | Loss: 2.5387 | LR: 1.14e-04 | 3113.2ms/step | 19.3 samples/s +Step 34720/50462 | Loss: 2.5334 | LR: 1.14e-04 | 3112.3ms/step | 19.3 samples/s +Step 34730/50462 | Loss: 2.5345 | LR: 1.14e-04 | 3112.5ms/step | 19.3 samples/s +Step 34740/50462 | Loss: 2.5481 | LR: 1.14e-04 | 3112.6ms/step | 19.3 samples/s +Switching attention mode for step 34750: dense (dense step interval) +Step 34750/50462 | Loss: 2.5465 | LR: 1.13e-04 | 3332.4ms/step | 18.0 samples/s +Validation at step 34750: Loss = 2.5308, Perplexity = 12.56 +Step 34760/50462 | Loss: 2.5159 | LR: 1.13e-04 | 7806.5ms/step | 7.7 samples/s +Step 34770/50462 | Loss: 2.4955 | LR: 1.13e-04 | 5250.7ms/step | 11.4 samples/s +Step 34780/50462 | Loss: 2.5101 | LR: 1.13e-04 | 5247.7ms/step | 11.4 samples/s +Step 34790/50462 | Loss: 2.5018 | LR: 1.13e-04 | 5249.4ms/step | 11.4 samples/s +Step 34800/50462 | Loss: 2.5154 | LR: 1.13e-04 | 5250.4ms/step | 11.4 samples/s +Step 34810/50462 | Loss: 2.5297 | LR: 1.13e-04 | 5249.3ms/step | 11.4 samples/s +Step 34820/50462 | Loss: 2.5218 | LR: 1.13e-04 | 5756.7ms/step | 10.4 samples/s +Step 34830/50462 | Loss: 2.5135 | LR: 1.12e-04 | 5224.0ms/step | 11.5 samples/s +Step 34840/50462 | Loss: 2.5271 | LR: 1.12e-04 | 5246.0ms/step | 11.4 samples/s +Step 34850/50462 | Loss: 2.5466 | LR: 1.12e-04 | 5244.8ms/step | 11.4 samples/s +Step 34860/50462 | Loss: 2.5292 | LR: 1.12e-04 | 5246.9ms/step | 11.4 samples/s +Step 34870/50462 | Loss: 2.4923 | LR: 1.12e-04 | 5249.5ms/step | 11.4 samples/s +Step 34880/50462 | Loss: 2.5157 | LR: 1.12e-04 | 5249.1ms/step | 11.4 samples/s +Step 34890/50462 | Loss: 2.5056 | LR: 1.12e-04 | 5256.1ms/step | 11.4 samples/s +Step 34900/50462 | Loss: 2.5135 | LR: 1.11e-04 | 5252.8ms/step | 11.4 samples/s +Step 34910/50462 | Loss: 2.5218 | LR: 1.11e-04 | 5251.2ms/step | 11.4 samples/s +Step 34920/50462 | Loss: 2.4586 | LR: 1.11e-04 | 5255.2ms/step | 11.4 samples/s +Step 34930/50462 | Loss: 2.4977 | LR: 1.11e-04 | 5249.5ms/step | 11.4 samples/s +Step 34940/50462 | Loss: 2.5130 | LR: 1.11e-04 | 5251.6ms/step | 11.4 samples/s +Step 34950/50462 | Loss: 2.5152 | LR: 1.11e-04 | 5255.5ms/step | 11.4 samples/s +Step 34960/50462 | Loss: 2.5162 | LR: 1.11e-04 | 5256.6ms/step | 11.4 samples/s +Step 34970/50462 | Loss: 2.4951 | LR: 1.11e-04 | 5252.0ms/step | 11.4 samples/s +Step 34980/50462 | Loss: 2.4997 | LR: 1.10e-04 | 5251.5ms/step | 11.4 samples/s +Step 34990/50462 | Loss: 2.5304 | LR: 1.10e-04 | 5252.4ms/step | 11.4 samples/s +Step 35000/50462 | Loss: 2.5410 | LR: 1.10e-04 | 5252.0ms/step | 11.4 samples/s +Validation at step 35000: Loss = 2.5230, Perplexity = 12.47 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-35000 +Switching attention mode for step 35001: sparse_stochastic (configured sparse attention) +Step 35010/50462 | Loss: 2.5500 | LR: 1.10e-04 | 6614.1ms/step | 9.1 samples/s +Step 35020/50462 | Loss: 2.5316 | LR: 1.10e-04 | 3095.9ms/step | 19.4 samples/s +Step 35030/50462 | Loss: 2.5421 | LR: 1.10e-04 | 3108.6ms/step | 19.3 samples/s +Step 35040/50462 | Loss: 2.5260 | LR: 1.10e-04 | 3111.9ms/step | 19.3 samples/s +Step 35050/50462 | Loss: 2.5295 | LR: 1.09e-04 | 3112.6ms/step | 19.3 samples/s +Step 35060/50462 | Loss: 2.5209 | LR: 1.09e-04 | 3109.9ms/step | 19.3 samples/s +Step 35070/50462 | Loss: 2.5166 | LR: 1.09e-04 | 3109.9ms/step | 19.3 samples/s +Step 35080/50462 | Loss: 2.5559 | LR: 1.09e-04 | 3115.0ms/step | 19.3 samples/s +Step 35090/50462 | Loss: 2.5115 | LR: 1.09e-04 | 3111.9ms/step | 19.3 samples/s +Step 35100/50462 | Loss: 2.5425 | LR: 1.09e-04 | 3111.9ms/step | 19.3 samples/s +Step 35110/50462 | Loss: 2.5445 | LR: 1.09e-04 | 3112.1ms/step | 19.3 samples/s +Step 35120/50462 | Loss: 2.5275 | LR: 1.09e-04 | 3112.3ms/step | 19.3 samples/s +Step 35130/50462 | Loss: 2.5069 | LR: 1.08e-04 | 3113.0ms/step | 19.3 samples/s +Step 35140/50462 | Loss: 2.5166 | LR: 1.08e-04 | 3111.0ms/step | 19.3 samples/s +Step 35150/50462 | Loss: 2.5364 | LR: 1.08e-04 | 3111.9ms/step | 19.3 samples/s +Step 35160/50462 | Loss: 2.5477 | LR: 1.08e-04 | 3110.4ms/step | 19.3 samples/s +Step 35170/50462 | Loss: 2.5413 | LR: 1.08e-04 | 3111.7ms/step | 19.3 samples/s +Step 35180/50462 | Loss: 2.5344 | LR: 1.08e-04 | 3114.6ms/step | 19.3 samples/s +Step 35190/50462 | Loss: 2.5437 | LR: 1.08e-04 | 3114.8ms/step | 19.3 samples/s +Step 35200/50462 | Loss: 2.5353 | LR: 1.08e-04 | 3111.6ms/step | 19.3 samples/s +Step 35210/50462 | Loss: 2.5344 | LR: 1.07e-04 | 3112.2ms/step | 19.3 samples/s +Step 35220/50462 | Loss: 2.5394 | LR: 1.07e-04 | 3110.3ms/step | 19.3 samples/s +Step 35230/50462 | Loss: 2.5482 | LR: 1.07e-04 | 3113.3ms/step | 19.3 samples/s +Step 35240/50462 | Loss: 2.5411 | LR: 1.07e-04 | 3112.2ms/step | 19.3 samples/s +Step 35250/50462 | Loss: 2.5154 | LR: 1.07e-04 | 3111.4ms/step | 19.3 samples/s +Validation at step 35250: Loss = 2.5479, Perplexity = 12.78 +Step 35260/50462 | Loss: 2.4964 | LR: 1.07e-04 | 4765.8ms/step | 12.6 samples/s +Step 35270/50462 | Loss: 2.5230 | LR: 1.07e-04 | 3113.4ms/step | 19.3 samples/s +Step 35280/50462 | Loss: 2.5261 | LR: 1.07e-04 | 3114.3ms/step | 19.3 samples/s +Step 35290/50462 | Loss: 2.5083 | LR: 1.06e-04 | 3113.9ms/step | 19.3 samples/s +Step 35300/50462 | Loss: 2.5290 | LR: 1.06e-04 | 3112.5ms/step | 19.3 samples/s +Step 35310/50462 | Loss: 2.5510 | LR: 1.06e-04 | 3648.0ms/step | 16.4 samples/s +Step 35320/50462 | Loss: 2.5426 | LR: 1.06e-04 | 3101.9ms/step | 19.3 samples/s +Step 35330/50462 | Loss: 2.5550 | LR: 1.06e-04 | 3111.0ms/step | 19.3 samples/s +Step 35340/50462 | Loss: 2.5349 | LR: 1.06e-04 | 3106.6ms/step | 19.3 samples/s +Step 35350/50462 | Loss: 2.5461 | LR: 1.06e-04 | 3106.9ms/step | 19.3 samples/s +Step 35360/50462 | Loss: 2.5356 | LR: 1.05e-04 | 3108.8ms/step | 19.3 samples/s +Step 35370/50462 | Loss: 2.5489 | LR: 1.05e-04 | 3111.0ms/step | 19.3 samples/s +Step 35380/50462 | Loss: 2.5384 | LR: 1.05e-04 | 3112.7ms/step | 19.3 samples/s +Step 35390/50462 | Loss: 2.5410 | LR: 1.05e-04 | 3643.5ms/step | 16.5 samples/s +Step 35400/50462 | Loss: 2.5608 | LR: 1.05e-04 | 3098.0ms/step | 19.4 samples/s +Step 35410/50462 | Loss: 2.5413 | LR: 1.05e-04 | 3100.2ms/step | 19.4 samples/s +Step 35420/50462 | Loss: 2.5373 | LR: 1.05e-04 | 3109.6ms/step | 19.3 samples/s +Step 35430/50462 | Loss: 2.5358 | LR: 1.05e-04 | 3109.9ms/step | 19.3 samples/s +Step 35440/50462 | Loss: 2.5370 | LR: 1.04e-04 | 3115.7ms/step | 19.3 samples/s +Step 35450/50462 | Loss: 2.5322 | LR: 1.04e-04 | 3113.7ms/step | 19.3 samples/s +Step 35460/50462 | Loss: 2.5371 | LR: 1.04e-04 | 3114.7ms/step | 19.3 samples/s +Step 35470/50462 | Loss: 2.5326 | LR: 1.04e-04 | 3114.7ms/step | 19.3 samples/s +Step 35480/50462 | Loss: 2.5241 | LR: 1.04e-04 | 3109.1ms/step | 19.3 samples/s +Step 35490/50462 | Loss: 2.5363 | LR: 1.04e-04 | 3115.5ms/step | 19.3 samples/s +Step 35500/50462 | Loss: 2.5550 | LR: 1.04e-04 | 3118.5ms/step | 19.2 samples/s +Validation at step 35500: Loss = 2.5475, Perplexity = 12.78 +Step 35510/50462 | Loss: 2.5403 | LR: 1.04e-04 | 4772.2ms/step | 12.6 samples/s +Step 35520/50462 | Loss: 2.5207 | LR: 1.03e-04 | 3113.1ms/step | 19.3 samples/s +Step 35530/50462 | Loss: 2.5393 | LR: 1.03e-04 | 3116.3ms/step | 19.3 samples/s +Step 35540/50462 | Loss: 2.4975 | LR: 1.03e-04 | 3118.5ms/step | 19.2 samples/s +Step 35550/50462 | Loss: 2.5403 | LR: 1.03e-04 | 3117.1ms/step | 19.2 samples/s +Step 35560/50462 | Loss: 2.5434 | LR: 1.03e-04 | 3119.2ms/step | 19.2 samples/s +Step 35570/50462 | Loss: 2.5498 | LR: 1.03e-04 | 3121.4ms/step | 19.2 samples/s +Step 35580/50462 | Loss: 2.5275 | LR: 1.03e-04 | 3116.9ms/step | 19.2 samples/s +Step 35590/50462 | Loss: 2.5512 | LR: 1.03e-04 | 3117.7ms/step | 19.2 samples/s +Step 35600/50462 | Loss: 2.5033 | LR: 1.02e-04 | 3118.1ms/step | 19.2 samples/s +Step 35610/50462 | Loss: 2.5208 | LR: 1.02e-04 | 3117.5ms/step | 19.2 samples/s +Step 35620/50462 | Loss: 2.5270 | LR: 1.02e-04 | 3117.3ms/step | 19.2 samples/s +Step 35630/50462 | Loss: 2.5419 | LR: 1.02e-04 | 3119.8ms/step | 19.2 samples/s +Step 35640/50462 | Loss: 2.5117 | LR: 1.02e-04 | 3116.6ms/step | 19.3 samples/s +Step 35650/50462 | Loss: 2.5363 | LR: 1.02e-04 | 3116.7ms/step | 19.3 samples/s +Step 35660/50462 | Loss: 2.5309 | LR: 1.02e-04 | 3116.3ms/step | 19.3 samples/s +Step 35670/50462 | Loss: 2.5145 | LR: 1.02e-04 | 3111.1ms/step | 19.3 samples/s +Step 35680/50462 | Loss: 2.5515 | LR: 1.01e-04 | 3116.6ms/step | 19.3 samples/s +Step 35690/50462 | Loss: 2.5179 | LR: 1.01e-04 | 3117.2ms/step | 19.2 samples/s +Step 35700/50462 | Loss: 2.5294 | LR: 1.01e-04 | 3118.3ms/step | 19.2 samples/s +Step 35710/50462 | Loss: 2.5637 | LR: 1.01e-04 | 3119.4ms/step | 19.2 samples/s +Step 35720/50462 | Loss: 2.5011 | LR: 1.01e-04 | 3116.6ms/step | 19.3 samples/s +Step 35730/50462 | Loss: 2.5618 | LR: 1.01e-04 | 3116.5ms/step | 19.3 samples/s +Step 35740/50462 | Loss: 2.5157 | LR: 1.01e-04 | 3116.0ms/step | 19.3 samples/s +Step 35750/50462 | Loss: 2.5356 | LR: 1.00e-04 | 3116.6ms/step | 19.3 samples/s +Validation at step 35750: Loss = 2.5465, Perplexity = 12.76 +Step 35760/50462 | Loss: 2.5296 | LR: 1.00e-04 | 4775.2ms/step | 12.6 samples/s +Step 35770/50462 | Loss: 2.5350 | LR: 1.00e-04 | 3118.0ms/step | 19.2 samples/s +Step 35780/50462 | Loss: 2.5140 | LR: 1.00e-04 | 3116.9ms/step | 19.3 samples/s +Step 35790/50462 | Loss: 2.5360 | LR: 1.00e-04 | 3116.4ms/step | 19.3 samples/s +Step 35800/50462 | Loss: 2.5234 | LR: 9.99e-05 | 3116.1ms/step | 19.3 samples/s +Step 35810/50462 | Loss: 2.5456 | LR: 9.97e-05 | 3115.7ms/step | 19.3 samples/s +Step 35820/50462 | Loss: 2.5462 | LR: 9.96e-05 | 3114.7ms/step | 19.3 samples/s +Step 35830/50462 | Loss: 2.5273 | LR: 9.95e-05 | 3115.1ms/step | 19.3 samples/s +Step 35840/50462 | Loss: 2.5326 | LR: 9.94e-05 | 3112.8ms/step | 19.3 samples/s +Step 35850/50462 | Loss: 2.5246 | LR: 9.92e-05 | 3111.4ms/step | 19.3 samples/s +Step 35860/50462 | Loss: 2.5259 | LR: 9.91e-05 | 3114.2ms/step | 19.3 samples/s +Step 35870/50462 | Loss: 2.5383 | LR: 9.90e-05 | 3112.3ms/step | 19.3 samples/s +Step 35880/50462 | Loss: 2.5467 | LR: 9.88e-05 | 3114.2ms/step | 19.3 samples/s +Step 35890/50462 | Loss: 2.5404 | LR: 9.87e-05 | 3115.8ms/step | 19.3 samples/s +Step 35900/50462 | Loss: 2.5182 | LR: 9.86e-05 | 3114.0ms/step | 19.3 samples/s +Step 35910/50462 | Loss: 2.5553 | LR: 9.85e-05 | 3112.0ms/step | 19.3 samples/s +Step 35920/50462 | Loss: 2.5485 | LR: 9.83e-05 | 3113.7ms/step | 19.3 samples/s +Step 35930/50462 | Loss: 2.5255 | LR: 9.82e-05 | 3113.9ms/step | 19.3 samples/s +Step 35940/50462 | Loss: 2.5251 | LR: 9.81e-05 | 3116.1ms/step | 19.3 samples/s +Step 35950/50462 | Loss: 2.5079 | LR: 9.80e-05 | 3118.3ms/step | 19.2 samples/s +Step 35960/50462 | Loss: 2.5318 | LR: 9.78e-05 | 3117.0ms/step | 19.2 samples/s +Step 35970/50462 | Loss: 2.5256 | LR: 9.77e-05 | 3116.5ms/step | 19.3 samples/s +Step 35980/50462 | Loss: 2.5215 | LR: 9.76e-05 | 3113.9ms/step | 19.3 samples/s +Step 35990/50462 | Loss: 2.5134 | LR: 9.75e-05 | 3113.3ms/step | 19.3 samples/s +Step 36000/50462 | Loss: 2.5139 | LR: 9.73e-05 | 3112.0ms/step | 19.3 samples/s +Validation at step 36000: Loss = 2.5461, Perplexity = 12.76 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-36000 +Step 36010/50462 | Loss: 2.5086 | LR: 9.72e-05 | 5713.6ms/step | 10.5 samples/s +Step 36020/50462 | Loss: 2.5361 | LR: 9.71e-05 | 3102.4ms/step | 19.3 samples/s +Step 36030/50462 | Loss: 2.5123 | LR: 9.70e-05 | 3110.0ms/step | 19.3 samples/s +Step 36040/50462 | Loss: 2.5671 | LR: 9.68e-05 | 3110.2ms/step | 19.3 samples/s +Step 36050/50462 | Loss: 2.5351 | LR: 9.67e-05 | 3114.7ms/step | 19.3 samples/s +Step 36060/50462 | Loss: 2.5453 | LR: 9.66e-05 | 3115.2ms/step | 19.3 samples/s +Step 36070/50462 | Loss: 2.5238 | LR: 9.65e-05 | 3115.1ms/step | 19.3 samples/s +Step 36080/50462 | Loss: 2.5400 | LR: 9.63e-05 | 3115.5ms/step | 19.3 samples/s +Step 36090/50462 | Loss: 2.5173 | LR: 9.62e-05 | 3113.3ms/step | 19.3 samples/s +Step 36100/50462 | Loss: 2.5514 | LR: 9.61e-05 | 3117.0ms/step | 19.2 samples/s +Step 36110/50462 | Loss: 2.5110 | LR: 9.60e-05 | 3113.8ms/step | 19.3 samples/s +Step 36120/50462 | Loss: 2.5420 | LR: 9.58e-05 | 3113.9ms/step | 19.3 samples/s +Step 36130/50462 | Loss: 2.5431 | LR: 9.57e-05 | 3109.6ms/step | 19.3 samples/s +Step 36140/50462 | Loss: 2.5437 | LR: 9.56e-05 | 3105.2ms/step | 19.3 samples/s +Step 36150/50462 | Loss: 2.5159 | LR: 9.55e-05 | 3110.7ms/step | 19.3 samples/s +Step 36160/50462 | Loss: 2.5165 | LR: 9.53e-05 | 3113.1ms/step | 19.3 samples/s +Step 36170/50462 | Loss: 2.5351 | LR: 9.52e-05 | 3114.8ms/step | 19.3 samples/s +Step 36180/50462 | Loss: 2.5309 | LR: 9.51e-05 | 3113.5ms/step | 19.3 samples/s +Step 36190/50462 | Loss: 2.5422 | LR: 9.50e-05 | 3112.6ms/step | 19.3 samples/s +Step 36200/50462 | Loss: 2.5608 | LR: 9.49e-05 | 3112.0ms/step | 19.3 samples/s +Step 36210/50462 | Loss: 2.5495 | LR: 9.47e-05 | 3111.2ms/step | 19.3 samples/s +Step 36220/50462 | Loss: 2.5226 | LR: 9.46e-05 | 3113.7ms/step | 19.3 samples/s +Step 36230/50462 | Loss: 2.5358 | LR: 9.45e-05 | 3116.0ms/step | 19.3 samples/s +Step 36240/50462 | Loss: 2.5548 | LR: 9.44e-05 | 3114.8ms/step | 19.3 samples/s +Step 36250/50462 | Loss: 2.5198 | LR: 9.42e-05 | 3109.0ms/step | 19.3 samples/s +Validation at step 36250: Loss = 2.5455, Perplexity = 12.75 +Step 36260/50462 | Loss: 2.5414 | LR: 9.41e-05 | 4764.6ms/step | 12.6 samples/s +Step 36270/50462 | Loss: 2.5161 | LR: 9.40e-05 | 3109.9ms/step | 19.3 samples/s +Step 36280/50462 | Loss: 2.5368 | LR: 9.39e-05 | 3112.8ms/step | 19.3 samples/s +Step 36290/50462 | Loss: 2.5034 | LR: 9.37e-05 | 3112.5ms/step | 19.3 samples/s +Step 36300/50462 | Loss: 2.4930 | LR: 9.36e-05 | 3111.8ms/step | 19.3 samples/s +Step 36310/50462 | Loss: 2.5196 | LR: 9.35e-05 | 3112.7ms/step | 19.3 samples/s +Step 36320/50462 | Loss: 2.5292 | LR: 9.34e-05 | 3113.0ms/step | 19.3 samples/s +Step 36330/50462 | Loss: 2.5415 | LR: 9.32e-05 | 3116.4ms/step | 19.3 samples/s +Step 36340/50462 | Loss: 2.5403 | LR: 9.31e-05 | 3118.2ms/step | 19.2 samples/s +Step 36350/50462 | Loss: 2.5355 | LR: 9.30e-05 | 3118.3ms/step | 19.2 samples/s +Step 36360/50462 | Loss: 2.5436 | LR: 9.29e-05 | 3118.0ms/step | 19.2 samples/s +Step 36370/50462 | Loss: 2.5750 | LR: 9.28e-05 | 3118.1ms/step | 19.2 samples/s +Step 36380/50462 | Loss: 2.5113 | LR: 9.26e-05 | 3119.8ms/step | 19.2 samples/s +Step 36390/50462 | Loss: 2.5150 | LR: 9.25e-05 | 3116.5ms/step | 19.3 samples/s +Step 36400/50462 | Loss: 2.5466 | LR: 9.24e-05 | 3115.8ms/step | 19.3 samples/s +Step 36410/50462 | Loss: 2.5187 | LR: 9.23e-05 | 3113.4ms/step | 19.3 samples/s +Step 36420/50462 | Loss: 2.5509 | LR: 9.21e-05 | 3111.2ms/step | 19.3 samples/s +Step 36430/50462 | Loss: 2.5604 | LR: 9.20e-05 | 3113.2ms/step | 19.3 samples/s +Step 36440/50462 | Loss: 2.5143 | LR: 9.19e-05 | 3115.5ms/step | 19.3 samples/s +Step 36450/50462 | Loss: 2.5134 | LR: 9.18e-05 | 3114.7ms/step | 19.3 samples/s +Step 36460/50462 | Loss: 2.5392 | LR: 9.17e-05 | 3114.4ms/step | 19.3 samples/s +Step 36470/50462 | Loss: 2.5358 | LR: 9.15e-05 | 3113.7ms/step | 19.3 samples/s +Step 36480/50462 | Loss: 2.5343 | LR: 9.14e-05 | 3115.2ms/step | 19.3 samples/s +Step 36490/50462 | Loss: 2.5565 | LR: 9.13e-05 | 3109.7ms/step | 19.3 samples/s +Step 36500/50462 | Loss: 2.5556 | LR: 9.12e-05 | 3111.3ms/step | 19.3 samples/s +Validation at step 36500: Loss = 2.5452, Perplexity = 12.75 +Step 36510/50462 | Loss: 2.5218 | LR: 9.10e-05 | 4767.7ms/step | 12.6 samples/s +Step 36520/50462 | Loss: 2.5175 | LR: 9.09e-05 | 3111.5ms/step | 19.3 samples/s +Step 36530/50462 | Loss: 2.5689 | LR: 9.08e-05 | 3111.9ms/step | 19.3 samples/s +Step 36540/50462 | Loss: 2.5312 | LR: 9.07e-05 | 3113.0ms/step | 19.3 samples/s +Step 36550/50462 | Loss: 2.5324 | LR: 9.06e-05 | 3112.6ms/step | 19.3 samples/s +Step 36560/50462 | Loss: 2.5255 | LR: 9.04e-05 | 3115.7ms/step | 19.3 samples/s +Step 36570/50462 | Loss: 2.5188 | LR: 9.03e-05 | 3111.8ms/step | 19.3 samples/s +Step 36580/50462 | Loss: 2.5294 | LR: 9.02e-05 | 3113.4ms/step | 19.3 samples/s +Step 36590/50462 | Loss: 2.5475 | LR: 9.01e-05 | 3111.9ms/step | 19.3 samples/s +Step 36600/50462 | Loss: 2.5085 | LR: 8.99e-05 | 3109.1ms/step | 19.3 samples/s +Step 36610/50462 | Loss: 2.5193 | LR: 8.98e-05 | 3109.6ms/step | 19.3 samples/s +Step 36620/50462 | Loss: 2.5226 | LR: 8.97e-05 | 3110.6ms/step | 19.3 samples/s +Step 36630/50462 | Loss: 2.5125 | LR: 8.96e-05 | 3110.4ms/step | 19.3 samples/s +Step 36640/50462 | Loss: 2.5525 | LR: 8.95e-05 | 3116.2ms/step | 19.3 samples/s +Step 36650/50462 | Loss: 2.5113 | LR: 8.93e-05 | 3111.1ms/step | 19.3 samples/s +Step 36660/50462 | Loss: 2.5473 | LR: 8.92e-05 | 3109.3ms/step | 19.3 samples/s +Step 36670/50462 | Loss: 2.5146 | LR: 8.91e-05 | 3112.8ms/step | 19.3 samples/s +Step 36680/50462 | Loss: 2.5211 | LR: 8.90e-05 | 3112.4ms/step | 19.3 samples/s +Step 36690/50462 | Loss: 2.5643 | LR: 8.89e-05 | 3114.1ms/step | 19.3 samples/s +Step 36700/50462 | Loss: 2.5299 | LR: 8.87e-05 | 3114.8ms/step | 19.3 samples/s +Step 36710/50462 | Loss: 2.5123 | LR: 8.86e-05 | 3117.9ms/step | 19.2 samples/s +Step 36720/50462 | Loss: 2.5199 | LR: 8.85e-05 | 3110.1ms/step | 19.3 samples/s +Step 36730/50462 | Loss: 2.5610 | LR: 8.84e-05 | 3111.8ms/step | 19.3 samples/s +Step 36740/50462 | Loss: 2.5362 | LR: 8.83e-05 | 3116.3ms/step | 19.3 samples/s +Step 36750/50462 | Loss: 2.5352 | LR: 8.81e-05 | 3114.6ms/step | 19.3 samples/s +Validation at step 36750: Loss = 2.5446, Perplexity = 12.74 +Step 36760/50462 | Loss: 2.5127 | LR: 8.80e-05 | 4771.5ms/step | 12.6 samples/s +Step 36770/50462 | Loss: 2.5313 | LR: 8.79e-05 | 3109.6ms/step | 19.3 samples/s +Step 36780/50462 | Loss: 2.5606 | LR: 8.78e-05 | 3113.8ms/step | 19.3 samples/s +Step 36790/50462 | Loss: 2.5156 | LR: 8.77e-05 | 3113.9ms/step | 19.3 samples/s +Step 36800/50462 | Loss: 2.5227 | LR: 8.75e-05 | 3116.2ms/step | 19.3 samples/s +Step 36810/50462 | Loss: 2.5450 | LR: 8.74e-05 | 3117.7ms/step | 19.2 samples/s +Step 36820/50462 | Loss: 2.5474 | LR: 8.73e-05 | 3116.2ms/step | 19.3 samples/s +Step 36830/50462 | Loss: 2.5487 | LR: 8.72e-05 | 3112.7ms/step | 19.3 samples/s +Step 36840/50462 | Loss: 2.5456 | LR: 8.71e-05 | 3114.4ms/step | 19.3 samples/s +Step 36850/50462 | Loss: 2.5429 | LR: 8.69e-05 | 3113.5ms/step | 19.3 samples/s +Step 36860/50462 | Loss: 2.5512 | LR: 8.68e-05 | 3110.6ms/step | 19.3 samples/s +Step 36870/50462 | Loss: 2.5184 | LR: 8.67e-05 | 3099.7ms/step | 19.4 samples/s +Step 36880/50462 | Loss: 2.5162 | LR: 8.66e-05 | 3093.8ms/step | 19.4 samples/s +Step 36890/50462 | Loss: 2.5019 | LR: 8.65e-05 | 3106.0ms/step | 19.3 samples/s +Step 36900/50462 | Loss: 2.5117 | LR: 8.63e-05 | 3108.8ms/step | 19.3 samples/s +Step 36910/50462 | Loss: 2.5534 | LR: 8.62e-05 | 3113.4ms/step | 19.3 samples/s +Step 36920/50462 | Loss: 2.5329 | LR: 8.61e-05 | 3113.9ms/step | 19.3 samples/s +Step 36930/50462 | Loss: 2.5311 | LR: 8.60e-05 | 3116.8ms/step | 19.3 samples/s +Step 36940/50462 | Loss: 2.5341 | LR: 8.59e-05 | 3117.9ms/step | 19.2 samples/s +Step 36950/50462 | Loss: 2.5350 | LR: 8.57e-05 | 3634.9ms/step | 16.5 samples/s +Step 36960/50462 | Loss: 2.5148 | LR: 8.56e-05 | 3100.9ms/step | 19.3 samples/s +Step 36970/50462 | Loss: 2.5192 | LR: 8.55e-05 | 3105.7ms/step | 19.3 samples/s +Step 36980/50462 | Loss: 2.4973 | LR: 8.54e-05 | 3108.8ms/step | 19.3 samples/s +Step 36990/50462 | Loss: 2.4977 | LR: 8.53e-05 | 3114.6ms/step | 19.3 samples/s +Step 37000/50462 | Loss: 2.5198 | LR: 8.51e-05 | 3113.8ms/step | 19.3 samples/s +Validation at step 37000: Loss = 2.5440, Perplexity = 12.73 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-37000 +Step 37010/50462 | Loss: 2.5596 | LR: 8.50e-05 | 5706.7ms/step | 10.5 samples/s +Step 37020/50462 | Loss: 2.5297 | LR: 8.49e-05 | 3101.1ms/step | 19.3 samples/s +Step 37030/50462 | Loss: 2.5183 | LR: 8.48e-05 | 3108.5ms/step | 19.3 samples/s +Step 37040/50462 | Loss: 2.4896 | LR: 8.47e-05 | 3106.2ms/step | 19.3 samples/s +Step 37050/50462 | Loss: 2.5543 | LR: 8.46e-05 | 3110.1ms/step | 19.3 samples/s +Step 37060/50462 | Loss: 2.5193 | LR: 8.44e-05 | 3114.3ms/step | 19.3 samples/s +Step 37070/50462 | Loss: 2.5173 | LR: 8.43e-05 | 3112.4ms/step | 19.3 samples/s +Step 37080/50462 | Loss: 2.5613 | LR: 8.42e-05 | 3111.9ms/step | 19.3 samples/s +Step 37090/50462 | Loss: 2.5290 | LR: 8.41e-05 | 3112.0ms/step | 19.3 samples/s +Step 37100/50462 | Loss: 2.5312 | LR: 8.40e-05 | 3114.7ms/step | 19.3 samples/s +Step 37110/50462 | Loss: 2.5287 | LR: 8.38e-05 | 3111.8ms/step | 19.3 samples/s +Step 37120/50462 | Loss: 2.5149 | LR: 8.37e-05 | 3114.1ms/step | 19.3 samples/s +Step 37130/50462 | Loss: 2.5419 | LR: 8.36e-05 | 3111.0ms/step | 19.3 samples/s +Step 37140/50462 | Loss: 2.5270 | LR: 8.35e-05 | 3111.0ms/step | 19.3 samples/s +Step 37150/50462 | Loss: 2.5593 | LR: 8.34e-05 | 3112.2ms/step | 19.3 samples/s +Step 37160/50462 | Loss: 2.5375 | LR: 8.33e-05 | 3110.0ms/step | 19.3 samples/s +Step 37170/50462 | Loss: 2.5444 | LR: 8.31e-05 | 3112.4ms/step | 19.3 samples/s +Step 37180/50462 | Loss: 2.5183 | LR: 8.30e-05 | 3113.8ms/step | 19.3 samples/s +Step 37190/50462 | Loss: 2.5337 | LR: 8.29e-05 | 3115.8ms/step | 19.3 samples/s +Step 37200/50462 | Loss: 2.5335 | LR: 8.28e-05 | 3115.0ms/step | 19.3 samples/s +Step 37210/50462 | Loss: 2.4847 | LR: 8.27e-05 | 3108.8ms/step | 19.3 samples/s +Step 37220/50462 | Loss: 2.5309 | LR: 8.25e-05 | 3107.1ms/step | 19.3 samples/s +Step 37230/50462 | Loss: 2.5186 | LR: 8.24e-05 | 3111.3ms/step | 19.3 samples/s +Step 37240/50462 | Loss: 2.5195 | LR: 8.23e-05 | 3110.4ms/step | 19.3 samples/s +Step 37250/50462 | Loss: 2.5254 | LR: 8.22e-05 | 3109.2ms/step | 19.3 samples/s +Validation at step 37250: Loss = 2.5433, Perplexity = 12.72 +Step 37260/50462 | Loss: 2.5328 | LR: 8.21e-05 | 4765.5ms/step | 12.6 samples/s +Step 37270/50462 | Loss: 2.5461 | LR: 8.20e-05 | 3112.7ms/step | 19.3 samples/s +Step 37280/50462 | Loss: 2.5437 | LR: 8.18e-05 | 3111.8ms/step | 19.3 samples/s +Step 37290/50462 | Loss: 2.5202 | LR: 8.17e-05 | 3112.8ms/step | 19.3 samples/s +Step 37300/50462 | Loss: 2.5335 | LR: 8.16e-05 | 3112.4ms/step | 19.3 samples/s +Step 37310/50462 | Loss: 2.5133 | LR: 8.15e-05 | 3111.4ms/step | 19.3 samples/s +Step 37320/50462 | Loss: 2.5353 | LR: 8.14e-05 | 3111.8ms/step | 19.3 samples/s +Step 37330/50462 | Loss: 2.5413 | LR: 8.13e-05 | 3111.7ms/step | 19.3 samples/s +Step 37340/50462 | Loss: 2.4992 | LR: 8.11e-05 | 3109.6ms/step | 19.3 samples/s +Step 37350/50462 | Loss: 2.5434 | LR: 8.10e-05 | 3107.3ms/step | 19.3 samples/s +Step 37360/50462 | Loss: 2.5475 | LR: 8.09e-05 | 3109.7ms/step | 19.3 samples/s +Step 37370/50462 | Loss: 2.5187 | LR: 8.08e-05 | 3110.5ms/step | 19.3 samples/s +Step 37380/50462 | Loss: 2.5329 | LR: 8.07e-05 | 3110.5ms/step | 19.3 samples/s +Step 37390/50462 | Loss: 2.5437 | LR: 8.06e-05 | 3112.7ms/step | 19.3 samples/s +Step 37400/50462 | Loss: 2.5354 | LR: 8.04e-05 | 3109.7ms/step | 19.3 samples/s +Step 37410/50462 | Loss: 2.5310 | LR: 8.03e-05 | 3112.6ms/step | 19.3 samples/s +Step 37420/50462 | Loss: 2.5368 | LR: 8.02e-05 | 3115.3ms/step | 19.3 samples/s +Step 37430/50462 | Loss: 2.5394 | LR: 8.01e-05 | 3115.1ms/step | 19.3 samples/s +Step 37440/50462 | Loss: 2.5250 | LR: 8.00e-05 | 3115.8ms/step | 19.3 samples/s +Step 37450/50462 | Loss: 2.5605 | LR: 7.99e-05 | 3114.7ms/step | 19.3 samples/s +Step 37460/50462 | Loss: 2.5254 | LR: 7.98e-05 | 3111.8ms/step | 19.3 samples/s +Step 37470/50462 | Loss: 2.5476 | LR: 7.96e-05 | 3111.4ms/step | 19.3 samples/s +Step 37480/50462 | Loss: 2.5238 | LR: 7.95e-05 | 3110.1ms/step | 19.3 samples/s +Step 37490/50462 | Loss: 2.5393 | LR: 7.94e-05 | 3110.9ms/step | 19.3 samples/s +Step 37500/50462 | Loss: 2.5222 | LR: 7.93e-05 | 3630.4ms/step | 16.5 samples/s +Validation at step 37500: Loss = 2.5431, Perplexity = 12.72 +Step 37510/50462 | Loss: 2.5343 | LR: 7.92e-05 | 4746.8ms/step | 12.6 samples/s +Step 37520/50462 | Loss: 2.5116 | LR: 7.91e-05 | 3106.6ms/step | 19.3 samples/s +Step 37530/50462 | Loss: 2.5327 | LR: 7.89e-05 | 3108.4ms/step | 19.3 samples/s +Step 37540/50462 | Loss: 2.5295 | LR: 7.88e-05 | 3115.0ms/step | 19.3 samples/s +Step 37550/50462 | Loss: 2.5470 | LR: 7.87e-05 | 3113.5ms/step | 19.3 samples/s +Step 37560/50462 | Loss: 2.5236 | LR: 7.86e-05 | 3113.9ms/step | 19.3 samples/s +Step 37570/50462 | Loss: 2.5355 | LR: 7.85e-05 | 3114.2ms/step | 19.3 samples/s +Step 37580/50462 | Loss: 2.5253 | LR: 7.84e-05 | 3114.8ms/step | 19.3 samples/s +Step 37590/50462 | Loss: 2.5271 | LR: 7.83e-05 | 3616.8ms/step | 16.6 samples/s +Step 37600/50462 | Loss: 2.5313 | LR: 7.81e-05 | 3096.6ms/step | 19.4 samples/s +Step 37610/50462 | Loss: 2.5385 | LR: 7.80e-05 | 3103.3ms/step | 19.3 samples/s +Step 37620/50462 | Loss: 2.5434 | LR: 7.79e-05 | 3108.4ms/step | 19.3 samples/s +Step 37630/50462 | Loss: 2.5183 | LR: 7.78e-05 | 3111.5ms/step | 19.3 samples/s +Step 37640/50462 | Loss: 2.5289 | LR: 7.77e-05 | 3110.2ms/step | 19.3 samples/s +Step 37650/50462 | Loss: 2.5292 | LR: 7.76e-05 | 3112.5ms/step | 19.3 samples/s +Step 37660/50462 | Loss: 2.5340 | LR: 7.75e-05 | 3114.9ms/step | 19.3 samples/s +Step 37670/50462 | Loss: 2.5022 | LR: 7.73e-05 | 3113.2ms/step | 19.3 samples/s +Step 37680/50462 | Loss: 2.5311 | LR: 7.72e-05 | 3111.0ms/step | 19.3 samples/s +Step 37690/50462 | Loss: 2.5485 | LR: 7.71e-05 | 3113.2ms/step | 19.3 samples/s +Step 37700/50462 | Loss: 2.5184 | LR: 7.70e-05 | 3113.5ms/step | 19.3 samples/s +Step 37710/50462 | Loss: 2.5574 | LR: 7.69e-05 | 3110.7ms/step | 19.3 samples/s +Step 37720/50462 | Loss: 2.4925 | LR: 7.68e-05 | 3111.7ms/step | 19.3 samples/s +Step 37730/50462 | Loss: 2.5409 | LR: 7.67e-05 | 3116.2ms/step | 19.3 samples/s +Step 37740/50462 | Loss: 2.5522 | LR: 7.65e-05 | 3114.9ms/step | 19.3 samples/s +Step 37750/50462 | Loss: 2.5070 | LR: 7.64e-05 | 3113.5ms/step | 19.3 samples/s +Validation at step 37750: Loss = 2.5428, Perplexity = 12.72 +Step 37760/50462 | Loss: 2.5346 | LR: 7.63e-05 | 4771.8ms/step | 12.6 samples/s +Step 37770/50462 | Loss: 2.5360 | LR: 7.62e-05 | 3112.8ms/step | 19.3 samples/s +Step 37780/50462 | Loss: 2.5100 | LR: 7.61e-05 | 3116.8ms/step | 19.3 samples/s +Step 37790/50462 | Loss: 2.4926 | LR: 7.60e-05 | 3109.6ms/step | 19.3 samples/s +Step 37800/50462 | Loss: 2.5110 | LR: 7.59e-05 | 3112.7ms/step | 19.3 samples/s +Step 37810/50462 | Loss: 2.5229 | LR: 7.57e-05 | 3111.6ms/step | 19.3 samples/s +Step 37820/50462 | Loss: 2.5407 | LR: 7.56e-05 | 3113.8ms/step | 19.3 samples/s +Step 37830/50462 | Loss: 2.5416 | LR: 7.55e-05 | 3116.1ms/step | 19.3 samples/s +Step 37840/50462 | Loss: 2.5481 | LR: 7.54e-05 | 3113.3ms/step | 19.3 samples/s +Step 37850/50462 | Loss: 2.5479 | LR: 7.53e-05 | 3113.4ms/step | 19.3 samples/s +Step 37860/50462 | Loss: 2.5496 | LR: 7.52e-05 | 3117.5ms/step | 19.2 samples/s +Step 37870/50462 | Loss: 2.5253 | LR: 7.51e-05 | 3128.3ms/step | 19.2 samples/s +Step 37880/50462 | Loss: 2.5571 | LR: 7.50e-05 | 3128.7ms/step | 19.2 samples/s +Step 37890/50462 | Loss: 2.5355 | LR: 7.48e-05 | 3137.0ms/step | 19.1 samples/s +Step 37900/50462 | Loss: 2.5061 | LR: 7.47e-05 | 3132.0ms/step | 19.2 samples/s +Step 37910/50462 | Loss: 2.5337 | LR: 7.46e-05 | 3125.8ms/step | 19.2 samples/s +Step 37920/50462 | Loss: 2.5047 | LR: 7.45e-05 | 3130.3ms/step | 19.2 samples/s +Step 37930/50462 | Loss: 2.5154 | LR: 7.44e-05 | 3126.9ms/step | 19.2 samples/s +Step 37940/50462 | Loss: 2.5282 | LR: 7.43e-05 | 3135.2ms/step | 19.1 samples/s +Step 37950/50462 | Loss: 2.5409 | LR: 7.42e-05 | 3126.4ms/step | 19.2 samples/s +Step 37960/50462 | Loss: 2.5429 | LR: 7.41e-05 | 3123.2ms/step | 19.2 samples/s +Step 37970/50462 | Loss: 2.5460 | LR: 7.39e-05 | 3131.4ms/step | 19.2 samples/s +Step 37980/50462 | Loss: 2.5140 | LR: 7.38e-05 | 3129.2ms/step | 19.2 samples/s +Step 37990/50462 | Loss: 2.5034 | LR: 7.37e-05 | 3127.8ms/step | 19.2 samples/s +Step 38000/50462 | Loss: 2.5278 | LR: 7.36e-05 | 3129.7ms/step | 19.2 samples/s +Validation at step 38000: Loss = 2.5419, Perplexity = 12.70 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-38000 +Step 38010/50462 | Loss: 2.5312 | LR: 7.35e-05 | 5727.4ms/step | 10.5 samples/s +Step 38020/50462 | Loss: 2.5203 | LR: 7.34e-05 | 3112.6ms/step | 19.3 samples/s +Step 38030/50462 | Loss: 2.5474 | LR: 7.33e-05 | 3124.0ms/step | 19.2 samples/s +Step 38040/50462 | Loss: 2.5155 | LR: 7.32e-05 | 3126.2ms/step | 19.2 samples/s +Step 38050/50462 | Loss: 2.5328 | LR: 7.30e-05 | 3132.4ms/step | 19.2 samples/s +Step 38060/50462 | Loss: 2.5300 | LR: 7.29e-05 | 3123.7ms/step | 19.2 samples/s +Step 38070/50462 | Loss: 2.5702 | LR: 7.28e-05 | 3113.5ms/step | 19.3 samples/s +Step 38080/50462 | Loss: 2.5293 | LR: 7.27e-05 | 3117.1ms/step | 19.2 samples/s +Step 38090/50462 | Loss: 2.5281 | LR: 7.26e-05 | 3115.6ms/step | 19.3 samples/s +Step 38100/50462 | Loss: 2.5510 | LR: 7.25e-05 | 3118.6ms/step | 19.2 samples/s +Step 38110/50462 | Loss: 2.4871 | LR: 7.24e-05 | 3118.5ms/step | 19.2 samples/s +Step 38120/50462 | Loss: 2.5363 | LR: 7.23e-05 | 3115.6ms/step | 19.3 samples/s +Step 38130/50462 | Loss: 2.5344 | LR: 7.22e-05 | 3118.2ms/step | 19.2 samples/s +Step 38140/50462 | Loss: 2.5374 | LR: 7.20e-05 | 3121.5ms/step | 19.2 samples/s +Step 38150/50462 | Loss: 2.5576 | LR: 7.19e-05 | 3114.8ms/step | 19.3 samples/s +Step 38160/50462 | Loss: 2.5138 | LR: 7.18e-05 | 3121.4ms/step | 19.2 samples/s +Step 38170/50462 | Loss: 2.5435 | LR: 7.17e-05 | 3115.7ms/step | 19.3 samples/s +Step 38180/50462 | Loss: 2.5387 | LR: 7.16e-05 | 3120.2ms/step | 19.2 samples/s +Step 38190/50462 | Loss: 2.5263 | LR: 7.15e-05 | 3118.0ms/step | 19.2 samples/s +Step 38200/50462 | Loss: 2.5620 | LR: 7.14e-05 | 3118.0ms/step | 19.2 samples/s +Step 38210/50462 | Loss: 2.5523 | LR: 7.13e-05 | 3120.9ms/step | 19.2 samples/s +Step 38220/50462 | Loss: 2.5207 | LR: 7.12e-05 | 3127.4ms/step | 19.2 samples/s +Step 38230/50462 | Loss: 2.5361 | LR: 7.10e-05 | 3121.0ms/step | 19.2 samples/s +Step 38240/50462 | Loss: 2.5342 | LR: 7.09e-05 | 3122.7ms/step | 19.2 samples/s +Step 38250/50462 | Loss: 2.5386 | LR: 7.08e-05 | 3130.1ms/step | 19.2 samples/s +Validation at step 38250: Loss = 2.5418, Perplexity = 12.70 +Step 38260/50462 | Loss: 2.5221 | LR: 7.07e-05 | 4772.6ms/step | 12.6 samples/s +Step 38270/50462 | Loss: 2.5589 | LR: 7.06e-05 | 3125.2ms/step | 19.2 samples/s +Step 38280/50462 | Loss: 2.5012 | LR: 7.05e-05 | 3119.0ms/step | 19.2 samples/s +Step 38290/50462 | Loss: 2.5224 | LR: 7.04e-05 | 3123.0ms/step | 19.2 samples/s +Step 38300/50462 | Loss: 2.5310 | LR: 7.03e-05 | 3117.9ms/step | 19.2 samples/s +Step 38310/50462 | Loss: 2.5574 | LR: 7.02e-05 | 3119.2ms/step | 19.2 samples/s +Step 38320/50462 | Loss: 2.5642 | LR: 7.01e-05 | 3123.8ms/step | 19.2 samples/s +Step 38330/50462 | Loss: 2.5214 | LR: 6.99e-05 | 3120.0ms/step | 19.2 samples/s +Step 38340/50462 | Loss: 2.5246 | LR: 6.98e-05 | 3120.7ms/step | 19.2 samples/s +Step 38350/50462 | Loss: 2.5432 | LR: 6.97e-05 | 3124.6ms/step | 19.2 samples/s +Step 38360/50462 | Loss: 2.5404 | LR: 6.96e-05 | 3120.6ms/step | 19.2 samples/s +Step 38370/50462 | Loss: 2.5387 | LR: 6.95e-05 | 3119.9ms/step | 19.2 samples/s +Step 38380/50462 | Loss: 2.5185 | LR: 6.94e-05 | 3114.0ms/step | 19.3 samples/s +Step 38390/50462 | Loss: 2.5365 | LR: 6.93e-05 | 3117.9ms/step | 19.2 samples/s +Step 38400/50462 | Loss: 2.5276 | LR: 6.92e-05 | 3121.5ms/step | 19.2 samples/s +Step 38410/50462 | Loss: 2.5310 | LR: 6.91e-05 | 3134.4ms/step | 19.1 samples/s +Step 38420/50462 | Loss: 2.5465 | LR: 6.90e-05 | 3122.2ms/step | 19.2 samples/s +Step 38430/50462 | Loss: 2.5316 | LR: 6.89e-05 | 3135.8ms/step | 19.1 samples/s +Step 38440/50462 | Loss: 2.5193 | LR: 6.87e-05 | 3126.4ms/step | 19.2 samples/s +Step 38450/50462 | Loss: 2.5417 | LR: 6.86e-05 | 3133.6ms/step | 19.1 samples/s +Step 38460/50462 | Loss: 2.5291 | LR: 6.85e-05 | 3132.0ms/step | 19.2 samples/s +Step 38470/50462 | Loss: 2.5281 | LR: 6.84e-05 | 3138.0ms/step | 19.1 samples/s +Step 38480/50462 | Loss: 2.5173 | LR: 6.83e-05 | 3131.5ms/step | 19.2 samples/s +Step 38490/50462 | Loss: 2.5551 | LR: 6.82e-05 | 3143.6ms/step | 19.1 samples/s +Step 38500/50462 | Loss: 2.5333 | LR: 6.81e-05 | 3130.5ms/step | 19.2 samples/s +Validation at step 38500: Loss = 2.5415, Perplexity = 12.70 +Step 38510/50462 | Loss: 2.5292 | LR: 6.80e-05 | 4800.4ms/step | 12.5 samples/s +Step 38520/50462 | Loss: 2.5176 | LR: 6.79e-05 | 3124.4ms/step | 19.2 samples/s +Step 38530/50462 | Loss: 2.5236 | LR: 6.78e-05 | 3138.5ms/step | 19.1 samples/s +Step 38540/50462 | Loss: 2.5025 | LR: 6.77e-05 | 3125.4ms/step | 19.2 samples/s +Step 38550/50462 | Loss: 2.5388 | LR: 6.76e-05 | 3131.5ms/step | 19.2 samples/s +Step 38560/50462 | Loss: 2.5125 | LR: 6.74e-05 | 3125.8ms/step | 19.2 samples/s +Step 38570/50462 | Loss: 2.5164 | LR: 6.73e-05 | 3122.2ms/step | 19.2 samples/s +Step 38580/50462 | Loss: 2.5263 | LR: 6.72e-05 | 3132.4ms/step | 19.2 samples/s +Step 38590/50462 | Loss: 2.5279 | LR: 6.71e-05 | 3133.6ms/step | 19.1 samples/s +Step 38600/50462 | Loss: 2.5332 | LR: 6.70e-05 | 3140.5ms/step | 19.1 samples/s +Step 38610/50462 | Loss: 2.5094 | LR: 6.69e-05 | 3128.9ms/step | 19.2 samples/s +Step 38620/50462 | Loss: 2.5167 | LR: 6.68e-05 | 3134.0ms/step | 19.1 samples/s +Step 38630/50462 | Loss: 2.5401 | LR: 6.67e-05 | 3127.7ms/step | 19.2 samples/s +Step 38640/50462 | Loss: 2.5169 | LR: 6.66e-05 | 3132.8ms/step | 19.2 samples/s +Step 38650/50462 | Loss: 2.5460 | LR: 6.65e-05 | 3138.7ms/step | 19.1 samples/s +Step 38660/50462 | Loss: 2.5145 | LR: 6.64e-05 | 3130.5ms/step | 19.2 samples/s +Step 38670/50462 | Loss: 2.5228 | LR: 6.63e-05 | 3131.2ms/step | 19.2 samples/s +Step 38680/50462 | Loss: 2.5120 | LR: 6.62e-05 | 3129.3ms/step | 19.2 samples/s +Step 38690/50462 | Loss: 2.4909 | LR: 6.60e-05 | 3120.7ms/step | 19.2 samples/s +Step 38700/50462 | Loss: 2.5424 | LR: 6.59e-05 | 3129.7ms/step | 19.2 samples/s +Step 38710/50462 | Loss: 2.5170 | LR: 6.58e-05 | 3138.0ms/step | 19.1 samples/s +Step 38720/50462 | Loss: 2.5185 | LR: 6.57e-05 | 3136.4ms/step | 19.1 samples/s +Step 38730/50462 | Loss: 2.5598 | LR: 6.56e-05 | 3124.2ms/step | 19.2 samples/s +Step 38740/50462 | Loss: 2.5273 | LR: 6.55e-05 | 3139.4ms/step | 19.1 samples/s +Step 38750/50462 | Loss: 2.5314 | LR: 6.54e-05 | 3134.4ms/step | 19.1 samples/s +Validation at step 38750: Loss = 2.5410, Perplexity = 12.69 +Step 38760/50462 | Loss: 2.5331 | LR: 6.53e-05 | 4798.3ms/step | 12.5 samples/s +Step 38770/50462 | Loss: 2.5486 | LR: 6.52e-05 | 3120.9ms/step | 19.2 samples/s +Step 38780/50462 | Loss: 2.5192 | LR: 6.51e-05 | 3132.6ms/step | 19.2 samples/s +Step 38790/50462 | Loss: 2.5714 | LR: 6.50e-05 | 3134.0ms/step | 19.1 samples/s +Step 38800/50462 | Loss: 2.5361 | LR: 6.49e-05 | 3135.9ms/step | 19.1 samples/s +Step 38810/50462 | Loss: 2.5203 | LR: 6.48e-05 | 3128.1ms/step | 19.2 samples/s +Step 38820/50462 | Loss: 2.5250 | LR: 6.47e-05 | 3142.2ms/step | 19.1 samples/s +Step 38830/50462 | Loss: 2.5329 | LR: 6.46e-05 | 3124.5ms/step | 19.2 samples/s +Step 38840/50462 | Loss: 2.5440 | LR: 6.45e-05 | 3138.3ms/step | 19.1 samples/s +Step 38850/50462 | Loss: 2.5530 | LR: 6.43e-05 | 3130.8ms/step | 19.2 samples/s +Step 38860/50462 | Loss: 2.5160 | LR: 6.42e-05 | 3131.8ms/step | 19.2 samples/s +Step 38870/50462 | Loss: 2.5314 | LR: 6.41e-05 | 3133.1ms/step | 19.2 samples/s +Step 38880/50462 | Loss: 2.5266 | LR: 6.40e-05 | 3129.6ms/step | 19.2 samples/s +Step 38890/50462 | Loss: 2.5379 | LR: 6.39e-05 | 3134.6ms/step | 19.1 samples/s +Step 38900/50462 | Loss: 2.5347 | LR: 6.38e-05 | 3139.2ms/step | 19.1 samples/s +Step 38910/50462 | Loss: 2.5360 | LR: 6.37e-05 | 3123.8ms/step | 19.2 samples/s +Step 38920/50462 | Loss: 2.5267 | LR: 6.36e-05 | 3133.8ms/step | 19.1 samples/s +Step 38930/50462 | Loss: 2.5423 | LR: 6.35e-05 | 3129.4ms/step | 19.2 samples/s +Step 38940/50462 | Loss: 2.5344 | LR: 6.34e-05 | 3138.8ms/step | 19.1 samples/s +Step 38950/50462 | Loss: 2.5423 | LR: 6.33e-05 | 3137.1ms/step | 19.1 samples/s +Step 38960/50462 | Loss: 2.5256 | LR: 6.32e-05 | 3123.8ms/step | 19.2 samples/s +Step 38970/50462 | Loss: 2.5201 | LR: 6.31e-05 | 3127.4ms/step | 19.2 samples/s +Step 38980/50462 | Loss: 2.5291 | LR: 6.30e-05 | 3129.0ms/step | 19.2 samples/s +Step 38990/50462 | Loss: 2.5036 | LR: 6.29e-05 | 3130.0ms/step | 19.2 samples/s +Step 39000/50462 | Loss: 2.5353 | LR: 6.28e-05 | 3134.9ms/step | 19.1 samples/s +Validation at step 39000: Loss = 2.5408, Perplexity = 12.69 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-39000 +Step 39010/50462 | Loss: 2.5510 | LR: 6.27e-05 | 5782.9ms/step | 10.4 samples/s +Step 39020/50462 | Loss: 2.5057 | LR: 6.26e-05 | 3121.3ms/step | 19.2 samples/s +Step 39030/50462 | Loss: 2.5351 | LR: 6.25e-05 | 3126.3ms/step | 19.2 samples/s +Step 39040/50462 | Loss: 2.5240 | LR: 6.23e-05 | 3123.0ms/step | 19.2 samples/s +Step 39050/50462 | Loss: 2.5026 | LR: 6.22e-05 | 3131.1ms/step | 19.2 samples/s +Step 39060/50462 | Loss: 2.5150 | LR: 6.21e-05 | 3141.7ms/step | 19.1 samples/s +Step 39070/50462 | Loss: 2.5367 | LR: 6.20e-05 | 3760.1ms/step | 16.0 samples/s +Step 39080/50462 | Loss: 2.5523 | LR: 6.19e-05 | 3116.4ms/step | 19.3 samples/s +Step 39090/50462 | Loss: 2.5107 | LR: 6.18e-05 | 3126.5ms/step | 19.2 samples/s +Step 39100/50462 | Loss: 2.5328 | LR: 6.17e-05 | 3124.8ms/step | 19.2 samples/s +Step 39110/50462 | Loss: 2.5062 | LR: 6.16e-05 | 3143.7ms/step | 19.1 samples/s +Step 39120/50462 | Loss: 2.5127 | LR: 6.15e-05 | 3129.9ms/step | 19.2 samples/s +Step 39130/50462 | Loss: 2.5509 | LR: 6.14e-05 | 3120.6ms/step | 19.2 samples/s +Step 39140/50462 | Loss: 2.5228 | LR: 6.13e-05 | 3123.2ms/step | 19.2 samples/s +Step 39150/50462 | Loss: 2.5272 | LR: 6.12e-05 | 3140.4ms/step | 19.1 samples/s +Step 39160/50462 | Loss: 2.5147 | LR: 6.11e-05 | 3130.1ms/step | 19.2 samples/s +Step 39170/50462 | Loss: 2.5422 | LR: 6.10e-05 | 3137.1ms/step | 19.1 samples/s +Step 39180/50462 | Loss: 2.5533 | LR: 6.09e-05 | 3130.3ms/step | 19.2 samples/s +Step 39190/50462 | Loss: 2.5367 | LR: 6.08e-05 | 3128.7ms/step | 19.2 samples/s +Step 39200/50462 | Loss: 2.5049 | LR: 6.07e-05 | 3126.4ms/step | 19.2 samples/s +Step 39210/50462 | Loss: 2.5287 | LR: 6.06e-05 | 3142.5ms/step | 19.1 samples/s +Step 39220/50462 | Loss: 2.5021 | LR: 6.05e-05 | 3130.9ms/step | 19.2 samples/s +Step 39230/50462 | Loss: 2.5469 | LR: 6.04e-05 | 3134.5ms/step | 19.1 samples/s +Step 39240/50462 | Loss: 2.5250 | LR: 6.03e-05 | 3124.0ms/step | 19.2 samples/s +Step 39250/50462 | Loss: 2.5456 | LR: 6.02e-05 | 3119.1ms/step | 19.2 samples/s +Validation at step 39250: Loss = 2.5410, Perplexity = 12.69 +Step 39260/50462 | Loss: 2.5293 | LR: 6.01e-05 | 4764.4ms/step | 12.6 samples/s +Step 39270/50462 | Loss: 2.5308 | LR: 6.00e-05 | 3111.4ms/step | 19.3 samples/s +Step 39280/50462 | Loss: 2.5443 | LR: 5.99e-05 | 3116.6ms/step | 19.3 samples/s +Step 39290/50462 | Loss: 2.5238 | LR: 5.98e-05 | 3118.2ms/step | 19.2 samples/s +Step 39300/50462 | Loss: 2.5423 | LR: 5.97e-05 | 3118.9ms/step | 19.2 samples/s +Step 39310/50462 | Loss: 2.5295 | LR: 5.96e-05 | 3122.1ms/step | 19.2 samples/s +Step 39320/50462 | Loss: 2.4850 | LR: 5.95e-05 | 3113.4ms/step | 19.3 samples/s +Step 39330/50462 | Loss: 2.5298 | LR: 5.94e-05 | 3120.9ms/step | 19.2 samples/s +Step 39340/50462 | Loss: 2.5339 | LR: 5.93e-05 | 3121.7ms/step | 19.2 samples/s +Step 39350/50462 | Loss: 2.5295 | LR: 5.91e-05 | 3121.5ms/step | 19.2 samples/s +Step 39360/50462 | Loss: 2.5030 | LR: 5.90e-05 | 3121.4ms/step | 19.2 samples/s +Step 39370/50462 | Loss: 2.5314 | LR: 5.89e-05 | 3122.0ms/step | 19.2 samples/s +Step 39380/50462 | Loss: 2.5193 | LR: 5.88e-05 | 3119.5ms/step | 19.2 samples/s +Step 39390/50462 | Loss: 2.5343 | LR: 5.87e-05 | 3118.2ms/step | 19.2 samples/s +Step 39400/50462 | Loss: 2.5417 | LR: 5.86e-05 | 3118.5ms/step | 19.2 samples/s +Step 39410/50462 | Loss: 2.4919 | LR: 5.85e-05 | 3115.8ms/step | 19.3 samples/s +Step 39420/50462 | Loss: 2.5099 | LR: 5.84e-05 | 3114.3ms/step | 19.3 samples/s +Step 39430/50462 | Loss: 2.5075 | LR: 5.83e-05 | 3111.5ms/step | 19.3 samples/s +Step 39440/50462 | Loss: 2.5279 | LR: 5.82e-05 | 3118.2ms/step | 19.2 samples/s +Step 39450/50462 | Loss: 2.5169 | LR: 5.81e-05 | 3116.7ms/step | 19.3 samples/s +Step 39460/50462 | Loss: 2.5303 | LR: 5.80e-05 | 3122.9ms/step | 19.2 samples/s +Step 39470/50462 | Loss: 2.5654 | LR: 5.79e-05 | 3112.2ms/step | 19.3 samples/s +Step 39480/50462 | Loss: 2.5504 | LR: 5.78e-05 | 3111.4ms/step | 19.3 samples/s +Step 39490/50462 | Loss: 2.5206 | LR: 5.77e-05 | 3114.8ms/step | 19.3 samples/s +Step 39500/50462 | Loss: 2.5340 | LR: 5.76e-05 | 3117.0ms/step | 19.2 samples/s +Validation at step 39500: Loss = 2.5402, Perplexity = 12.68 +Step 39510/50462 | Loss: 2.5313 | LR: 5.75e-05 | 4784.0ms/step | 12.5 samples/s +Step 39520/50462 | Loss: 2.5432 | LR: 5.74e-05 | 3115.9ms/step | 19.3 samples/s +Step 39530/50462 | Loss: 2.5310 | LR: 5.73e-05 | 3118.4ms/step | 19.2 samples/s +Step 39540/50462 | Loss: 2.5314 | LR: 5.72e-05 | 3124.2ms/step | 19.2 samples/s +Step 39550/50462 | Loss: 2.5339 | LR: 5.71e-05 | 3116.2ms/step | 19.3 samples/s +Step 39560/50462 | Loss: 2.5234 | LR: 5.70e-05 | 3119.8ms/step | 19.2 samples/s +Step 39570/50462 | Loss: 2.5336 | LR: 5.69e-05 | 3120.0ms/step | 19.2 samples/s +Step 39580/50462 | Loss: 2.5155 | LR: 5.68e-05 | 3117.4ms/step | 19.2 samples/s +Step 39590/50462 | Loss: 2.5405 | LR: 5.67e-05 | 3109.3ms/step | 19.3 samples/s +Step 39600/50462 | Loss: 2.5239 | LR: 5.66e-05 | 3115.2ms/step | 19.3 samples/s +Step 39610/50462 | Loss: 2.5162 | LR: 5.65e-05 | 3120.0ms/step | 19.2 samples/s +Step 39620/50462 | Loss: 2.5111 | LR: 5.64e-05 | 3119.0ms/step | 19.2 samples/s +Step 39630/50462 | Loss: 2.5017 | LR: 5.63e-05 | 3118.1ms/step | 19.2 samples/s +Step 39640/50462 | Loss: 2.5561 | LR: 5.62e-05 | 3119.6ms/step | 19.2 samples/s +Step 39650/50462 | Loss: 2.5312 | LR: 5.61e-05 | 3128.8ms/step | 19.2 samples/s +Step 39660/50462 | Loss: 2.5311 | LR: 5.60e-05 | 3120.7ms/step | 19.2 samples/s +Step 39670/50462 | Loss: 2.5295 | LR: 5.59e-05 | 3122.5ms/step | 19.2 samples/s +Step 39680/50462 | Loss: 2.5393 | LR: 5.58e-05 | 3120.0ms/step | 19.2 samples/s +Step 39690/50462 | Loss: 2.5255 | LR: 5.57e-05 | 3121.1ms/step | 19.2 samples/s +Step 39700/50462 | Loss: 2.5213 | LR: 5.56e-05 | 3655.1ms/step | 16.4 samples/s +Step 39710/50462 | Loss: 2.5119 | LR: 5.55e-05 | 3111.1ms/step | 19.3 samples/s +Step 39720/50462 | Loss: 2.5021 | LR: 5.54e-05 | 3117.8ms/step | 19.2 samples/s +Step 39730/50462 | Loss: 2.5059 | LR: 5.53e-05 | 3119.7ms/step | 19.2 samples/s +Step 39740/50462 | Loss: 2.5262 | LR: 5.52e-05 | 3126.9ms/step | 19.2 samples/s +Switching attention mode for step 39750: dense (dense step interval) +Step 39750/50462 | Loss: 2.5346 | LR: 5.51e-05 | 3336.2ms/step | 18.0 samples/s +Validation at step 39750: Loss = 2.5229, Perplexity = 12.47 +Step 39760/50462 | Loss: 2.5038 | LR: 5.50e-05 | 7825.2ms/step | 7.7 samples/s +Step 39770/50462 | Loss: 2.5255 | LR: 5.49e-05 | 5257.5ms/step | 11.4 samples/s +Step 39780/50462 | Loss: 2.5237 | LR: 5.48e-05 | 5259.9ms/step | 11.4 samples/s +Step 39790/50462 | Loss: 2.5006 | LR: 5.47e-05 | 5773.7ms/step | 10.4 samples/s +Step 39800/50462 | Loss: 2.4949 | LR: 5.46e-05 | 5226.1ms/step | 11.5 samples/s +Step 39810/50462 | Loss: 2.5076 | LR: 5.45e-05 | 5235.9ms/step | 11.5 samples/s +Step 39820/50462 | Loss: 2.5273 | LR: 5.44e-05 | 5252.8ms/step | 11.4 samples/s +Step 39830/50462 | Loss: 2.4995 | LR: 5.43e-05 | 5259.7ms/step | 11.4 samples/s +Step 39840/50462 | Loss: 2.5110 | LR: 5.42e-05 | 5257.7ms/step | 11.4 samples/s +Step 39850/50462 | Loss: 2.5160 | LR: 5.41e-05 | 5257.5ms/step | 11.4 samples/s +Step 39860/50462 | Loss: 2.4785 | LR: 5.40e-05 | 5263.6ms/step | 11.4 samples/s +Step 39870/50462 | Loss: 2.5049 | LR: 5.39e-05 | 5255.3ms/step | 11.4 samples/s +Step 39880/50462 | Loss: 2.5049 | LR: 5.38e-05 | 5255.1ms/step | 11.4 samples/s +Step 39890/50462 | Loss: 2.5154 | LR: 5.38e-05 | 5257.6ms/step | 11.4 samples/s +Step 39900/50462 | Loss: 2.4957 | LR: 5.37e-05 | 5251.1ms/step | 11.4 samples/s +Step 39910/50462 | Loss: 2.5018 | LR: 5.36e-05 | 5246.5ms/step | 11.4 samples/s +Step 39920/50462 | Loss: 2.5035 | LR: 5.35e-05 | 5255.0ms/step | 11.4 samples/s +Step 39930/50462 | Loss: 2.5061 | LR: 5.34e-05 | 5253.4ms/step | 11.4 samples/s +Step 39940/50462 | Loss: 2.4968 | LR: 5.33e-05 | 5251.8ms/step | 11.4 samples/s +Step 39950/50462 | Loss: 2.4891 | LR: 5.32e-05 | 5253.4ms/step | 11.4 samples/s +Step 39960/50462 | Loss: 2.5046 | LR: 5.31e-05 | 5251.7ms/step | 11.4 samples/s +Step 39970/50462 | Loss: 2.5280 | LR: 5.30e-05 | 5250.8ms/step | 11.4 samples/s +Step 39980/50462 | Loss: 2.5048 | LR: 5.29e-05 | 5253.7ms/step | 11.4 samples/s +Step 39990/50462 | Loss: 2.5384 | LR: 5.28e-05 | 5255.0ms/step | 11.4 samples/s +Step 40000/50462 | Loss: 2.4773 | LR: 5.27e-05 | 5245.1ms/step | 11.4 samples/s +Validation at step 40000: Loss = 2.5152, Perplexity = 12.37 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-40000 +Switching attention mode for step 40001: sparse_stochastic (configured sparse attention) +Step 40010/50462 | Loss: 2.5489 | LR: 5.26e-05 | 6606.9ms/step | 9.1 samples/s +Step 40020/50462 | Loss: 2.5682 | LR: 5.25e-05 | 3104.6ms/step | 19.3 samples/s +Step 40030/50462 | Loss: 2.5130 | LR: 5.24e-05 | 3109.6ms/step | 19.3 samples/s +Step 40040/50462 | Loss: 2.5447 | LR: 5.23e-05 | 3112.3ms/step | 19.3 samples/s +Step 40050/50462 | Loss: 2.5316 | LR: 5.22e-05 | 3122.7ms/step | 19.2 samples/s +Step 40060/50462 | Loss: 2.5256 | LR: 5.21e-05 | 3120.5ms/step | 19.2 samples/s +Step 40070/50462 | Loss: 2.5353 | LR: 5.20e-05 | 3124.6ms/step | 19.2 samples/s +Step 40080/50462 | Loss: 2.5100 | LR: 5.19e-05 | 3117.5ms/step | 19.2 samples/s +Step 40090/50462 | Loss: 2.5442 | LR: 5.18e-05 | 3124.9ms/step | 19.2 samples/s +Step 40100/50462 | Loss: 2.5400 | LR: 5.17e-05 | 3120.1ms/step | 19.2 samples/s +Step 40110/50462 | Loss: 2.5285 | LR: 5.16e-05 | 3119.4ms/step | 19.2 samples/s +Step 40120/50462 | Loss: 2.5416 | LR: 5.15e-05 | 3125.8ms/step | 19.2 samples/s +Step 40130/50462 | Loss: 2.5234 | LR: 5.14e-05 | 3125.3ms/step | 19.2 samples/s +Step 40140/50462 | Loss: 2.5204 | LR: 5.13e-05 | 3122.9ms/step | 19.2 samples/s +Step 40150/50462 | Loss: 2.4984 | LR: 5.12e-05 | 3119.2ms/step | 19.2 samples/s +Step 40160/50462 | Loss: 2.4956 | LR: 5.11e-05 | 3116.8ms/step | 19.3 samples/s +Step 40170/50462 | Loss: 2.5352 | LR: 5.10e-05 | 3119.8ms/step | 19.2 samples/s +Step 40180/50462 | Loss: 2.5357 | LR: 5.09e-05 | 3116.2ms/step | 19.3 samples/s +Step 40190/50462 | Loss: 2.5324 | LR: 5.09e-05 | 3116.0ms/step | 19.3 samples/s +Step 40200/50462 | Loss: 2.5417 | LR: 5.08e-05 | 3116.1ms/step | 19.3 samples/s +Step 40210/50462 | Loss: 2.5469 | LR: 5.07e-05 | 3121.8ms/step | 19.2 samples/s +Step 40220/50462 | Loss: 2.5477 | LR: 5.06e-05 | 3117.9ms/step | 19.2 samples/s +Step 40230/50462 | Loss: 2.5379 | LR: 5.05e-05 | 3114.4ms/step | 19.3 samples/s +Step 40240/50462 | Loss: 2.5689 | LR: 5.04e-05 | 3112.3ms/step | 19.3 samples/s +Step 40250/50462 | Loss: 2.5280 | LR: 5.03e-05 | 3115.0ms/step | 19.3 samples/s +Validation at step 40250: Loss = 2.5398, Perplexity = 12.68 +Step 40260/50462 | Loss: 2.5341 | LR: 5.02e-05 | 4784.5ms/step | 12.5 samples/s +Step 40270/50462 | Loss: 2.5092 | LR: 5.01e-05 | 3115.6ms/step | 19.3 samples/s +Step 40280/50462 | Loss: 2.5334 | LR: 5.00e-05 | 3120.1ms/step | 19.2 samples/s +Step 40290/50462 | Loss: 2.5314 | LR: 4.99e-05 | 3115.8ms/step | 19.3 samples/s +Step 40300/50462 | Loss: 2.5273 | LR: 4.98e-05 | 3118.5ms/step | 19.2 samples/s +Step 40310/50462 | Loss: 2.5127 | LR: 4.97e-05 | 3118.4ms/step | 19.2 samples/s +Step 40320/50462 | Loss: 2.5312 | LR: 4.96e-05 | 3116.0ms/step | 19.3 samples/s +Step 40330/50462 | Loss: 2.5541 | LR: 4.95e-05 | 3118.2ms/step | 19.2 samples/s +Step 40340/50462 | Loss: 2.5146 | LR: 4.94e-05 | 3129.7ms/step | 19.2 samples/s +Step 40350/50462 | Loss: 2.5046 | LR: 4.93e-05 | 3121.9ms/step | 19.2 samples/s +Step 40360/50462 | Loss: 2.5338 | LR: 4.92e-05 | 3121.9ms/step | 19.2 samples/s +Step 40370/50462 | Loss: 2.5466 | LR: 4.91e-05 | 3126.2ms/step | 19.2 samples/s +Step 40380/50462 | Loss: 2.5329 | LR: 4.90e-05 | 3123.9ms/step | 19.2 samples/s +Step 40390/50462 | Loss: 2.5282 | LR: 4.90e-05 | 3121.2ms/step | 19.2 samples/s +Step 40400/50462 | Loss: 2.5404 | LR: 4.89e-05 | 3117.2ms/step | 19.2 samples/s +Step 40410/50462 | Loss: 2.5546 | LR: 4.88e-05 | 3119.4ms/step | 19.2 samples/s +Step 40420/50462 | Loss: 2.5526 | LR: 4.87e-05 | 3122.5ms/step | 19.2 samples/s +Step 40430/50462 | Loss: 2.5205 | LR: 4.86e-05 | 3123.1ms/step | 19.2 samples/s +Step 40440/50462 | Loss: 2.5428 | LR: 4.85e-05 | 3126.4ms/step | 19.2 samples/s +Step 40450/50462 | Loss: 2.5281 | LR: 4.84e-05 | 3121.0ms/step | 19.2 samples/s +Step 40460/50462 | Loss: 2.5603 | LR: 4.83e-05 | 3119.0ms/step | 19.2 samples/s +Step 40470/50462 | Loss: 2.5156 | LR: 4.82e-05 | 3117.9ms/step | 19.2 samples/s +Step 40480/50462 | Loss: 2.5265 | LR: 4.81e-05 | 3120.8ms/step | 19.2 samples/s +Step 40490/50462 | Loss: 2.5414 | LR: 4.80e-05 | 3119.3ms/step | 19.2 samples/s +Step 40500/50462 | Loss: 2.5323 | LR: 4.79e-05 | 3122.1ms/step | 19.2 samples/s +Validation at step 40500: Loss = 2.5395, Perplexity = 12.67 +Step 40510/50462 | Loss: 2.5310 | LR: 4.78e-05 | 4777.4ms/step | 12.6 samples/s +Step 40520/50462 | Loss: 2.5168 | LR: 4.77e-05 | 3115.8ms/step | 19.3 samples/s +Step 40530/50462 | Loss: 2.5208 | LR: 4.76e-05 | 3125.0ms/step | 19.2 samples/s +Step 40540/50462 | Loss: 2.4941 | LR: 4.76e-05 | 3118.8ms/step | 19.2 samples/s +Step 40550/50462 | Loss: 2.5489 | LR: 4.75e-05 | 3116.2ms/step | 19.3 samples/s +Step 40560/50462 | Loss: 2.5402 | LR: 4.74e-05 | 3114.7ms/step | 19.3 samples/s +Step 40570/50462 | Loss: 2.5150 | LR: 4.73e-05 | 3118.4ms/step | 19.2 samples/s +Step 40580/50462 | Loss: 2.5422 | LR: 4.72e-05 | 3115.5ms/step | 19.3 samples/s +Step 40590/50462 | Loss: 2.5266 | LR: 4.71e-05 | 3116.6ms/step | 19.3 samples/s +Step 40600/50462 | Loss: 2.5275 | LR: 4.70e-05 | 3111.6ms/step | 19.3 samples/s +Step 40610/50462 | Loss: 2.5460 | LR: 4.69e-05 | 3111.1ms/step | 19.3 samples/s +Step 40620/50462 | Loss: 2.5028 | LR: 4.68e-05 | 3112.1ms/step | 19.3 samples/s +Step 40630/50462 | Loss: 2.5243 | LR: 4.67e-05 | 3112.4ms/step | 19.3 samples/s +Step 40640/50462 | Loss: 2.5356 | LR: 4.66e-05 | 3114.4ms/step | 19.3 samples/s +Step 40650/50462 | Loss: 2.5341 | LR: 4.65e-05 | 3116.3ms/step | 19.3 samples/s +Step 40660/50462 | Loss: 2.5468 | LR: 4.64e-05 | 3115.3ms/step | 19.3 samples/s +Step 40670/50462 | Loss: 2.5234 | LR: 4.64e-05 | 3116.2ms/step | 19.3 samples/s +Step 40680/50462 | Loss: 2.5285 | LR: 4.63e-05 | 3117.5ms/step | 19.2 samples/s +Step 40690/50462 | Loss: 2.5253 | LR: 4.62e-05 | 3113.3ms/step | 19.3 samples/s +Step 40700/50462 | Loss: 2.4970 | LR: 4.61e-05 | 3115.8ms/step | 19.3 samples/s +Step 40710/50462 | Loss: 2.5366 | LR: 4.60e-05 | 3117.5ms/step | 19.2 samples/s +Step 40720/50462 | Loss: 2.5035 | LR: 4.59e-05 | 3114.2ms/step | 19.3 samples/s +Step 40730/50462 | Loss: 2.5141 | LR: 4.58e-05 | 3115.7ms/step | 19.3 samples/s +Step 40740/50462 | Loss: 2.5109 | LR: 4.57e-05 | 3118.3ms/step | 19.2 samples/s +Step 40750/50462 | Loss: 2.5210 | LR: 4.56e-05 | 3114.2ms/step | 19.3 samples/s +Validation at step 40750: Loss = 2.5394, Perplexity = 12.67 +Step 40760/50462 | Loss: 2.5464 | LR: 4.55e-05 | 4775.0ms/step | 12.6 samples/s +Step 40770/50462 | Loss: 2.5264 | LR: 4.54e-05 | 3114.9ms/step | 19.3 samples/s +Step 40780/50462 | Loss: 2.5420 | LR: 4.54e-05 | 3113.3ms/step | 19.3 samples/s +Step 40790/50462 | Loss: 2.5270 | LR: 4.53e-05 | 3116.3ms/step | 19.3 samples/s +Step 40800/50462 | Loss: 2.5354 | LR: 4.52e-05 | 3119.3ms/step | 19.2 samples/s +Step 40810/50462 | Loss: 2.4927 | LR: 4.51e-05 | 3115.1ms/step | 19.3 samples/s +Step 40820/50462 | Loss: 2.5673 | LR: 4.50e-05 | 3116.5ms/step | 19.3 samples/s +Step 40830/50462 | Loss: 2.4812 | LR: 4.49e-05 | 3115.5ms/step | 19.3 samples/s +Step 40840/50462 | Loss: 2.5059 | LR: 4.48e-05 | 3117.7ms/step | 19.2 samples/s +Step 40850/50462 | Loss: 2.5491 | LR: 4.47e-05 | 3117.2ms/step | 19.2 samples/s +Step 40860/50462 | Loss: 2.4950 | LR: 4.46e-05 | 3113.8ms/step | 19.3 samples/s +Step 40870/50462 | Loss: 2.5494 | LR: 4.45e-05 | 3113.5ms/step | 19.3 samples/s +Step 40880/50462 | Loss: 2.5565 | LR: 4.45e-05 | 3116.9ms/step | 19.3 samples/s +Step 40890/50462 | Loss: 2.4901 | LR: 4.44e-05 | 3121.1ms/step | 19.2 samples/s +Step 40900/50462 | Loss: 2.5148 | LR: 4.43e-05 | 3115.6ms/step | 19.3 samples/s +Step 40910/50462 | Loss: 2.5352 | LR: 4.42e-05 | 3116.0ms/step | 19.3 samples/s +Step 40920/50462 | Loss: 2.5431 | LR: 4.41e-05 | 3118.5ms/step | 19.2 samples/s +Step 40930/50462 | Loss: 2.5348 | LR: 4.40e-05 | 3119.7ms/step | 19.2 samples/s +Step 40940/50462 | Loss: 2.5515 | LR: 4.39e-05 | 3116.6ms/step | 19.3 samples/s +Step 40950/50462 | Loss: 2.5057 | LR: 4.38e-05 | 3117.2ms/step | 19.2 samples/s +Step 40960/50462 | Loss: 2.5363 | LR: 4.37e-05 | 3118.5ms/step | 19.2 samples/s +Step 40970/50462 | Loss: 2.5264 | LR: 4.36e-05 | 3117.6ms/step | 19.2 samples/s +Step 40980/50462 | Loss: 2.5262 | LR: 4.36e-05 | 3113.5ms/step | 19.3 samples/s +Step 40990/50462 | Loss: 2.4939 | LR: 4.35e-05 | 3113.0ms/step | 19.3 samples/s +Step 41000/50462 | Loss: 2.5392 | LR: 4.34e-05 | 3117.2ms/step | 19.2 samples/s +Validation at step 41000: Loss = 2.5390, Perplexity = 12.67 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-41000 +Step 41010/50462 | Loss: 2.5465 | LR: 4.33e-05 | 5716.5ms/step | 10.5 samples/s +Step 41020/50462 | Loss: 2.5413 | LR: 4.32e-05 | 3106.7ms/step | 19.3 samples/s +Step 41030/50462 | Loss: 2.5435 | LR: 4.31e-05 | 3110.0ms/step | 19.3 samples/s +Step 41040/50462 | Loss: 2.5073 | LR: 4.30e-05 | 3115.0ms/step | 19.3 samples/s +Step 41050/50462 | Loss: 2.5390 | LR: 4.29e-05 | 3114.0ms/step | 19.3 samples/s +Step 41060/50462 | Loss: 2.5448 | LR: 4.28e-05 | 3118.0ms/step | 19.2 samples/s +Step 41070/50462 | Loss: 2.5217 | LR: 4.28e-05 | 3113.9ms/step | 19.3 samples/s +Step 41080/50462 | Loss: 2.5150 | LR: 4.27e-05 | 3115.4ms/step | 19.3 samples/s +Step 41090/50462 | Loss: 2.4933 | LR: 4.26e-05 | 3117.2ms/step | 19.2 samples/s +Step 41100/50462 | Loss: 2.5286 | LR: 4.25e-05 | 3118.1ms/step | 19.2 samples/s +Step 41110/50462 | Loss: 2.5314 | LR: 4.24e-05 | 3118.1ms/step | 19.2 samples/s +Step 41120/50462 | Loss: 2.5218 | LR: 4.23e-05 | 3115.6ms/step | 19.3 samples/s +Step 41130/50462 | Loss: 2.5454 | LR: 4.22e-05 | 3117.8ms/step | 19.2 samples/s +Step 41140/50462 | Loss: 2.5183 | LR: 4.21e-05 | 3118.8ms/step | 19.2 samples/s +Step 41150/50462 | Loss: 2.5321 | LR: 4.21e-05 | 3116.8ms/step | 19.3 samples/s +Step 41160/50462 | Loss: 2.5300 | LR: 4.20e-05 | 3115.9ms/step | 19.3 samples/s +Step 41170/50462 | Loss: 2.5347 | LR: 4.19e-05 | 3117.3ms/step | 19.2 samples/s +Step 41180/50462 | Loss: 2.5197 | LR: 4.18e-05 | 3117.5ms/step | 19.2 samples/s +Step 41190/50462 | Loss: 2.4792 | LR: 4.17e-05 | 3117.5ms/step | 19.2 samples/s +Step 41200/50462 | Loss: 2.5164 | LR: 4.16e-05 | 3621.4ms/step | 16.6 samples/s +Step 41210/50462 | Loss: 2.4932 | LR: 4.15e-05 | 3101.7ms/step | 19.3 samples/s +Step 41220/50462 | Loss: 2.5101 | LR: 4.14e-05 | 3108.6ms/step | 19.3 samples/s +Step 41230/50462 | Loss: 2.5316 | LR: 4.14e-05 | 3114.3ms/step | 19.3 samples/s +Step 41240/50462 | Loss: 2.5216 | LR: 4.13e-05 | 3113.9ms/step | 19.3 samples/s +Step 41250/50462 | Loss: 2.5363 | LR: 4.12e-05 | 3116.4ms/step | 19.3 samples/s +Validation at step 41250: Loss = 2.5390, Perplexity = 12.67 +Step 41260/50462 | Loss: 2.5318 | LR: 4.11e-05 | 4777.5ms/step | 12.6 samples/s +Step 41270/50462 | Loss: 2.5056 | LR: 4.10e-05 | 3116.0ms/step | 19.3 samples/s +Step 41280/50462 | Loss: 2.4838 | LR: 4.09e-05 | 3115.2ms/step | 19.3 samples/s +Step 41290/50462 | Loss: 2.5453 | LR: 4.08e-05 | 3115.7ms/step | 19.3 samples/s +Step 41300/50462 | Loss: 2.5354 | LR: 4.07e-05 | 3114.5ms/step | 19.3 samples/s +Step 41310/50462 | Loss: 2.4977 | LR: 4.07e-05 | 3117.0ms/step | 19.2 samples/s +Step 41320/50462 | Loss: 2.5800 | LR: 4.06e-05 | 3114.5ms/step | 19.3 samples/s +Step 41330/50462 | Loss: 2.5162 | LR: 4.05e-05 | 3116.9ms/step | 19.2 samples/s +Step 41340/50462 | Loss: 2.5090 | LR: 4.04e-05 | 3115.7ms/step | 19.3 samples/s +Step 41350/50462 | Loss: 2.5295 | LR: 4.03e-05 | 3116.7ms/step | 19.3 samples/s +Step 41360/50462 | Loss: 2.5574 | LR: 4.02e-05 | 3115.9ms/step | 19.3 samples/s +Step 41370/50462 | Loss: 2.5279 | LR: 4.01e-05 | 3118.3ms/step | 19.2 samples/s +Step 41380/50462 | Loss: 2.5208 | LR: 4.01e-05 | 3114.4ms/step | 19.3 samples/s +Step 41390/50462 | Loss: 2.4966 | LR: 4.00e-05 | 3113.2ms/step | 19.3 samples/s +Step 41400/50462 | Loss: 2.5363 | LR: 3.99e-05 | 3113.8ms/step | 19.3 samples/s +Step 41410/50462 | Loss: 2.5428 | LR: 3.98e-05 | 3112.6ms/step | 19.3 samples/s +Step 41420/50462 | Loss: 2.5066 | LR: 3.97e-05 | 3114.2ms/step | 19.3 samples/s +Step 41430/50462 | Loss: 2.5381 | LR: 3.96e-05 | 3115.3ms/step | 19.3 samples/s +Step 41440/50462 | Loss: 2.5066 | LR: 3.95e-05 | 3115.0ms/step | 19.3 samples/s +Step 41450/50462 | Loss: 2.5263 | LR: 3.95e-05 | 3117.0ms/step | 19.2 samples/s +Step 41460/50462 | Loss: 2.5081 | LR: 3.94e-05 | 3118.7ms/step | 19.2 samples/s +Step 41470/50462 | Loss: 2.4883 | LR: 3.93e-05 | 3117.3ms/step | 19.2 samples/s +Step 41480/50462 | Loss: 2.5453 | LR: 3.92e-05 | 3119.5ms/step | 19.2 samples/s +Step 41490/50462 | Loss: 2.4995 | LR: 3.91e-05 | 3117.2ms/step | 19.2 samples/s +Step 41500/50462 | Loss: 2.5415 | LR: 3.90e-05 | 3115.8ms/step | 19.3 samples/s +Validation at step 41500: Loss = 2.5391, Perplexity = 12.67 +Step 41510/50462 | Loss: 2.5122 | LR: 3.90e-05 | 4773.2ms/step | 12.6 samples/s +Step 41520/50462 | Loss: 2.5267 | LR: 3.89e-05 | 3116.1ms/step | 19.3 samples/s +Step 41530/50462 | Loss: 2.5376 | LR: 3.88e-05 | 3119.0ms/step | 19.2 samples/s +Step 41540/50462 | Loss: 2.5067 | LR: 3.87e-05 | 3115.8ms/step | 19.3 samples/s +Step 41550/50462 | Loss: 2.5330 | LR: 3.86e-05 | 3117.7ms/step | 19.2 samples/s +Step 41560/50462 | Loss: 2.5450 | LR: 3.85e-05 | 3117.3ms/step | 19.2 samples/s +Step 41570/50462 | Loss: 2.5316 | LR: 3.84e-05 | 3117.8ms/step | 19.2 samples/s +Step 41580/50462 | Loss: 2.5374 | LR: 3.84e-05 | 3117.2ms/step | 19.2 samples/s +Step 41590/50462 | Loss: 2.5204 | LR: 3.83e-05 | 3116.5ms/step | 19.3 samples/s +Step 41600/50462 | Loss: 2.5341 | LR: 3.82e-05 | 3115.7ms/step | 19.3 samples/s +Step 41610/50462 | Loss: 2.5159 | LR: 3.81e-05 | 3115.5ms/step | 19.3 samples/s +Step 41620/50462 | Loss: 2.5216 | LR: 3.80e-05 | 3113.8ms/step | 19.3 samples/s +Step 41630/50462 | Loss: 2.5035 | LR: 3.79e-05 | 3117.0ms/step | 19.2 samples/s +Step 41640/50462 | Loss: 2.5188 | LR: 3.79e-05 | 3115.8ms/step | 19.3 samples/s +Step 41650/50462 | Loss: 2.5054 | LR: 3.78e-05 | 3115.3ms/step | 19.3 samples/s +Step 41660/50462 | Loss: 2.5342 | LR: 3.77e-05 | 3114.5ms/step | 19.3 samples/s +Step 41670/50462 | Loss: 2.5265 | LR: 3.76e-05 | 3113.0ms/step | 19.3 samples/s +Step 41680/50462 | Loss: 2.5118 | LR: 3.75e-05 | 3117.5ms/step | 19.2 samples/s +Step 41690/50462 | Loss: 2.5088 | LR: 3.74e-05 | 3115.6ms/step | 19.3 samples/s +Step 41700/50462 | Loss: 2.5207 | LR: 3.74e-05 | 3116.9ms/step | 19.3 samples/s +Step 41710/50462 | Loss: 2.5118 | LR: 3.73e-05 | 3112.7ms/step | 19.3 samples/s +Step 41720/50462 | Loss: 2.5084 | LR: 3.72e-05 | 3116.6ms/step | 19.3 samples/s +Step 41730/50462 | Loss: 2.5149 | LR: 3.71e-05 | 3110.7ms/step | 19.3 samples/s +Step 41740/50462 | Loss: 2.5079 | LR: 3.70e-05 | 3115.2ms/step | 19.3 samples/s +Step 41750/50462 | Loss: 2.5148 | LR: 3.69e-05 | 3114.9ms/step | 19.3 samples/s +Validation at step 41750: Loss = 2.5389, Perplexity = 12.67 +Step 41760/50462 | Loss: 2.5274 | LR: 3.69e-05 | 4771.5ms/step | 12.6 samples/s +Step 41770/50462 | Loss: 2.5334 | LR: 3.68e-05 | 3110.9ms/step | 19.3 samples/s +Step 41780/50462 | Loss: 2.5207 | LR: 3.67e-05 | 3113.9ms/step | 19.3 samples/s +Step 41790/50462 | Loss: 2.5066 | LR: 3.66e-05 | 3115.8ms/step | 19.3 samples/s +Step 41800/50462 | Loss: 2.5103 | LR: 3.65e-05 | 3112.7ms/step | 19.3 samples/s +Step 41810/50462 | Loss: 2.5453 | LR: 3.64e-05 | 3115.7ms/step | 19.3 samples/s +Step 41820/50462 | Loss: 2.4704 | LR: 3.64e-05 | 3115.2ms/step | 19.3 samples/s +Step 41830/50462 | Loss: 2.5178 | LR: 3.63e-05 | 3113.8ms/step | 19.3 samples/s +Step 41840/50462 | Loss: 2.4923 | LR: 3.62e-05 | 3116.6ms/step | 19.3 samples/s +Step 41850/50462 | Loss: 2.5351 | LR: 3.61e-05 | 3116.4ms/step | 19.3 samples/s +Step 41860/50462 | Loss: 2.5207 | LR: 3.60e-05 | 3117.1ms/step | 19.2 samples/s +Step 41870/50462 | Loss: 2.5183 | LR: 3.60e-05 | 3115.0ms/step | 19.3 samples/s +Step 41880/50462 | Loss: 2.5303 | LR: 3.59e-05 | 3116.8ms/step | 19.3 samples/s +Step 41890/50462 | Loss: 2.5224 | LR: 3.58e-05 | 3623.7ms/step | 16.6 samples/s +Step 41900/50462 | Loss: 2.5469 | LR: 3.57e-05 | 3096.0ms/step | 19.4 samples/s +Step 41910/50462 | Loss: 2.5388 | LR: 3.56e-05 | 3104.6ms/step | 19.3 samples/s +Step 41920/50462 | Loss: 2.5290 | LR: 3.55e-05 | 3110.5ms/step | 19.3 samples/s +Step 41930/50462 | Loss: 2.5087 | LR: 3.55e-05 | 3111.4ms/step | 19.3 samples/s +Step 41940/50462 | Loss: 2.5374 | LR: 3.54e-05 | 3109.4ms/step | 19.3 samples/s +Step 41950/50462 | Loss: 2.5078 | LR: 3.53e-05 | 3112.3ms/step | 19.3 samples/s +Step 41960/50462 | Loss: 2.5350 | LR: 3.52e-05 | 3115.5ms/step | 19.3 samples/s +Step 41970/50462 | Loss: 2.5227 | LR: 3.51e-05 | 3114.9ms/step | 19.3 samples/s +Step 41980/50462 | Loss: 2.5216 | LR: 3.51e-05 | 3115.2ms/step | 19.3 samples/s +Step 41990/50462 | Loss: 2.5532 | LR: 3.50e-05 | 3633.9ms/step | 16.5 samples/s +Step 42000/50462 | Loss: 2.5319 | LR: 3.49e-05 | 3099.1ms/step | 19.4 samples/s +Validation at step 42000: Loss = 2.5387, Perplexity = 12.66 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-42000 +Step 42010/50462 | Loss: 2.4995 | LR: 3.48e-05 | 5662.1ms/step | 10.6 samples/s +Step 42020/50462 | Loss: 2.5424 | LR: 3.47e-05 | 3103.0ms/step | 19.3 samples/s +Step 42030/50462 | Loss: 2.5608 | LR: 3.47e-05 | 3107.6ms/step | 19.3 samples/s +Step 42040/50462 | Loss: 2.5260 | LR: 3.46e-05 | 3110.7ms/step | 19.3 samples/s +Step 42050/50462 | Loss: 2.5281 | LR: 3.45e-05 | 3117.0ms/step | 19.2 samples/s +Step 42060/50462 | Loss: 2.5610 | LR: 3.44e-05 | 3114.7ms/step | 19.3 samples/s +Step 42070/50462 | Loss: 2.5427 | LR: 3.43e-05 | 3115.9ms/step | 19.3 samples/s +Step 42080/50462 | Loss: 2.5139 | LR: 3.43e-05 | 3114.5ms/step | 19.3 samples/s +Step 42090/50462 | Loss: 2.5449 | LR: 3.42e-05 | 3110.9ms/step | 19.3 samples/s +Step 42100/50462 | Loss: 2.5263 | LR: 3.41e-05 | 3112.9ms/step | 19.3 samples/s +Step 42110/50462 | Loss: 2.4940 | LR: 3.40e-05 | 3111.5ms/step | 19.3 samples/s +Step 42120/50462 | Loss: 2.5289 | LR: 3.39e-05 | 3114.4ms/step | 19.3 samples/s +Step 42130/50462 | Loss: 2.5413 | LR: 3.39e-05 | 3111.6ms/step | 19.3 samples/s +Step 42140/50462 | Loss: 2.5371 | LR: 3.38e-05 | 3112.5ms/step | 19.3 samples/s +Step 42150/50462 | Loss: 2.5332 | LR: 3.37e-05 | 3115.8ms/step | 19.3 samples/s +Step 42160/50462 | Loss: 2.4917 | LR: 3.36e-05 | 3113.7ms/step | 19.3 samples/s +Step 42170/50462 | Loss: 2.5183 | LR: 3.35e-05 | 3112.2ms/step | 19.3 samples/s +Step 42180/50462 | Loss: 2.5251 | LR: 3.35e-05 | 3111.5ms/step | 19.3 samples/s +Step 42190/50462 | Loss: 2.5549 | LR: 3.34e-05 | 3109.3ms/step | 19.3 samples/s +Step 42200/50462 | Loss: 2.5108 | LR: 3.33e-05 | 3113.7ms/step | 19.3 samples/s +Step 42210/50462 | Loss: 2.5158 | LR: 3.32e-05 | 3113.6ms/step | 19.3 samples/s +Step 42220/50462 | Loss: 2.5442 | LR: 3.32e-05 | 3113.6ms/step | 19.3 samples/s +Step 42230/50462 | Loss: 2.5300 | LR: 3.31e-05 | 3112.8ms/step | 19.3 samples/s +Step 42240/50462 | Loss: 2.4855 | LR: 3.30e-05 | 3109.7ms/step | 19.3 samples/s +Step 42250/50462 | Loss: 2.5361 | LR: 3.29e-05 | 3112.8ms/step | 19.3 samples/s +Validation at step 42250: Loss = 2.5382, Perplexity = 12.66 +Step 42260/50462 | Loss: 2.5424 | LR: 3.28e-05 | 4770.0ms/step | 12.6 samples/s +Step 42270/50462 | Loss: 2.5543 | LR: 3.28e-05 | 3114.7ms/step | 19.3 samples/s +Step 42280/50462 | Loss: 2.5231 | LR: 3.27e-05 | 3113.3ms/step | 19.3 samples/s +Step 42290/50462 | Loss: 2.5377 | LR: 3.26e-05 | 3115.1ms/step | 19.3 samples/s +Step 42300/50462 | Loss: 2.5231 | LR: 3.25e-05 | 3111.9ms/step | 19.3 samples/s +Step 42310/50462 | Loss: 2.5165 | LR: 3.24e-05 | 3115.4ms/step | 19.3 samples/s +Step 42320/50462 | Loss: 2.4983 | LR: 3.24e-05 | 3114.7ms/step | 19.3 samples/s +Step 42330/50462 | Loss: 2.5457 | LR: 3.23e-05 | 3111.1ms/step | 19.3 samples/s +Step 42340/50462 | Loss: 2.5532 | LR: 3.22e-05 | 3112.1ms/step | 19.3 samples/s +Step 42350/50462 | Loss: 2.5000 | LR: 3.21e-05 | 3111.2ms/step | 19.3 samples/s +Step 42360/50462 | Loss: 2.5103 | LR: 3.21e-05 | 3109.5ms/step | 19.3 samples/s +Step 42370/50462 | Loss: 2.5241 | LR: 3.20e-05 | 3110.6ms/step | 19.3 samples/s +Step 42380/50462 | Loss: 2.5105 | LR: 3.19e-05 | 3113.0ms/step | 19.3 samples/s +Step 42390/50462 | Loss: 2.5375 | LR: 3.18e-05 | 3111.7ms/step | 19.3 samples/s +Step 42400/50462 | Loss: 2.5457 | LR: 3.18e-05 | 3111.6ms/step | 19.3 samples/s +Step 42410/50462 | Loss: 2.5387 | LR: 3.17e-05 | 3113.0ms/step | 19.3 samples/s +Step 42420/50462 | Loss: 2.5318 | LR: 3.16e-05 | 3110.6ms/step | 19.3 samples/s +Step 42430/50462 | Loss: 2.5195 | LR: 3.15e-05 | 3110.6ms/step | 19.3 samples/s +Step 42440/50462 | Loss: 2.5186 | LR: 3.14e-05 | 3113.7ms/step | 19.3 samples/s +Step 42450/50462 | Loss: 2.5371 | LR: 3.14e-05 | 3109.2ms/step | 19.3 samples/s +Step 42460/50462 | Loss: 2.5181 | LR: 3.13e-05 | 3115.0ms/step | 19.3 samples/s +Step 42470/50462 | Loss: 2.5291 | LR: 3.12e-05 | 3111.7ms/step | 19.3 samples/s +Step 42480/50462 | Loss: 2.4855 | LR: 3.11e-05 | 3110.8ms/step | 19.3 samples/s +Step 42490/50462 | Loss: 2.5325 | LR: 3.11e-05 | 3115.0ms/step | 19.3 samples/s +Step 42500/50462 | Loss: 2.5421 | LR: 3.10e-05 | 3116.9ms/step | 19.3 samples/s +Validation at step 42500: Loss = 2.5384, Perplexity = 12.66 +Step 42510/50462 | Loss: 2.5377 | LR: 3.09e-05 | 4769.6ms/step | 12.6 samples/s +Step 42520/50462 | Loss: 2.5228 | LR: 3.08e-05 | 3114.8ms/step | 19.3 samples/s +Step 42530/50462 | Loss: 2.5158 | LR: 3.08e-05 | 3114.7ms/step | 19.3 samples/s +Step 42540/50462 | Loss: 2.5215 | LR: 3.07e-05 | 3113.1ms/step | 19.3 samples/s +Step 42550/50462 | Loss: 2.5466 | LR: 3.06e-05 | 3113.0ms/step | 19.3 samples/s +Step 42560/50462 | Loss: 2.6047 | LR: 3.05e-05 | 3108.5ms/step | 19.3 samples/s +Step 42570/50462 | Loss: 2.5340 | LR: 3.05e-05 | 3110.1ms/step | 19.3 samples/s +Step 42580/50462 | Loss: 2.5275 | LR: 3.04e-05 | 3112.9ms/step | 19.3 samples/s +Step 42590/50462 | Loss: 2.5221 | LR: 3.03e-05 | 3113.4ms/step | 19.3 samples/s +Step 42600/50462 | Loss: 2.5706 | LR: 3.02e-05 | 3111.4ms/step | 19.3 samples/s +Step 42610/50462 | Loss: 2.5195 | LR: 3.02e-05 | 3112.2ms/step | 19.3 samples/s +Step 42620/50462 | Loss: 2.5127 | LR: 3.01e-05 | 3113.6ms/step | 19.3 samples/s +Step 42630/50462 | Loss: 2.5445 | LR: 3.00e-05 | 3111.9ms/step | 19.3 samples/s +Step 42640/50462 | Loss: 2.5695 | LR: 2.99e-05 | 3113.9ms/step | 19.3 samples/s +Step 42650/50462 | Loss: 2.5393 | LR: 2.99e-05 | 3116.3ms/step | 19.3 samples/s +Step 42660/50462 | Loss: 2.5414 | LR: 2.98e-05 | 3112.2ms/step | 19.3 samples/s +Step 42670/50462 | Loss: 2.5109 | LR: 2.97e-05 | 3111.9ms/step | 19.3 samples/s +Step 42680/50462 | Loss: 2.5114 | LR: 2.96e-05 | 3117.1ms/step | 19.2 samples/s +Step 42690/50462 | Loss: 2.5311 | LR: 2.96e-05 | 3114.4ms/step | 19.3 samples/s +Step 42700/50462 | Loss: 2.5392 | LR: 2.95e-05 | 3113.5ms/step | 19.3 samples/s +Step 42710/50462 | Loss: 2.5215 | LR: 2.94e-05 | 3114.9ms/step | 19.3 samples/s +Step 42720/50462 | Loss: 2.5223 | LR: 2.93e-05 | 3112.7ms/step | 19.3 samples/s +Step 42730/50462 | Loss: 2.5225 | LR: 2.93e-05 | 3114.6ms/step | 19.3 samples/s +Step 42740/50462 | Loss: 2.5268 | LR: 2.92e-05 | 3114.7ms/step | 19.3 samples/s +Step 42750/50462 | Loss: 2.5240 | LR: 2.91e-05 | 3116.7ms/step | 19.3 samples/s +Validation at step 42750: Loss = 2.5383, Perplexity = 12.66 +Step 42760/50462 | Loss: 2.5375 | LR: 2.90e-05 | 4772.7ms/step | 12.6 samples/s +Step 42770/50462 | Loss: 2.5284 | LR: 2.90e-05 | 3114.4ms/step | 19.3 samples/s +Step 42780/50462 | Loss: 2.5254 | LR: 2.89e-05 | 3112.0ms/step | 19.3 samples/s +Step 42790/50462 | Loss: 2.5435 | LR: 2.88e-05 | 3110.8ms/step | 19.3 samples/s +Step 42800/50462 | Loss: 2.5438 | LR: 2.87e-05 | 3113.4ms/step | 19.3 samples/s +Step 42810/50462 | Loss: 2.5481 | LR: 2.87e-05 | 3114.0ms/step | 19.3 samples/s +Step 42820/50462 | Loss: 2.5197 | LR: 2.86e-05 | 3113.6ms/step | 19.3 samples/s +Step 42830/50462 | Loss: 2.5125 | LR: 2.85e-05 | 3114.0ms/step | 19.3 samples/s +Step 42840/50462 | Loss: 2.5399 | LR: 2.84e-05 | 3115.7ms/step | 19.3 samples/s +Step 42850/50462 | Loss: 2.5290 | LR: 2.84e-05 | 3116.3ms/step | 19.3 samples/s +Step 42860/50462 | Loss: 2.5573 | LR: 2.83e-05 | 3111.3ms/step | 19.3 samples/s +Step 42870/50462 | Loss: 2.5395 | LR: 2.82e-05 | 3115.4ms/step | 19.3 samples/s +Step 42880/50462 | Loss: 2.5349 | LR: 2.82e-05 | 3114.5ms/step | 19.3 samples/s +Step 42890/50462 | Loss: 2.5199 | LR: 2.81e-05 | 3114.3ms/step | 19.3 samples/s +Step 42900/50462 | Loss: 2.5274 | LR: 2.80e-05 | 3113.3ms/step | 19.3 samples/s +Step 42910/50462 | Loss: 2.5574 | LR: 2.79e-05 | 3107.6ms/step | 19.3 samples/s +Step 42920/50462 | Loss: 2.5168 | LR: 2.79e-05 | 3112.7ms/step | 19.3 samples/s +Step 42930/50462 | Loss: 2.5156 | LR: 2.78e-05 | 3114.0ms/step | 19.3 samples/s +Step 42940/50462 | Loss: 2.5059 | LR: 2.77e-05 | 3117.3ms/step | 19.2 samples/s +Step 42950/50462 | Loss: 2.5562 | LR: 2.76e-05 | 3114.2ms/step | 19.3 samples/s +Step 42960/50462 | Loss: 2.5394 | LR: 2.76e-05 | 3116.6ms/step | 19.3 samples/s +Step 42970/50462 | Loss: 2.5107 | LR: 2.75e-05 | 3116.2ms/step | 19.3 samples/s +Step 42980/50462 | Loss: 2.5242 | LR: 2.74e-05 | 3118.3ms/step | 19.2 samples/s +Step 42990/50462 | Loss: 2.5320 | LR: 2.74e-05 | 3113.3ms/step | 19.3 samples/s +Step 43000/50462 | Loss: 2.5099 | LR: 2.73e-05 | 3113.6ms/step | 19.3 samples/s +Validation at step 43000: Loss = 2.5381, Perplexity = 12.66 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-43000 +Step 43010/50462 | Loss: 2.5506 | LR: 2.72e-05 | 5712.3ms/step | 10.5 samples/s +Step 43020/50462 | Loss: 2.5236 | LR: 2.71e-05 | 3113.6ms/step | 19.3 samples/s +Step 43030/50462 | Loss: 2.5291 | LR: 2.71e-05 | 3115.8ms/step | 19.3 samples/s +Step 43040/50462 | Loss: 2.5165 | LR: 2.70e-05 | 3126.7ms/step | 19.2 samples/s +Step 43050/50462 | Loss: 2.5375 | LR: 2.69e-05 | 3116.6ms/step | 19.3 samples/s +Step 43060/50462 | Loss: 2.5121 | LR: 2.69e-05 | 3119.7ms/step | 19.2 samples/s +Step 43070/50462 | Loss: 2.5304 | LR: 2.68e-05 | 3117.8ms/step | 19.2 samples/s +Step 43080/50462 | Loss: 2.5088 | LR: 2.67e-05 | 3115.3ms/step | 19.3 samples/s +Step 43090/50462 | Loss: 2.5045 | LR: 2.66e-05 | 3120.6ms/step | 19.2 samples/s +Step 43100/50462 | Loss: 2.5043 | LR: 2.66e-05 | 3120.0ms/step | 19.2 samples/s +Step 43110/50462 | Loss: 2.5129 | LR: 2.65e-05 | 3120.7ms/step | 19.2 samples/s +Step 43120/50462 | Loss: 2.5269 | LR: 2.64e-05 | 3120.6ms/step | 19.2 samples/s +Step 43130/50462 | Loss: 2.5371 | LR: 2.64e-05 | 3118.5ms/step | 19.2 samples/s +Step 43140/50462 | Loss: 2.5077 | LR: 2.63e-05 | 3119.0ms/step | 19.2 samples/s +Step 43150/50462 | Loss: 2.5090 | LR: 2.62e-05 | 3120.5ms/step | 19.2 samples/s +Step 43160/50462 | Loss: 2.5482 | LR: 2.61e-05 | 3119.6ms/step | 19.2 samples/s +Step 43170/50462 | Loss: 2.5393 | LR: 2.61e-05 | 3119.5ms/step | 19.2 samples/s +Step 43180/50462 | Loss: 2.4938 | LR: 2.60e-05 | 3120.9ms/step | 19.2 samples/s +Step 43190/50462 | Loss: 2.5491 | LR: 2.59e-05 | 3122.3ms/step | 19.2 samples/s +Step 43200/50462 | Loss: 2.5265 | LR: 2.59e-05 | 3122.9ms/step | 19.2 samples/s +Step 43210/50462 | Loss: 2.5327 | LR: 2.58e-05 | 3121.7ms/step | 19.2 samples/s +Step 43220/50462 | Loss: 2.5404 | LR: 2.57e-05 | 3119.3ms/step | 19.2 samples/s +Step 43230/50462 | Loss: 2.5250 | LR: 2.57e-05 | 3118.5ms/step | 19.2 samples/s +Step 43240/50462 | Loss: 2.5462 | LR: 2.56e-05 | 3119.4ms/step | 19.2 samples/s +Step 43250/50462 | Loss: 2.4971 | LR: 2.55e-05 | 3119.7ms/step | 19.2 samples/s +Validation at step 43250: Loss = 2.5379, Perplexity = 12.65 +Step 43260/50462 | Loss: 2.4988 | LR: 2.54e-05 | 4790.7ms/step | 12.5 samples/s +Step 43270/50462 | Loss: 2.5099 | LR: 2.54e-05 | 3121.9ms/step | 19.2 samples/s +Step 43280/50462 | Loss: 2.5313 | LR: 2.53e-05 | 3121.0ms/step | 19.2 samples/s +Step 43290/50462 | Loss: 2.5108 | LR: 2.52e-05 | 3116.5ms/step | 19.3 samples/s +Step 43300/50462 | Loss: 2.5547 | LR: 2.52e-05 | 3119.7ms/step | 19.2 samples/s +Step 43310/50462 | Loss: 2.5196 | LR: 2.51e-05 | 3116.6ms/step | 19.3 samples/s +Step 43320/50462 | Loss: 2.5231 | LR: 2.50e-05 | 3635.9ms/step | 16.5 samples/s +Step 43330/50462 | Loss: 2.5091 | LR: 2.50e-05 | 3110.2ms/step | 19.3 samples/s +Step 43340/50462 | Loss: 2.5209 | LR: 2.49e-05 | 3114.3ms/step | 19.3 samples/s +Step 43350/50462 | Loss: 2.5065 | LR: 2.48e-05 | 3114.5ms/step | 19.3 samples/s +Step 43360/50462 | Loss: 2.5395 | LR: 2.48e-05 | 3118.6ms/step | 19.2 samples/s +Step 43370/50462 | Loss: 2.5308 | LR: 2.47e-05 | 3120.5ms/step | 19.2 samples/s +Step 43380/50462 | Loss: 2.5097 | LR: 2.46e-05 | 3122.8ms/step | 19.2 samples/s +Step 43390/50462 | Loss: 2.5388 | LR: 2.46e-05 | 3120.3ms/step | 19.2 samples/s +Step 43400/50462 | Loss: 2.5209 | LR: 2.45e-05 | 3142.6ms/step | 19.1 samples/s +Step 43410/50462 | Loss: 2.5410 | LR: 2.44e-05 | 3121.5ms/step | 19.2 samples/s +Step 43420/50462 | Loss: 2.5212 | LR: 2.44e-05 | 3118.7ms/step | 19.2 samples/s +Step 43430/50462 | Loss: 2.5260 | LR: 2.43e-05 | 3120.6ms/step | 19.2 samples/s +Step 43440/50462 | Loss: 2.5228 | LR: 2.42e-05 | 3119.0ms/step | 19.2 samples/s +Step 43450/50462 | Loss: 2.5468 | LR: 2.41e-05 | 3119.9ms/step | 19.2 samples/s +Step 43460/50462 | Loss: 2.5341 | LR: 2.41e-05 | 3117.9ms/step | 19.2 samples/s +Step 43470/50462 | Loss: 2.5496 | LR: 2.40e-05 | 3115.5ms/step | 19.3 samples/s +Step 43480/50462 | Loss: 2.5489 | LR: 2.39e-05 | 3118.8ms/step | 19.2 samples/s +Step 43490/50462 | Loss: 2.5171 | LR: 2.39e-05 | 3119.6ms/step | 19.2 samples/s +Step 43500/50462 | Loss: 2.5352 | LR: 2.38e-05 | 3117.0ms/step | 19.2 samples/s +Validation at step 43500: Loss = 2.5378, Perplexity = 12.65 +Step 43510/50462 | Loss: 2.5373 | LR: 2.37e-05 | 4782.2ms/step | 12.5 samples/s +Step 43520/50462 | Loss: 2.5010 | LR: 2.37e-05 | 3117.3ms/step | 19.2 samples/s +Step 43530/50462 | Loss: 2.5224 | LR: 2.36e-05 | 3116.3ms/step | 19.3 samples/s +Step 43540/50462 | Loss: 2.5189 | LR: 2.35e-05 | 3116.5ms/step | 19.3 samples/s +Step 43550/50462 | Loss: 2.5272 | LR: 2.35e-05 | 3114.8ms/step | 19.3 samples/s +Step 43560/50462 | Loss: 2.5064 | LR: 2.34e-05 | 3114.6ms/step | 19.3 samples/s +Step 43570/50462 | Loss: 2.5271 | LR: 2.33e-05 | 3115.1ms/step | 19.3 samples/s +Step 43580/50462 | Loss: 2.5555 | LR: 2.33e-05 | 3117.1ms/step | 19.2 samples/s +Step 43590/50462 | Loss: 2.5127 | LR: 2.32e-05 | 3116.0ms/step | 19.3 samples/s +Step 43600/50462 | Loss: 2.5469 | LR: 2.31e-05 | 3147.6ms/step | 19.1 samples/s +Step 43610/50462 | Loss: 2.5330 | LR: 2.31e-05 | 3143.6ms/step | 19.1 samples/s +Step 43620/50462 | Loss: 2.5161 | LR: 2.30e-05 | 3119.7ms/step | 19.2 samples/s +Step 43630/50462 | Loss: 2.5533 | LR: 2.29e-05 | 3118.2ms/step | 19.2 samples/s +Step 43640/50462 | Loss: 2.5161 | LR: 2.29e-05 | 3116.0ms/step | 19.3 samples/s +Step 43650/50462 | Loss: 2.5348 | LR: 2.28e-05 | 3118.9ms/step | 19.2 samples/s +Step 43660/50462 | Loss: 2.5476 | LR: 2.27e-05 | 3122.4ms/step | 19.2 samples/s +Step 43670/50462 | Loss: 2.5348 | LR: 2.27e-05 | 3121.9ms/step | 19.2 samples/s +Step 43680/50462 | Loss: 2.5394 | LR: 2.26e-05 | 3120.0ms/step | 19.2 samples/s +Step 43690/50462 | Loss: 2.5321 | LR: 2.25e-05 | 3117.0ms/step | 19.2 samples/s +Step 43700/50462 | Loss: 2.5491 | LR: 2.25e-05 | 3118.4ms/step | 19.2 samples/s +Step 43710/50462 | Loss: 2.5545 | LR: 2.24e-05 | 3119.6ms/step | 19.2 samples/s +Step 43720/50462 | Loss: 2.5105 | LR: 2.24e-05 | 3117.4ms/step | 19.2 samples/s +Step 43730/50462 | Loss: 2.4995 | LR: 2.23e-05 | 3118.4ms/step | 19.2 samples/s +Step 43740/50462 | Loss: 2.5234 | LR: 2.22e-05 | 3119.4ms/step | 19.2 samples/s +Step 43750/50462 | Loss: 2.5244 | LR: 2.22e-05 | 3116.2ms/step | 19.3 samples/s +Validation at step 43750: Loss = 2.5379, Perplexity = 12.65 +Step 43760/50462 | Loss: 2.5305 | LR: 2.21e-05 | 4776.3ms/step | 12.6 samples/s +Step 43770/50462 | Loss: 2.5153 | LR: 2.20e-05 | 3118.6ms/step | 19.2 samples/s +Step 43780/50462 | Loss: 2.5240 | LR: 2.20e-05 | 3120.0ms/step | 19.2 samples/s +Step 43790/50462 | Loss: 2.5308 | LR: 2.19e-05 | 3119.4ms/step | 19.2 samples/s +Step 43800/50462 | Loss: 2.5450 | LR: 2.18e-05 | 3117.2ms/step | 19.2 samples/s +Step 43810/50462 | Loss: 2.4886 | LR: 2.18e-05 | 3120.6ms/step | 19.2 samples/s +Step 43820/50462 | Loss: 2.4928 | LR: 2.17e-05 | 3118.7ms/step | 19.2 samples/s +Step 43830/50462 | Loss: 2.5362 | LR: 2.16e-05 | 3118.4ms/step | 19.2 samples/s +Step 43840/50462 | Loss: 2.5509 | LR: 2.16e-05 | 3119.5ms/step | 19.2 samples/s +Step 43850/50462 | Loss: 2.5466 | LR: 2.15e-05 | 3121.5ms/step | 19.2 samples/s +Step 43860/50462 | Loss: 2.5295 | LR: 2.14e-05 | 3118.6ms/step | 19.2 samples/s +Step 43870/50462 | Loss: 2.5184 | LR: 2.14e-05 | 3117.2ms/step | 19.2 samples/s +Step 43880/50462 | Loss: 2.5393 | LR: 2.13e-05 | 3118.3ms/step | 19.2 samples/s +Step 43890/50462 | Loss: 2.5301 | LR: 2.13e-05 | 3118.4ms/step | 19.2 samples/s +Step 43900/50462 | Loss: 2.5828 | LR: 2.12e-05 | 3118.2ms/step | 19.2 samples/s +Step 43910/50462 | Loss: 2.5489 | LR: 2.11e-05 | 3118.9ms/step | 19.2 samples/s +Step 43920/50462 | Loss: 2.5248 | LR: 2.11e-05 | 3121.6ms/step | 19.2 samples/s +Step 43930/50462 | Loss: 2.5501 | LR: 2.10e-05 | 3117.7ms/step | 19.2 samples/s +Step 43940/50462 | Loss: 2.5256 | LR: 2.09e-05 | 3115.6ms/step | 19.3 samples/s +Step 43950/50462 | Loss: 2.5381 | LR: 2.09e-05 | 3117.6ms/step | 19.2 samples/s +Step 43960/50462 | Loss: 2.5208 | LR: 2.08e-05 | 3121.5ms/step | 19.2 samples/s +Step 43970/50462 | Loss: 2.5159 | LR: 2.07e-05 | 3119.5ms/step | 19.2 samples/s +Step 43980/50462 | Loss: 2.5068 | LR: 2.07e-05 | 3119.3ms/step | 19.2 samples/s +Step 43990/50462 | Loss: 2.5441 | LR: 2.06e-05 | 3121.2ms/step | 19.2 samples/s +Step 44000/50462 | Loss: 2.5496 | LR: 2.06e-05 | 3120.9ms/step | 19.2 samples/s +Validation at step 44000: Loss = 2.5378, Perplexity = 12.65 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-44000 +Step 44010/50462 | Loss: 2.5350 | LR: 2.05e-05 | 5699.2ms/step | 10.5 samples/s +Step 44020/50462 | Loss: 2.5268 | LR: 2.04e-05 | 3104.7ms/step | 19.3 samples/s +Step 44030/50462 | Loss: 2.5233 | LR: 2.04e-05 | 3112.7ms/step | 19.3 samples/s +Step 44040/50462 | Loss: 2.5174 | LR: 2.03e-05 | 3117.7ms/step | 19.2 samples/s +Step 44050/50462 | Loss: 2.5514 | LR: 2.02e-05 | 3118.5ms/step | 19.2 samples/s +Step 44060/50462 | Loss: 2.5502 | LR: 2.02e-05 | 3115.1ms/step | 19.3 samples/s +Step 44070/50462 | Loss: 2.5499 | LR: 2.01e-05 | 3115.1ms/step | 19.3 samples/s +Step 44080/50462 | Loss: 2.5314 | LR: 2.01e-05 | 3116.4ms/step | 19.3 samples/s +Step 44090/50462 | Loss: 2.5005 | LR: 2.00e-05 | 3654.5ms/step | 16.4 samples/s +Step 44100/50462 | Loss: 2.5201 | LR: 1.99e-05 | 3103.7ms/step | 19.3 samples/s +Step 44110/50462 | Loss: 2.5216 | LR: 1.99e-05 | 3112.2ms/step | 19.3 samples/s +Step 44120/50462 | Loss: 2.5381 | LR: 1.98e-05 | 3118.2ms/step | 19.2 samples/s +Step 44130/50462 | Loss: 2.5437 | LR: 1.97e-05 | 3117.5ms/step | 19.2 samples/s +Step 44140/50462 | Loss: 2.5127 | LR: 1.97e-05 | 3119.7ms/step | 19.2 samples/s +Step 44150/50462 | Loss: 2.5079 | LR: 1.96e-05 | 3115.8ms/step | 19.3 samples/s +Step 44160/50462 | Loss: 2.5370 | LR: 1.96e-05 | 3120.1ms/step | 19.2 samples/s +Step 44170/50462 | Loss: 2.5420 | LR: 1.95e-05 | 3644.0ms/step | 16.5 samples/s +Step 44180/50462 | Loss: 2.5076 | LR: 1.94e-05 | 3106.4ms/step | 19.3 samples/s +Step 44190/50462 | Loss: 2.5142 | LR: 1.94e-05 | 3112.0ms/step | 19.3 samples/s +Step 44200/50462 | Loss: 2.5360 | LR: 1.93e-05 | 3115.5ms/step | 19.3 samples/s +Step 44210/50462 | Loss: 2.5246 | LR: 1.93e-05 | 3117.5ms/step | 19.2 samples/s +Step 44220/50462 | Loss: 2.5299 | LR: 1.92e-05 | 3114.5ms/step | 19.3 samples/s +Step 44230/50462 | Loss: 2.5418 | LR: 1.91e-05 | 3113.2ms/step | 19.3 samples/s +Step 44240/50462 | Loss: 2.5294 | LR: 1.91e-05 | 3114.3ms/step | 19.3 samples/s +Step 44250/50462 | Loss: 2.5424 | LR: 1.90e-05 | 3112.1ms/step | 19.3 samples/s +Validation at step 44250: Loss = 2.5379, Perplexity = 12.65 +Step 44260/50462 | Loss: 2.5367 | LR: 1.90e-05 | 4772.4ms/step | 12.6 samples/s +Step 44270/50462 | Loss: 2.5083 | LR: 1.89e-05 | 3119.5ms/step | 19.2 samples/s +Step 44280/50462 | Loss: 2.5309 | LR: 1.88e-05 | 3119.8ms/step | 19.2 samples/s +Step 44290/50462 | Loss: 2.5274 | LR: 1.88e-05 | 3119.6ms/step | 19.2 samples/s +Step 44300/50462 | Loss: 2.4790 | LR: 1.87e-05 | 3120.1ms/step | 19.2 samples/s +Step 44310/50462 | Loss: 2.5077 | LR: 1.87e-05 | 3119.0ms/step | 19.2 samples/s +Step 44320/50462 | Loss: 2.5413 | LR: 1.86e-05 | 3117.0ms/step | 19.2 samples/s +Step 44330/50462 | Loss: 2.5315 | LR: 1.85e-05 | 3118.3ms/step | 19.2 samples/s +Step 44340/50462 | Loss: 2.5270 | LR: 1.85e-05 | 3115.0ms/step | 19.3 samples/s +Step 44350/50462 | Loss: 2.5331 | LR: 1.84e-05 | 3117.0ms/step | 19.2 samples/s +Step 44360/50462 | Loss: 2.5051 | LR: 1.84e-05 | 3114.7ms/step | 19.3 samples/s +Step 44370/50462 | Loss: 2.5304 | LR: 1.83e-05 | 3116.4ms/step | 19.3 samples/s +Step 44380/50462 | Loss: 2.5216 | LR: 1.82e-05 | 3118.1ms/step | 19.2 samples/s +Step 44390/50462 | Loss: 2.5418 | LR: 1.82e-05 | 3116.4ms/step | 19.3 samples/s +Step 44400/50462 | Loss: 2.5082 | LR: 1.81e-05 | 3117.8ms/step | 19.2 samples/s +Step 44410/50462 | Loss: 2.5170 | LR: 1.81e-05 | 3118.5ms/step | 19.2 samples/s +Step 44420/50462 | Loss: 2.5277 | LR: 1.80e-05 | 3118.8ms/step | 19.2 samples/s +Step 44430/50462 | Loss: 2.5355 | LR: 1.79e-05 | 3113.9ms/step | 19.3 samples/s +Step 44440/50462 | Loss: 2.5237 | LR: 1.79e-05 | 3116.2ms/step | 19.3 samples/s +Step 44450/50462 | Loss: 2.5126 | LR: 1.78e-05 | 3115.6ms/step | 19.3 samples/s +Step 44460/50462 | Loss: 2.5289 | LR: 1.78e-05 | 3114.9ms/step | 19.3 samples/s +Step 44470/50462 | Loss: 2.5410 | LR: 1.77e-05 | 3116.4ms/step | 19.3 samples/s +Step 44480/50462 | Loss: 2.5552 | LR: 1.77e-05 | 3111.8ms/step | 19.3 samples/s +Step 44490/50462 | Loss: 2.5516 | LR: 1.76e-05 | 3115.0ms/step | 19.3 samples/s +Step 44500/50462 | Loss: 2.5142 | LR: 1.75e-05 | 3119.6ms/step | 19.2 samples/s +Validation at step 44500: Loss = 2.5380, Perplexity = 12.65 +Step 44510/50462 | Loss: 2.5028 | LR: 1.75e-05 | 4780.0ms/step | 12.6 samples/s +Step 44520/50462 | Loss: 2.5018 | LR: 1.74e-05 | 3114.0ms/step | 19.3 samples/s +Step 44530/50462 | Loss: 2.5275 | LR: 1.74e-05 | 3118.0ms/step | 19.2 samples/s +Step 44540/50462 | Loss: 2.5165 | LR: 1.73e-05 | 3118.0ms/step | 19.2 samples/s +Step 44550/50462 | Loss: 2.5298 | LR: 1.72e-05 | 3118.0ms/step | 19.2 samples/s +Step 44560/50462 | Loss: 2.5345 | LR: 1.72e-05 | 3117.4ms/step | 19.2 samples/s +Step 44570/50462 | Loss: 2.5217 | LR: 1.71e-05 | 3119.1ms/step | 19.2 samples/s +Step 44580/50462 | Loss: 2.4996 | LR: 1.71e-05 | 3118.8ms/step | 19.2 samples/s +Step 44590/50462 | Loss: 2.5182 | LR: 1.70e-05 | 3116.3ms/step | 19.3 samples/s +Step 44600/50462 | Loss: 2.5414 | LR: 1.70e-05 | 3117.0ms/step | 19.2 samples/s +Step 44610/50462 | Loss: 2.5233 | LR: 1.69e-05 | 3116.7ms/step | 19.3 samples/s +Step 44620/50462 | Loss: 2.5405 | LR: 1.68e-05 | 3116.9ms/step | 19.2 samples/s +Step 44630/50462 | Loss: 2.5255 | LR: 1.68e-05 | 3118.9ms/step | 19.2 samples/s +Step 44640/50462 | Loss: 2.5170 | LR: 1.67e-05 | 3120.9ms/step | 19.2 samples/s +Step 44650/50462 | Loss: 2.5060 | LR: 1.67e-05 | 3117.8ms/step | 19.2 samples/s +Step 44660/50462 | Loss: 2.5434 | LR: 1.66e-05 | 3120.2ms/step | 19.2 samples/s +Step 44670/50462 | Loss: 2.5524 | LR: 1.66e-05 | 3118.3ms/step | 19.2 samples/s +Step 44680/50462 | Loss: 2.5382 | LR: 1.65e-05 | 3118.9ms/step | 19.2 samples/s +Step 44690/50462 | Loss: 2.5349 | LR: 1.64e-05 | 3119.5ms/step | 19.2 samples/s +Step 44700/50462 | Loss: 2.5405 | LR: 1.64e-05 | 3117.6ms/step | 19.2 samples/s +Step 44710/50462 | Loss: 2.5244 | LR: 1.63e-05 | 3115.7ms/step | 19.3 samples/s +Step 44720/50462 | Loss: 2.5257 | LR: 1.63e-05 | 3115.7ms/step | 19.3 samples/s +Step 44730/50462 | Loss: 2.5144 | LR: 1.62e-05 | 3122.1ms/step | 19.2 samples/s +Step 44740/50462 | Loss: 2.5095 | LR: 1.62e-05 | 3119.3ms/step | 19.2 samples/s +Switching attention mode for step 44750: dense (dense step interval) +Step 44750/50462 | Loss: 2.5325 | LR: 1.61e-05 | 3333.4ms/step | 18.0 samples/s +Validation at step 44750: Loss = 2.5207, Perplexity = 12.44 +Step 44760/50462 | Loss: 2.5321 | LR: 1.61e-05 | 7829.3ms/step | 7.7 samples/s +Step 44770/50462 | Loss: 2.4780 | LR: 1.60e-05 | 5259.1ms/step | 11.4 samples/s +Step 44780/50462 | Loss: 2.5055 | LR: 1.59e-05 | 5259.9ms/step | 11.4 samples/s +Step 44790/50462 | Loss: 2.4949 | LR: 1.59e-05 | 5256.9ms/step | 11.4 samples/s +Step 44800/50462 | Loss: 2.4964 | LR: 1.58e-05 | 5255.4ms/step | 11.4 samples/s +Step 44810/50462 | Loss: 2.5222 | LR: 1.58e-05 | 5255.1ms/step | 11.4 samples/s +Step 44820/50462 | Loss: 2.5114 | LR: 1.57e-05 | 5255.8ms/step | 11.4 samples/s +Step 44830/50462 | Loss: 2.5332 | LR: 1.57e-05 | 5257.0ms/step | 11.4 samples/s +Step 44840/50462 | Loss: 2.4889 | LR: 1.56e-05 | 5259.2ms/step | 11.4 samples/s +Step 44850/50462 | Loss: 2.4931 | LR: 1.56e-05 | 5257.0ms/step | 11.4 samples/s +Step 44860/50462 | Loss: 2.4917 | LR: 1.55e-05 | 5257.9ms/step | 11.4 samples/s +Step 44870/50462 | Loss: 2.4815 | LR: 1.54e-05 | 5259.1ms/step | 11.4 samples/s +Step 44880/50462 | Loss: 2.5057 | LR: 1.54e-05 | 5262.3ms/step | 11.4 samples/s +Step 44890/50462 | Loss: 2.4690 | LR: 1.53e-05 | 5261.0ms/step | 11.4 samples/s +Step 44900/50462 | Loss: 2.4688 | LR: 1.53e-05 | 5261.7ms/step | 11.4 samples/s +Step 44910/50462 | Loss: 2.4909 | LR: 1.52e-05 | 5257.8ms/step | 11.4 samples/s +Step 44920/50462 | Loss: 2.4934 | LR: 1.52e-05 | 5255.6ms/step | 11.4 samples/s +Step 44930/50462 | Loss: 2.4967 | LR: 1.51e-05 | 5260.3ms/step | 11.4 samples/s +Step 44940/50462 | Loss: 2.4990 | LR: 1.51e-05 | 5257.5ms/step | 11.4 samples/s +Step 44950/50462 | Loss: 2.4565 | LR: 1.50e-05 | 5259.1ms/step | 11.4 samples/s +Step 44960/50462 | Loss: 2.5356 | LR: 1.50e-05 | 5254.4ms/step | 11.4 samples/s +Step 44970/50462 | Loss: 2.4856 | LR: 1.49e-05 | 5255.0ms/step | 11.4 samples/s +Step 44980/50462 | Loss: 2.4975 | LR: 1.49e-05 | 5257.5ms/step | 11.4 samples/s +Step 44990/50462 | Loss: 2.5260 | LR: 1.48e-05 | 5259.6ms/step | 11.4 samples/s +Step 45000/50462 | Loss: 2.5166 | LR: 1.47e-05 | 5258.4ms/step | 11.4 samples/s +Validation at step 45000: Loss = 2.5146, Perplexity = 12.36 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-45000 +Switching attention mode for step 45001: sparse_stochastic (configured sparse attention) +Step 45010/50462 | Loss: 2.5360 | LR: 1.47e-05 | 6617.9ms/step | 9.1 samples/s +Step 45020/50462 | Loss: 2.5184 | LR: 1.46e-05 | 3110.8ms/step | 19.3 samples/s +Step 45030/50462 | Loss: 2.5242 | LR: 1.46e-05 | 3116.4ms/step | 19.3 samples/s +Step 45040/50462 | Loss: 2.5369 | LR: 1.45e-05 | 3118.8ms/step | 19.2 samples/s +Step 45050/50462 | Loss: 2.5109 | LR: 1.45e-05 | 3119.6ms/step | 19.2 samples/s +Step 45060/50462 | Loss: 2.5342 | LR: 1.44e-05 | 3120.0ms/step | 19.2 samples/s +Step 45070/50462 | Loss: 2.5280 | LR: 1.44e-05 | 3125.2ms/step | 19.2 samples/s +Step 45080/50462 | Loss: 2.5196 | LR: 1.43e-05 | 3143.4ms/step | 19.1 samples/s +Step 45090/50462 | Loss: 2.5195 | LR: 1.43e-05 | 3113.3ms/step | 19.3 samples/s +Step 45100/50462 | Loss: 2.5536 | LR: 1.42e-05 | 3113.6ms/step | 19.3 samples/s +Step 45110/50462 | Loss: 2.5519 | LR: 1.42e-05 | 3118.4ms/step | 19.2 samples/s +Step 45120/50462 | Loss: 2.5350 | LR: 1.41e-05 | 3120.7ms/step | 19.2 samples/s +Step 45130/50462 | Loss: 2.5432 | LR: 1.41e-05 | 3118.6ms/step | 19.2 samples/s +Step 45140/50462 | Loss: 2.5132 | LR: 1.40e-05 | 3118.8ms/step | 19.2 samples/s +Step 45150/50462 | Loss: 2.5370 | LR: 1.40e-05 | 3121.1ms/step | 19.2 samples/s +Step 45160/50462 | Loss: 2.5437 | LR: 1.39e-05 | 3122.0ms/step | 19.2 samples/s +Step 45170/50462 | Loss: 2.5053 | LR: 1.39e-05 | 3121.5ms/step | 19.2 samples/s +Step 45180/50462 | Loss: 2.5332 | LR: 1.38e-05 | 3121.4ms/step | 19.2 samples/s +Step 45190/50462 | Loss: 2.5198 | LR: 1.37e-05 | 3122.2ms/step | 19.2 samples/s +Step 45200/50462 | Loss: 2.5430 | LR: 1.37e-05 | 3119.1ms/step | 19.2 samples/s +Step 45210/50462 | Loss: 2.5284 | LR: 1.36e-05 | 3119.2ms/step | 19.2 samples/s +Step 45220/50462 | Loss: 2.5137 | LR: 1.36e-05 | 3119.1ms/step | 19.2 samples/s +Step 45230/50462 | Loss: 2.5097 | LR: 1.35e-05 | 3117.8ms/step | 19.2 samples/s +Step 45240/50462 | Loss: 2.5437 | LR: 1.35e-05 | 3119.4ms/step | 19.2 samples/s +Step 45250/50462 | Loss: 2.5215 | LR: 1.34e-05 | 3121.3ms/step | 19.2 samples/s +Validation at step 45250: Loss = 2.5377, Perplexity = 12.65 +Step 45260/50462 | Loss: 2.5332 | LR: 1.34e-05 | 4783.6ms/step | 12.5 samples/s +Step 45270/50462 | Loss: 2.5415 | LR: 1.33e-05 | 3118.2ms/step | 19.2 samples/s +Step 45280/50462 | Loss: 2.4988 | LR: 1.33e-05 | 3118.3ms/step | 19.2 samples/s +Step 45290/50462 | Loss: 2.5252 | LR: 1.32e-05 | 3155.2ms/step | 19.0 samples/s +Step 45300/50462 | Loss: 2.5444 | LR: 1.32e-05 | 3116.5ms/step | 19.3 samples/s +Step 45310/50462 | Loss: 2.5129 | LR: 1.31e-05 | 3147.8ms/step | 19.1 samples/s +Step 45320/50462 | Loss: 2.5203 | LR: 1.31e-05 | 3119.8ms/step | 19.2 samples/s +Step 45330/50462 | Loss: 2.5364 | LR: 1.30e-05 | 3119.1ms/step | 19.2 samples/s +Step 45340/50462 | Loss: 2.5285 | LR: 1.30e-05 | 3117.2ms/step | 19.2 samples/s +Step 45350/50462 | Loss: 2.5253 | LR: 1.29e-05 | 3117.1ms/step | 19.2 samples/s +Step 45360/50462 | Loss: 2.5454 | LR: 1.29e-05 | 3116.8ms/step | 19.3 samples/s +Step 45370/50462 | Loss: 2.5183 | LR: 1.28e-05 | 3114.5ms/step | 19.3 samples/s +Step 45380/50462 | Loss: 2.5140 | LR: 1.28e-05 | 3118.8ms/step | 19.2 samples/s +Step 45390/50462 | Loss: 2.5209 | LR: 1.27e-05 | 3115.5ms/step | 19.3 samples/s +Step 45400/50462 | Loss: 2.5052 | LR: 1.27e-05 | 3117.6ms/step | 19.2 samples/s +Step 45410/50462 | Loss: 2.5498 | LR: 1.26e-05 | 3118.4ms/step | 19.2 samples/s +Step 45420/50462 | Loss: 2.5199 | LR: 1.26e-05 | 3117.7ms/step | 19.2 samples/s +Step 45430/50462 | Loss: 2.4952 | LR: 1.25e-05 | 3119.7ms/step | 19.2 samples/s +Step 45440/50462 | Loss: 2.5365 | LR: 1.25e-05 | 3632.3ms/step | 16.5 samples/s +Step 45450/50462 | Loss: 2.5270 | LR: 1.24e-05 | 3107.8ms/step | 19.3 samples/s +Step 45460/50462 | Loss: 2.5165 | LR: 1.24e-05 | 3116.8ms/step | 19.3 samples/s +Step 45470/50462 | Loss: 2.5259 | LR: 1.23e-05 | 3118.2ms/step | 19.2 samples/s +Step 45480/50462 | Loss: 2.5155 | LR: 1.23e-05 | 3118.1ms/step | 19.2 samples/s +Step 45490/50462 | Loss: 2.5371 | LR: 1.22e-05 | 3120.8ms/step | 19.2 samples/s +Step 45500/50462 | Loss: 2.5364 | LR: 1.22e-05 | 3120.1ms/step | 19.2 samples/s +Validation at step 45500: Loss = 2.5374, Perplexity = 12.65 +Step 45510/50462 | Loss: 2.5111 | LR: 1.21e-05 | 4781.6ms/step | 12.5 samples/s +Step 45520/50462 | Loss: 2.5572 | LR: 1.21e-05 | 3114.9ms/step | 19.3 samples/s +Step 45530/50462 | Loss: 2.5035 | LR: 1.20e-05 | 3118.3ms/step | 19.2 samples/s +Step 45540/50462 | Loss: 2.5524 | LR: 1.20e-05 | 3119.1ms/step | 19.2 samples/s +Step 45550/50462 | Loss: 2.5115 | LR: 1.19e-05 | 3119.9ms/step | 19.2 samples/s +Step 45560/50462 | Loss: 2.5360 | LR: 1.19e-05 | 3122.2ms/step | 19.2 samples/s +Step 45570/50462 | Loss: 2.5405 | LR: 1.19e-05 | 3122.6ms/step | 19.2 samples/s +Step 45580/50462 | Loss: 2.5182 | LR: 1.18e-05 | 3121.1ms/step | 19.2 samples/s +Step 45590/50462 | Loss: 2.5063 | LR: 1.18e-05 | 3120.3ms/step | 19.2 samples/s +Step 45600/50462 | Loss: 2.5151 | LR: 1.17e-05 | 3119.8ms/step | 19.2 samples/s +Step 45610/50462 | Loss: 2.4939 | LR: 1.17e-05 | 3114.3ms/step | 19.3 samples/s +Step 45620/50462 | Loss: 2.5256 | LR: 1.16e-05 | 3115.7ms/step | 19.3 samples/s +Step 45630/50462 | Loss: 2.5425 | LR: 1.16e-05 | 3117.0ms/step | 19.2 samples/s +Step 45640/50462 | Loss: 2.5203 | LR: 1.15e-05 | 3115.3ms/step | 19.3 samples/s +Step 45650/50462 | Loss: 2.5448 | LR: 1.15e-05 | 3117.8ms/step | 19.2 samples/s +Step 45660/50462 | Loss: 2.5298 | LR: 1.14e-05 | 3120.0ms/step | 19.2 samples/s +Step 45670/50462 | Loss: 2.4975 | LR: 1.14e-05 | 3118.4ms/step | 19.2 samples/s +Step 45680/50462 | Loss: 2.5165 | LR: 1.13e-05 | 3118.0ms/step | 19.2 samples/s +Step 45690/50462 | Loss: 2.5365 | LR: 1.13e-05 | 3118.9ms/step | 19.2 samples/s +Step 45700/50462 | Loss: 2.5302 | LR: 1.12e-05 | 3120.6ms/step | 19.2 samples/s +Step 45710/50462 | Loss: 2.5321 | LR: 1.12e-05 | 3121.2ms/step | 19.2 samples/s +Step 45720/50462 | Loss: 2.5231 | LR: 1.11e-05 | 3119.9ms/step | 19.2 samples/s +Step 45730/50462 | Loss: 2.5061 | LR: 1.11e-05 | 3116.3ms/step | 19.3 samples/s +Step 45740/50462 | Loss: 2.5185 | LR: 1.10e-05 | 3117.4ms/step | 19.2 samples/s +Step 45750/50462 | Loss: 2.5369 | LR: 1.10e-05 | 3119.4ms/step | 19.2 samples/s +Validation at step 45750: Loss = 2.5380, Perplexity = 12.65 +Step 45760/50462 | Loss: 2.4909 | LR: 1.10e-05 | 4787.4ms/step | 12.5 samples/s +Step 45770/50462 | Loss: 2.5238 | LR: 1.09e-05 | 3118.2ms/step | 19.2 samples/s +Step 45780/50462 | Loss: 2.5165 | LR: 1.09e-05 | 3117.5ms/step | 19.2 samples/s +Step 45790/50462 | Loss: 2.5544 | LR: 1.08e-05 | 3119.6ms/step | 19.2 samples/s +Step 45800/50462 | Loss: 2.4685 | LR: 1.08e-05 | 3119.4ms/step | 19.2 samples/s +Step 45810/50462 | Loss: 2.5285 | LR: 1.07e-05 | 3116.5ms/step | 19.3 samples/s +Step 45820/50462 | Loss: 2.5245 | LR: 1.07e-05 | 3120.2ms/step | 19.2 samples/s +Step 45830/50462 | Loss: 2.5308 | LR: 1.06e-05 | 3120.9ms/step | 19.2 samples/s +Step 45840/50462 | Loss: 2.5445 | LR: 1.06e-05 | 3114.8ms/step | 19.3 samples/s +Step 45850/50462 | Loss: 2.5260 | LR: 1.05e-05 | 3117.3ms/step | 19.2 samples/s +Step 45860/50462 | Loss: 2.5512 | LR: 1.05e-05 | 3117.9ms/step | 19.2 samples/s +Step 45870/50462 | Loss: 2.5357 | LR: 1.05e-05 | 3119.8ms/step | 19.2 samples/s +Step 45880/50462 | Loss: 2.5365 | LR: 1.04e-05 | 3120.8ms/step | 19.2 samples/s +Step 45890/50462 | Loss: 2.4990 | LR: 1.04e-05 | 3117.4ms/step | 19.2 samples/s +Step 45900/50462 | Loss: 2.5220 | LR: 1.03e-05 | 3116.0ms/step | 19.3 samples/s +Step 45910/50462 | Loss: 2.5526 | LR: 1.03e-05 | 3121.9ms/step | 19.2 samples/s +Step 45920/50462 | Loss: 2.5046 | LR: 1.02e-05 | 3119.7ms/step | 19.2 samples/s +Step 45930/50462 | Loss: 2.5172 | LR: 1.02e-05 | 3119.1ms/step | 19.2 samples/s +Step 45940/50462 | Loss: 2.5356 | LR: 1.01e-05 | 3119.2ms/step | 19.2 samples/s +Step 45950/50462 | Loss: 2.5237 | LR: 1.01e-05 | 3113.8ms/step | 19.3 samples/s +Step 45960/50462 | Loss: 2.5085 | LR: 1.01e-05 | 3116.8ms/step | 19.3 samples/s +Step 45970/50462 | Loss: 2.5144 | LR: 1.00e-05 | 3117.5ms/step | 19.2 samples/s +Step 45980/50462 | Loss: 2.5592 | LR: 9.96e-06 | 3114.9ms/step | 19.3 samples/s +Step 45990/50462 | Loss: 2.5123 | LR: 9.92e-06 | 3117.5ms/step | 19.2 samples/s +Step 46000/50462 | Loss: 2.5268 | LR: 9.87e-06 | 3119.3ms/step | 19.2 samples/s +Validation at step 46000: Loss = 2.5377, Perplexity = 12.65 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-46000 +Step 46010/50462 | Loss: 2.5295 | LR: 9.83e-06 | 5673.5ms/step | 10.6 samples/s +Step 46020/50462 | Loss: 2.5167 | LR: 9.79e-06 | 3105.6ms/step | 19.3 samples/s +Step 46030/50462 | Loss: 2.5407 | LR: 9.74e-06 | 3114.4ms/step | 19.3 samples/s +Step 46040/50462 | Loss: 2.5401 | LR: 9.70e-06 | 3116.8ms/step | 19.3 samples/s +Step 46050/50462 | Loss: 2.5224 | LR: 9.65e-06 | 3118.4ms/step | 19.2 samples/s +Step 46060/50462 | Loss: 2.5176 | LR: 9.61e-06 | 3117.9ms/step | 19.2 samples/s +Step 46070/50462 | Loss: 2.5371 | LR: 9.57e-06 | 3116.8ms/step | 19.3 samples/s +Step 46080/50462 | Loss: 2.5311 | LR: 9.52e-06 | 3119.0ms/step | 19.2 samples/s +Step 46090/50462 | Loss: 2.5011 | LR: 9.48e-06 | 3118.6ms/step | 19.2 samples/s +Step 46100/50462 | Loss: 2.5275 | LR: 9.44e-06 | 3121.4ms/step | 19.2 samples/s +Step 46110/50462 | Loss: 2.5230 | LR: 9.40e-06 | 3121.3ms/step | 19.2 samples/s +Step 46120/50462 | Loss: 2.5181 | LR: 9.35e-06 | 3120.4ms/step | 19.2 samples/s +Step 46130/50462 | Loss: 2.5201 | LR: 9.31e-06 | 3123.3ms/step | 19.2 samples/s +Step 46140/50462 | Loss: 2.5192 | LR: 9.27e-06 | 3122.2ms/step | 19.2 samples/s +Step 46150/50462 | Loss: 2.5169 | LR: 9.22e-06 | 3124.6ms/step | 19.2 samples/s +Step 46160/50462 | Loss: 2.5359 | LR: 9.18e-06 | 3119.3ms/step | 19.2 samples/s +Step 46170/50462 | Loss: 2.5358 | LR: 9.14e-06 | 3120.8ms/step | 19.2 samples/s +Step 46180/50462 | Loss: 2.5060 | LR: 9.10e-06 | 3118.6ms/step | 19.2 samples/s +Step 46190/50462 | Loss: 2.5074 | LR: 9.06e-06 | 3119.9ms/step | 19.2 samples/s +Step 46200/50462 | Loss: 2.5136 | LR: 9.01e-06 | 3121.0ms/step | 19.2 samples/s +Step 46210/50462 | Loss: 2.5313 | LR: 8.97e-06 | 3121.2ms/step | 19.2 samples/s +Step 46220/50462 | Loss: 2.5330 | LR: 8.93e-06 | 3120.2ms/step | 19.2 samples/s +Step 46230/50462 | Loss: 2.5400 | LR: 8.89e-06 | 3121.0ms/step | 19.2 samples/s +Step 46240/50462 | Loss: 2.5489 | LR: 8.85e-06 | 3119.5ms/step | 19.2 samples/s +Step 46250/50462 | Loss: 2.5460 | LR: 8.80e-06 | 3121.4ms/step | 19.2 samples/s +Validation at step 46250: Loss = 2.5379, Perplexity = 12.65 +Step 46260/50462 | Loss: 2.5466 | LR: 8.76e-06 | 4793.5ms/step | 12.5 samples/s +Step 46270/50462 | Loss: 2.5326 | LR: 8.72e-06 | 3117.8ms/step | 19.2 samples/s +Step 46280/50462 | Loss: 2.5635 | LR: 8.68e-06 | 3120.2ms/step | 19.2 samples/s +Step 46290/50462 | Loss: 2.5514 | LR: 8.64e-06 | 3122.0ms/step | 19.2 samples/s +Step 46300/50462 | Loss: 2.5207 | LR: 8.60e-06 | 3646.9ms/step | 16.5 samples/s +Step 46310/50462 | Loss: 2.5289 | LR: 8.56e-06 | 3114.9ms/step | 19.3 samples/s +Step 46320/50462 | Loss: 2.5551 | LR: 8.52e-06 | 3120.1ms/step | 19.2 samples/s +Step 46330/50462 | Loss: 2.5344 | LR: 8.47e-06 | 3122.9ms/step | 19.2 samples/s +Step 46340/50462 | Loss: 2.5331 | LR: 8.43e-06 | 3120.8ms/step | 19.2 samples/s +Step 46350/50462 | Loss: 2.5548 | LR: 8.39e-06 | 3119.7ms/step | 19.2 samples/s +Step 46360/50462 | Loss: 2.5175 | LR: 8.35e-06 | 3121.2ms/step | 19.2 samples/s +Step 46370/50462 | Loss: 2.5021 | LR: 8.31e-06 | 3122.2ms/step | 19.2 samples/s +Step 46380/50462 | Loss: 2.5273 | LR: 8.27e-06 | 3641.1ms/step | 16.5 samples/s +Step 46390/50462 | Loss: 2.5172 | LR: 8.23e-06 | 3097.9ms/step | 19.4 samples/s +Step 46400/50462 | Loss: 2.5089 | LR: 8.19e-06 | 3112.6ms/step | 19.3 samples/s +Step 46410/50462 | Loss: 2.5207 | LR: 8.15e-06 | 3119.1ms/step | 19.2 samples/s +Step 46420/50462 | Loss: 2.5212 | LR: 8.11e-06 | 3117.8ms/step | 19.2 samples/s +Step 46430/50462 | Loss: 2.5417 | LR: 8.07e-06 | 3116.2ms/step | 19.3 samples/s +Step 46440/50462 | Loss: 2.5241 | LR: 8.03e-06 | 3118.7ms/step | 19.2 samples/s +Step 46450/50462 | Loss: 2.5509 | LR: 7.99e-06 | 3119.0ms/step | 19.2 samples/s +Step 46460/50462 | Loss: 2.5288 | LR: 7.95e-06 | 3119.8ms/step | 19.2 samples/s +Step 46470/50462 | Loss: 2.5207 | LR: 7.91e-06 | 3118.9ms/step | 19.2 samples/s +Step 46480/50462 | Loss: 2.5151 | LR: 7.87e-06 | 3115.9ms/step | 19.3 samples/s +Step 46490/50462 | Loss: 2.5103 | LR: 7.83e-06 | 3114.2ms/step | 19.3 samples/s +Step 46500/50462 | Loss: 2.5291 | LR: 7.80e-06 | 3118.1ms/step | 19.2 samples/s +Validation at step 46500: Loss = 2.5379, Perplexity = 12.65 +Step 46510/50462 | Loss: 2.5225 | LR: 7.76e-06 | 4781.6ms/step | 12.5 samples/s +Step 46520/50462 | Loss: 2.5454 | LR: 7.72e-06 | 3113.4ms/step | 19.3 samples/s +Step 46530/50462 | Loss: 2.5259 | LR: 7.68e-06 | 3117.8ms/step | 19.2 samples/s +Step 46540/50462 | Loss: 2.5388 | LR: 7.64e-06 | 3116.2ms/step | 19.3 samples/s +Step 46550/50462 | Loss: 2.5000 | LR: 7.60e-06 | 3115.9ms/step | 19.3 samples/s +Step 46560/50462 | Loss: 2.5162 | LR: 7.56e-06 | 3121.1ms/step | 19.2 samples/s +Step 46570/50462 | Loss: 2.5282 | LR: 7.52e-06 | 3117.9ms/step | 19.2 samples/s +Step 46580/50462 | Loss: 2.5194 | LR: 7.49e-06 | 3118.1ms/step | 19.2 samples/s +Step 46590/50462 | Loss: 2.5120 | LR: 7.45e-06 | 3121.2ms/step | 19.2 samples/s +Step 46600/50462 | Loss: 2.5279 | LR: 7.41e-06 | 3120.4ms/step | 19.2 samples/s +Step 46610/50462 | Loss: 2.5254 | LR: 7.37e-06 | 3120.1ms/step | 19.2 samples/s +Step 46620/50462 | Loss: 2.5082 | LR: 7.33e-06 | 3119.6ms/step | 19.2 samples/s +Step 46630/50462 | Loss: 2.5228 | LR: 7.29e-06 | 3116.1ms/step | 19.3 samples/s +Step 46640/50462 | Loss: 2.5347 | LR: 7.26e-06 | 3118.1ms/step | 19.2 samples/s +Step 46650/50462 | Loss: 2.5273 | LR: 7.22e-06 | 3118.7ms/step | 19.2 samples/s +Step 46660/50462 | Loss: 2.5171 | LR: 7.18e-06 | 3119.4ms/step | 19.2 samples/s +Step 46670/50462 | Loss: 2.5321 | LR: 7.14e-06 | 3120.6ms/step | 19.2 samples/s +Step 46680/50462 | Loss: 2.5397 | LR: 7.11e-06 | 3116.9ms/step | 19.2 samples/s +Step 46690/50462 | Loss: 2.5568 | LR: 7.07e-06 | 3117.4ms/step | 19.2 samples/s +Step 46700/50462 | Loss: 2.5168 | LR: 7.03e-06 | 3117.7ms/step | 19.2 samples/s +Step 46710/50462 | Loss: 2.5499 | LR: 6.99e-06 | 3115.5ms/step | 19.3 samples/s +Step 46720/50462 | Loss: 2.5338 | LR: 6.96e-06 | 3116.6ms/step | 19.3 samples/s +Step 46730/50462 | Loss: 2.5138 | LR: 6.92e-06 | 3117.3ms/step | 19.2 samples/s +Step 46740/50462 | Loss: 2.5403 | LR: 6.88e-06 | 3115.6ms/step | 19.3 samples/s +Step 46750/50462 | Loss: 2.5230 | LR: 6.85e-06 | 3115.3ms/step | 19.3 samples/s +Validation at step 46750: Loss = 2.5379, Perplexity = 12.65 +Step 46760/50462 | Loss: 2.5382 | LR: 6.81e-06 | 4773.3ms/step | 12.6 samples/s +Step 46770/50462 | Loss: 2.5273 | LR: 6.77e-06 | 3118.4ms/step | 19.2 samples/s +Step 46780/50462 | Loss: 2.5527 | LR: 6.74e-06 | 3117.1ms/step | 19.2 samples/s +Step 46790/50462 | Loss: 2.5095 | LR: 6.70e-06 | 3119.5ms/step | 19.2 samples/s +Step 46800/50462 | Loss: 2.5476 | LR: 6.66e-06 | 3119.0ms/step | 19.2 samples/s +Step 46810/50462 | Loss: 2.5440 | LR: 6.63e-06 | 3116.6ms/step | 19.3 samples/s +Step 46820/50462 | Loss: 2.5072 | LR: 6.59e-06 | 3118.6ms/step | 19.2 samples/s +Step 46830/50462 | Loss: 2.5057 | LR: 6.56e-06 | 3118.7ms/step | 19.2 samples/s +Step 46840/50462 | Loss: 2.5316 | LR: 6.52e-06 | 3117.8ms/step | 19.2 samples/s +Step 46850/50462 | Loss: 2.5269 | LR: 6.48e-06 | 3117.8ms/step | 19.2 samples/s +Step 46860/50462 | Loss: 2.5269 | LR: 6.45e-06 | 3121.1ms/step | 19.2 samples/s +Step 46870/50462 | Loss: 2.4845 | LR: 6.41e-06 | 3121.2ms/step | 19.2 samples/s +Step 46880/50462 | Loss: 2.5198 | LR: 6.38e-06 | 3118.0ms/step | 19.2 samples/s +Step 46890/50462 | Loss: 2.5263 | LR: 6.34e-06 | 3120.8ms/step | 19.2 samples/s +Step 46900/50462 | Loss: 2.5181 | LR: 6.31e-06 | 3117.8ms/step | 19.2 samples/s +Step 46910/50462 | Loss: 2.5106 | LR: 6.27e-06 | 3121.9ms/step | 19.2 samples/s +Step 46920/50462 | Loss: 2.5497 | LR: 6.24e-06 | 3118.9ms/step | 19.2 samples/s +Step 46930/50462 | Loss: 2.5257 | LR: 6.20e-06 | 3118.8ms/step | 19.2 samples/s +Step 46940/50462 | Loss: 2.5277 | LR: 6.17e-06 | 3118.9ms/step | 19.2 samples/s +Step 46950/50462 | Loss: 2.5107 | LR: 6.13e-06 | 3118.8ms/step | 19.2 samples/s +Step 46960/50462 | Loss: 2.5348 | LR: 6.10e-06 | 3115.1ms/step | 19.3 samples/s +Step 46970/50462 | Loss: 2.5380 | LR: 6.06e-06 | 3116.7ms/step | 19.3 samples/s +Step 46980/50462 | Loss: 2.5367 | LR: 6.03e-06 | 3115.2ms/step | 19.3 samples/s +Step 46990/50462 | Loss: 2.5571 | LR: 5.99e-06 | 3116.4ms/step | 19.3 samples/s +Step 47000/50462 | Loss: 2.5380 | LR: 5.96e-06 | 3116.7ms/step | 19.3 samples/s +Validation at step 47000: Loss = 2.5377, Perplexity = 12.65 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-47000 +Step 47010/50462 | Loss: 2.5489 | LR: 5.93e-06 | 5727.6ms/step | 10.5 samples/s +Step 47020/50462 | Loss: 2.5258 | LR: 5.89e-06 | 3112.2ms/step | 19.3 samples/s +Step 47030/50462 | Loss: 2.5462 | LR: 5.86e-06 | 3116.2ms/step | 19.3 samples/s +Step 47040/50462 | Loss: 2.4910 | LR: 5.82e-06 | 3122.0ms/step | 19.2 samples/s +Step 47050/50462 | Loss: 2.5284 | LR: 5.79e-06 | 3119.5ms/step | 19.2 samples/s +Step 47060/50462 | Loss: 2.5265 | LR: 5.76e-06 | 3121.0ms/step | 19.2 samples/s +Step 47070/50462 | Loss: 2.5182 | LR: 5.72e-06 | 3122.7ms/step | 19.2 samples/s +Step 47080/50462 | Loss: 2.5277 | LR: 5.69e-06 | 3117.1ms/step | 19.2 samples/s +Step 47090/50462 | Loss: 2.5217 | LR: 5.65e-06 | 3118.9ms/step | 19.2 samples/s +Step 47100/50462 | Loss: 2.5259 | LR: 5.62e-06 | 3119.4ms/step | 19.2 samples/s +Step 47110/50462 | Loss: 2.5293 | LR: 5.59e-06 | 3124.0ms/step | 19.2 samples/s +Step 47120/50462 | Loss: 2.4800 | LR: 5.56e-06 | 3121.7ms/step | 19.2 samples/s +Step 47130/50462 | Loss: 2.5259 | LR: 5.52e-06 | 3120.0ms/step | 19.2 samples/s +Step 47140/50462 | Loss: 2.5388 | LR: 5.49e-06 | 3117.0ms/step | 19.2 samples/s +Step 47150/50462 | Loss: 2.4959 | LR: 5.46e-06 | 3121.3ms/step | 19.2 samples/s +Step 47160/50462 | Loss: 2.5374 | LR: 5.42e-06 | 3119.2ms/step | 19.2 samples/s +Step 47170/50462 | Loss: 2.5200 | LR: 5.39e-06 | 3118.5ms/step | 19.2 samples/s +Step 47180/50462 | Loss: 2.5298 | LR: 5.36e-06 | 3113.9ms/step | 19.3 samples/s +Step 47190/50462 | Loss: 2.5244 | LR: 5.33e-06 | 3114.3ms/step | 19.3 samples/s +Step 47200/50462 | Loss: 2.5216 | LR: 5.29e-06 | 3115.4ms/step | 19.3 samples/s +Step 47210/50462 | Loss: 2.4956 | LR: 5.26e-06 | 3120.3ms/step | 19.2 samples/s +Step 47220/50462 | Loss: 2.5364 | LR: 5.23e-06 | 3121.1ms/step | 19.2 samples/s +Step 47230/50462 | Loss: 2.5350 | LR: 5.20e-06 | 3119.7ms/step | 19.2 samples/s +Step 47240/50462 | Loss: 2.5033 | LR: 5.16e-06 | 3121.2ms/step | 19.2 samples/s +Step 47250/50462 | Loss: 2.5341 | LR: 5.13e-06 | 3115.7ms/step | 19.3 samples/s +Validation at step 47250: Loss = 2.5377, Perplexity = 12.65 +Step 47260/50462 | Loss: 2.5277 | LR: 5.10e-06 | 4777.4ms/step | 12.6 samples/s +Step 47270/50462 | Loss: 2.5400 | LR: 5.07e-06 | 3120.3ms/step | 19.2 samples/s +Step 47280/50462 | Loss: 2.5140 | LR: 5.04e-06 | 3119.1ms/step | 19.2 samples/s +Step 47290/50462 | Loss: 2.5416 | LR: 5.01e-06 | 3114.3ms/step | 19.3 samples/s +Step 47300/50462 | Loss: 2.5400 | LR: 4.97e-06 | 3114.6ms/step | 19.3 samples/s +Step 47310/50462 | Loss: 2.5019 | LR: 4.94e-06 | 3117.1ms/step | 19.2 samples/s +Step 47320/50462 | Loss: 2.5187 | LR: 4.91e-06 | 3117.1ms/step | 19.2 samples/s +Step 47330/50462 | Loss: 2.5226 | LR: 4.88e-06 | 3117.8ms/step | 19.2 samples/s +Step 47340/50462 | Loss: 2.4870 | LR: 4.85e-06 | 3119.5ms/step | 19.2 samples/s +Step 47350/50462 | Loss: 2.5353 | LR: 4.82e-06 | 3120.9ms/step | 19.2 samples/s +Step 47360/50462 | Loss: 2.5051 | LR: 4.79e-06 | 3119.3ms/step | 19.2 samples/s +Step 47370/50462 | Loss: 2.5436 | LR: 4.76e-06 | 3120.7ms/step | 19.2 samples/s +Step 47380/50462 | Loss: 2.5357 | LR: 4.73e-06 | 3119.1ms/step | 19.2 samples/s +Step 47390/50462 | Loss: 2.5191 | LR: 4.70e-06 | 3119.6ms/step | 19.2 samples/s +Step 47400/50462 | Loss: 2.5211 | LR: 4.67e-06 | 3120.9ms/step | 19.2 samples/s +Step 47410/50462 | Loss: 2.5335 | LR: 4.64e-06 | 3118.3ms/step | 19.2 samples/s +Step 47420/50462 | Loss: 2.4843 | LR: 4.61e-06 | 3114.5ms/step | 19.3 samples/s +Step 47430/50462 | Loss: 2.5130 | LR: 4.58e-06 | 3114.0ms/step | 19.3 samples/s +Step 47440/50462 | Loss: 2.5219 | LR: 4.55e-06 | 3114.2ms/step | 19.3 samples/s +Step 47450/50462 | Loss: 2.4959 | LR: 4.52e-06 | 3115.3ms/step | 19.3 samples/s +Step 47460/50462 | Loss: 2.5413 | LR: 4.49e-06 | 3117.5ms/step | 19.2 samples/s +Step 47470/50462 | Loss: 2.5389 | LR: 4.46e-06 | 3117.7ms/step | 19.2 samples/s +Step 47480/50462 | Loss: 2.5370 | LR: 4.43e-06 | 3112.1ms/step | 19.3 samples/s +Step 47490/50462 | Loss: 2.5303 | LR: 4.40e-06 | 3118.9ms/step | 19.2 samples/s +Step 47500/50462 | Loss: 2.5215 | LR: 4.37e-06 | 3118.8ms/step | 19.2 samples/s +Validation at step 47500: Loss = 2.5375, Perplexity = 12.65 +Step 47510/50462 | Loss: 2.4913 | LR: 4.34e-06 | 4779.1ms/step | 12.6 samples/s +Step 47520/50462 | Loss: 2.5405 | LR: 4.31e-06 | 3141.0ms/step | 19.1 samples/s +Step 47530/50462 | Loss: 2.5355 | LR: 4.28e-06 | 3136.6ms/step | 19.1 samples/s +Step 47540/50462 | Loss: 2.5041 | LR: 4.25e-06 | 3122.5ms/step | 19.2 samples/s +Step 47550/50462 | Loss: 2.5336 | LR: 4.22e-06 | 3119.9ms/step | 19.2 samples/s +Step 47560/50462 | Loss: 2.5182 | LR: 4.19e-06 | 3123.2ms/step | 19.2 samples/s +Step 47570/50462 | Loss: 2.5295 | LR: 4.16e-06 | 3122.1ms/step | 19.2 samples/s +Step 47580/50462 | Loss: 2.5645 | LR: 4.13e-06 | 3628.3ms/step | 16.5 samples/s +Step 47590/50462 | Loss: 2.5349 | LR: 4.11e-06 | 3102.6ms/step | 19.3 samples/s +Step 47600/50462 | Loss: 2.5255 | LR: 4.08e-06 | 3109.4ms/step | 19.3 samples/s +Step 47610/50462 | Loss: 2.5052 | LR: 4.05e-06 | 3114.5ms/step | 19.3 samples/s +Step 47620/50462 | Loss: 2.5189 | LR: 4.02e-06 | 3117.2ms/step | 19.2 samples/s +Step 47630/50462 | Loss: 2.5468 | LR: 3.99e-06 | 3119.2ms/step | 19.2 samples/s +Step 47640/50462 | Loss: 2.5018 | LR: 3.97e-06 | 3120.8ms/step | 19.2 samples/s +Step 47650/50462 | Loss: 2.5208 | LR: 3.94e-06 | 3117.4ms/step | 19.2 samples/s +Step 47660/50462 | Loss: 2.5200 | LR: 3.91e-06 | 3115.4ms/step | 19.3 samples/s +Step 47670/50462 | Loss: 2.5130 | LR: 3.88e-06 | 3117.2ms/step | 19.2 samples/s +Step 47680/50462 | Loss: 2.5308 | LR: 3.85e-06 | 3115.9ms/step | 19.3 samples/s +Step 47690/50462 | Loss: 2.5262 | LR: 3.83e-06 | 3116.7ms/step | 19.3 samples/s +Step 47700/50462 | Loss: 2.5341 | LR: 3.80e-06 | 3116.6ms/step | 19.3 samples/s +Step 47710/50462 | Loss: 2.5004 | LR: 3.77e-06 | 3114.0ms/step | 19.3 samples/s +Step 47720/50462 | Loss: 2.5274 | LR: 3.74e-06 | 3114.7ms/step | 19.3 samples/s +Step 47730/50462 | Loss: 2.5275 | LR: 3.72e-06 | 3114.6ms/step | 19.3 samples/s +Step 47740/50462 | Loss: 2.5124 | LR: 3.69e-06 | 3113.2ms/step | 19.3 samples/s +Step 47750/50462 | Loss: 2.5402 | LR: 3.66e-06 | 3117.2ms/step | 19.2 samples/s +Validation at step 47750: Loss = 2.5378, Perplexity = 12.65 +Step 47760/50462 | Loss: 2.4968 | LR: 3.64e-06 | 4779.6ms/step | 12.6 samples/s +Step 47770/50462 | Loss: 2.5595 | LR: 3.61e-06 | 3117.9ms/step | 19.2 samples/s +Step 47780/50462 | Loss: 2.5178 | LR: 3.58e-06 | 3118.1ms/step | 19.2 samples/s +Step 47790/50462 | Loss: 2.5116 | LR: 3.56e-06 | 3120.3ms/step | 19.2 samples/s +Step 47800/50462 | Loss: 2.5035 | LR: 3.53e-06 | 3120.2ms/step | 19.2 samples/s +Step 47810/50462 | Loss: 2.5362 | LR: 3.50e-06 | 3112.8ms/step | 19.3 samples/s +Step 47820/50462 | Loss: 2.5305 | LR: 3.48e-06 | 3115.9ms/step | 19.3 samples/s +Step 47830/50462 | Loss: 2.5226 | LR: 3.45e-06 | 3114.8ms/step | 19.3 samples/s +Step 47840/50462 | Loss: 2.5446 | LR: 3.42e-06 | 3113.0ms/step | 19.3 samples/s +Step 47850/50462 | Loss: 2.5359 | LR: 3.40e-06 | 3116.4ms/step | 19.3 samples/s +Step 47860/50462 | Loss: 2.5384 | LR: 3.37e-06 | 3114.8ms/step | 19.3 samples/s +Step 47870/50462 | Loss: 2.5315 | LR: 3.35e-06 | 3114.6ms/step | 19.3 samples/s +Step 47880/50462 | Loss: 2.4867 | LR: 3.32e-06 | 3113.9ms/step | 19.3 samples/s +Step 47890/50462 | Loss: 2.5445 | LR: 3.30e-06 | 3113.1ms/step | 19.3 samples/s +Step 47900/50462 | Loss: 2.5335 | LR: 3.27e-06 | 3115.4ms/step | 19.3 samples/s +Step 47910/50462 | Loss: 2.5322 | LR: 3.24e-06 | 3116.4ms/step | 19.3 samples/s +Step 47920/50462 | Loss: 2.5060 | LR: 3.22e-06 | 3111.9ms/step | 19.3 samples/s +Step 47930/50462 | Loss: 2.5436 | LR: 3.19e-06 | 3107.1ms/step | 19.3 samples/s +Step 47940/50462 | Loss: 2.5202 | LR: 3.17e-06 | 3109.9ms/step | 19.3 samples/s +Step 47950/50462 | Loss: 2.5268 | LR: 3.14e-06 | 3113.6ms/step | 19.3 samples/s +Step 47960/50462 | Loss: 2.4938 | LR: 3.12e-06 | 3114.5ms/step | 19.3 samples/s +Step 47970/50462 | Loss: 2.5299 | LR: 3.09e-06 | 3116.8ms/step | 19.3 samples/s +Step 47980/50462 | Loss: 2.5268 | LR: 3.07e-06 | 3114.7ms/step | 19.3 samples/s +Step 47990/50462 | Loss: 2.5099 | LR: 3.04e-06 | 3118.7ms/step | 19.2 samples/s +Step 48000/50462 | Loss: 2.5199 | LR: 3.02e-06 | 3111.9ms/step | 19.3 samples/s +Validation at step 48000: Loss = 2.5376, Perplexity = 12.65 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-48000 +Step 48010/50462 | Loss: 2.5329 | LR: 3.00e-06 | 5645.9ms/step | 10.6 samples/s +Step 48020/50462 | Loss: 2.5612 | LR: 2.97e-06 | 3100.9ms/step | 19.3 samples/s +Step 48030/50462 | Loss: 2.5032 | LR: 2.95e-06 | 3107.6ms/step | 19.3 samples/s +Step 48040/50462 | Loss: 2.5162 | LR: 2.92e-06 | 3112.7ms/step | 19.3 samples/s +Step 48050/50462 | Loss: 2.5104 | LR: 2.90e-06 | 3114.5ms/step | 19.3 samples/s +Step 48060/50462 | Loss: 2.5265 | LR: 2.87e-06 | 3112.6ms/step | 19.3 samples/s +Step 48070/50462 | Loss: 2.5293 | LR: 2.85e-06 | 3111.0ms/step | 19.3 samples/s +Step 48080/50462 | Loss: 2.5524 | LR: 2.83e-06 | 3111.2ms/step | 19.3 samples/s +Step 48090/50462 | Loss: 2.5290 | LR: 2.80e-06 | 3114.5ms/step | 19.3 samples/s +Step 48100/50462 | Loss: 2.5321 | LR: 2.78e-06 | 3116.3ms/step | 19.3 samples/s +Step 48110/50462 | Loss: 2.5220 | LR: 2.76e-06 | 3115.8ms/step | 19.3 samples/s +Step 48120/50462 | Loss: 2.5203 | LR: 2.73e-06 | 3113.6ms/step | 19.3 samples/s +Step 48130/50462 | Loss: 2.5201 | LR: 2.71e-06 | 3115.4ms/step | 19.3 samples/s +Step 48140/50462 | Loss: 2.5025 | LR: 2.69e-06 | 3113.7ms/step | 19.3 samples/s +Step 48150/50462 | Loss: 2.5159 | LR: 2.66e-06 | 3115.0ms/step | 19.3 samples/s +Step 48160/50462 | Loss: 2.5364 | LR: 2.64e-06 | 3117.3ms/step | 19.2 samples/s +Step 48170/50462 | Loss: 2.5154 | LR: 2.62e-06 | 3110.8ms/step | 19.3 samples/s +Step 48180/50462 | Loss: 2.5289 | LR: 2.60e-06 | 3117.3ms/step | 19.2 samples/s +Step 48190/50462 | Loss: 2.5236 | LR: 2.57e-06 | 3116.1ms/step | 19.3 samples/s +Step 48200/50462 | Loss: 2.5336 | LR: 2.55e-06 | 3114.3ms/step | 19.3 samples/s +Step 48210/50462 | Loss: 2.5094 | LR: 2.53e-06 | 3115.3ms/step | 19.3 samples/s +Step 48220/50462 | Loss: 2.5162 | LR: 2.51e-06 | 3114.0ms/step | 19.3 samples/s +Step 48230/50462 | Loss: 2.5231 | LR: 2.48e-06 | 3112.7ms/step | 19.3 samples/s +Step 48240/50462 | Loss: 2.5211 | LR: 2.46e-06 | 3115.8ms/step | 19.3 samples/s +Step 48250/50462 | Loss: 2.5590 | LR: 2.44e-06 | 3114.2ms/step | 19.3 samples/s +Validation at step 48250: Loss = 2.5379, Perplexity = 12.65 +Step 48260/50462 | Loss: 2.4973 | LR: 2.42e-06 | 4773.6ms/step | 12.6 samples/s +Step 48270/50462 | Loss: 2.5233 | LR: 2.39e-06 | 3114.7ms/step | 19.3 samples/s +Step 48280/50462 | Loss: 2.5317 | LR: 2.37e-06 | 3112.4ms/step | 19.3 samples/s +Step 48290/50462 | Loss: 2.5189 | LR: 2.35e-06 | 3116.3ms/step | 19.3 samples/s +Step 48300/50462 | Loss: 2.5116 | LR: 2.33e-06 | 3114.8ms/step | 19.3 samples/s +Step 48310/50462 | Loss: 2.5322 | LR: 2.31e-06 | 3112.1ms/step | 19.3 samples/s +Step 48320/50462 | Loss: 2.5306 | LR: 2.29e-06 | 3116.9ms/step | 19.2 samples/s +Step 48330/50462 | Loss: 2.5139 | LR: 2.27e-06 | 3116.2ms/step | 19.3 samples/s +Step 48340/50462 | Loss: 2.5374 | LR: 2.24e-06 | 3113.3ms/step | 19.3 samples/s +Step 48350/50462 | Loss: 2.5517 | LR: 2.22e-06 | 3120.5ms/step | 19.2 samples/s +Step 48360/50462 | Loss: 2.5077 | LR: 2.20e-06 | 3113.8ms/step | 19.3 samples/s +Step 48370/50462 | Loss: 2.5447 | LR: 2.18e-06 | 3113.9ms/step | 19.3 samples/s +Step 48380/50462 | Loss: 2.5114 | LR: 2.16e-06 | 3112.7ms/step | 19.3 samples/s +Step 48390/50462 | Loss: 2.5590 | LR: 2.14e-06 | 3115.6ms/step | 19.3 samples/s +Step 48400/50462 | Loss: 2.5134 | LR: 2.12e-06 | 3114.8ms/step | 19.3 samples/s +Step 48410/50462 | Loss: 2.5422 | LR: 2.10e-06 | 3115.9ms/step | 19.3 samples/s +Step 48420/50462 | Loss: 2.5191 | LR: 2.08e-06 | 3114.1ms/step | 19.3 samples/s +Step 48430/50462 | Loss: 2.5150 | LR: 2.06e-06 | 3113.2ms/step | 19.3 samples/s +Step 48440/50462 | Loss: 2.5321 | LR: 2.04e-06 | 3115.1ms/step | 19.3 samples/s +Step 48450/50462 | Loss: 2.4984 | LR: 2.02e-06 | 3112.9ms/step | 19.3 samples/s +Step 48460/50462 | Loss: 2.5459 | LR: 2.00e-06 | 3110.9ms/step | 19.3 samples/s +Step 48470/50462 | Loss: 2.5337 | LR: 1.98e-06 | 3113.3ms/step | 19.3 samples/s +Step 48480/50462 | Loss: 2.5177 | LR: 1.96e-06 | 3113.1ms/step | 19.3 samples/s +Step 48490/50462 | Loss: 2.5248 | LR: 1.94e-06 | 3111.7ms/step | 19.3 samples/s +Step 48500/50462 | Loss: 2.4932 | LR: 1.92e-06 | 3619.7ms/step | 16.6 samples/s +Validation at step 48500: Loss = 2.5376, Perplexity = 12.65 +Step 48510/50462 | Loss: 2.5279 | LR: 1.90e-06 | 4751.8ms/step | 12.6 samples/s +Step 48520/50462 | Loss: 2.5264 | LR: 1.88e-06 | 3109.3ms/step | 19.3 samples/s +Step 48530/50462 | Loss: 2.5439 | LR: 1.86e-06 | 3113.1ms/step | 19.3 samples/s +Step 48540/50462 | Loss: 2.5004 | LR: 1.84e-06 | 3108.7ms/step | 19.3 samples/s +Step 48550/50462 | Loss: 2.5103 | LR: 1.82e-06 | 3111.3ms/step | 19.3 samples/s +Step 48560/50462 | Loss: 2.5207 | LR: 1.80e-06 | 3643.8ms/step | 16.5 samples/s +Step 48570/50462 | Loss: 2.5225 | LR: 1.78e-06 | 3101.8ms/step | 19.3 samples/s +Step 48580/50462 | Loss: 2.5150 | LR: 1.77e-06 | 3110.2ms/step | 19.3 samples/s +Step 48590/50462 | Loss: 2.5397 | LR: 1.75e-06 | 3113.3ms/step | 19.3 samples/s +Step 48600/50462 | Loss: 2.5311 | LR: 1.73e-06 | 3110.7ms/step | 19.3 samples/s +Step 48610/50462 | Loss: 2.5164 | LR: 1.71e-06 | 3113.0ms/step | 19.3 samples/s +Step 48620/50462 | Loss: 2.5219 | LR: 1.69e-06 | 3111.1ms/step | 19.3 samples/s +Step 48630/50462 | Loss: 2.5444 | LR: 1.67e-06 | 3112.7ms/step | 19.3 samples/s +Step 48640/50462 | Loss: 2.5260 | LR: 1.66e-06 | 3114.6ms/step | 19.3 samples/s +Step 48650/50462 | Loss: 2.5436 | LR: 1.64e-06 | 3112.8ms/step | 19.3 samples/s +Step 48660/50462 | Loss: 2.5393 | LR: 1.62e-06 | 3116.2ms/step | 19.3 samples/s +Step 48670/50462 | Loss: 2.5399 | LR: 1.60e-06 | 3116.8ms/step | 19.3 samples/s +Step 48680/50462 | Loss: 2.5216 | LR: 1.58e-06 | 3116.2ms/step | 19.3 samples/s +Step 48690/50462 | Loss: 2.5357 | LR: 1.57e-06 | 3116.8ms/step | 19.3 samples/s +Step 48700/50462 | Loss: 2.4959 | LR: 1.55e-06 | 3115.6ms/step | 19.3 samples/s +Step 48710/50462 | Loss: 2.5187 | LR: 1.53e-06 | 3116.4ms/step | 19.3 samples/s +Step 48720/50462 | Loss: 2.5401 | LR: 1.51e-06 | 3117.0ms/step | 19.2 samples/s +Step 48730/50462 | Loss: 2.5259 | LR: 1.50e-06 | 3116.5ms/step | 19.3 samples/s +Step 48740/50462 | Loss: 2.5033 | LR: 1.48e-06 | 3115.6ms/step | 19.3 samples/s +Step 48750/50462 | Loss: 2.5466 | LR: 1.46e-06 | 3116.5ms/step | 19.3 samples/s +Validation at step 48750: Loss = 2.5377, Perplexity = 12.65 +Step 48760/50462 | Loss: 2.5575 | LR: 1.44e-06 | 4780.9ms/step | 12.5 samples/s +Step 48770/50462 | Loss: 2.4974 | LR: 1.43e-06 | 3116.5ms/step | 19.3 samples/s +Step 48780/50462 | Loss: 2.5200 | LR: 1.41e-06 | 3115.2ms/step | 19.3 samples/s +Step 48790/50462 | Loss: 2.5289 | LR: 1.39e-06 | 3115.6ms/step | 19.3 samples/s +Step 48800/50462 | Loss: 2.5081 | LR: 1.38e-06 | 3114.7ms/step | 19.3 samples/s +Step 48810/50462 | Loss: 2.5172 | LR: 1.36e-06 | 3111.0ms/step | 19.3 samples/s +Step 48820/50462 | Loss: 2.5224 | LR: 1.34e-06 | 3114.0ms/step | 19.3 samples/s +Step 48830/50462 | Loss: 2.5380 | LR: 1.33e-06 | 3116.4ms/step | 19.3 samples/s +Step 48840/50462 | Loss: 2.5519 | LR: 1.31e-06 | 3115.6ms/step | 19.3 samples/s +Step 48850/50462 | Loss: 2.5236 | LR: 1.30e-06 | 3115.4ms/step | 19.3 samples/s +Step 48860/50462 | Loss: 2.4898 | LR: 1.28e-06 | 3115.2ms/step | 19.3 samples/s +Step 48870/50462 | Loss: 2.5323 | LR: 1.26e-06 | 3113.3ms/step | 19.3 samples/s +Step 48880/50462 | Loss: 2.5146 | LR: 1.25e-06 | 3113.4ms/step | 19.3 samples/s +Step 48890/50462 | Loss: 2.5453 | LR: 1.23e-06 | 3117.6ms/step | 19.2 samples/s +Step 48900/50462 | Loss: 2.5235 | LR: 1.22e-06 | 3116.3ms/step | 19.3 samples/s +Step 48910/50462 | Loss: 2.5373 | LR: 1.20e-06 | 3114.7ms/step | 19.3 samples/s +Step 48920/50462 | Loss: 2.5235 | LR: 1.19e-06 | 3112.3ms/step | 19.3 samples/s +Step 48930/50462 | Loss: 2.5205 | LR: 1.17e-06 | 3111.4ms/step | 19.3 samples/s +Step 48940/50462 | Loss: 2.5007 | LR: 1.16e-06 | 3114.7ms/step | 19.3 samples/s +Step 48950/50462 | Loss: 2.5012 | LR: 1.14e-06 | 3117.6ms/step | 19.2 samples/s +Step 48960/50462 | Loss: 2.5331 | LR: 1.13e-06 | 3116.5ms/step | 19.3 samples/s +Step 48970/50462 | Loss: 2.5000 | LR: 1.11e-06 | 3113.2ms/step | 19.3 samples/s +Step 48980/50462 | Loss: 2.5671 | LR: 1.10e-06 | 3110.4ms/step | 19.3 samples/s +Step 48990/50462 | Loss: 2.5517 | LR: 1.08e-06 | 3109.6ms/step | 19.3 samples/s +Step 49000/50462 | Loss: 2.5340 | LR: 1.07e-06 | 3112.6ms/step | 19.3 samples/s +Validation at step 49000: Loss = 2.5377, Perplexity = 12.65 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-49000 +Step 49010/50462 | Loss: 2.5226 | LR: 1.05e-06 | 5697.3ms/step | 10.5 samples/s +Step 49020/50462 | Loss: 2.5301 | LR: 1.04e-06 | 3110.7ms/step | 19.3 samples/s +Step 49030/50462 | Loss: 2.5306 | LR: 1.02e-06 | 3113.8ms/step | 19.3 samples/s +Step 49040/50462 | Loss: 2.5279 | LR: 1.01e-06 | 3110.7ms/step | 19.3 samples/s +Step 49050/50462 | Loss: 2.5283 | LR: 9.95e-07 | 3111.8ms/step | 19.3 samples/s +Step 49060/50462 | Loss: 2.5483 | LR: 9.81e-07 | 3116.8ms/step | 19.3 samples/s +Step 49070/50462 | Loss: 2.5451 | LR: 9.67e-07 | 3113.8ms/step | 19.3 samples/s +Step 49080/50462 | Loss: 2.5475 | LR: 9.53e-07 | 3112.7ms/step | 19.3 samples/s +Step 49090/50462 | Loss: 2.5012 | LR: 9.39e-07 | 3115.2ms/step | 19.3 samples/s +Step 49100/50462 | Loss: 2.5499 | LR: 9.25e-07 | 3111.9ms/step | 19.3 samples/s +Step 49110/50462 | Loss: 2.5350 | LR: 9.12e-07 | 3112.7ms/step | 19.3 samples/s +Step 49120/50462 | Loss: 2.5043 | LR: 8.99e-07 | 3116.0ms/step | 19.3 samples/s +Step 49130/50462 | Loss: 2.5345 | LR: 8.85e-07 | 3114.5ms/step | 19.3 samples/s +Step 49140/50462 | Loss: 2.5058 | LR: 8.72e-07 | 3114.6ms/step | 19.3 samples/s +Step 49150/50462 | Loss: 2.5367 | LR: 8.59e-07 | 3115.1ms/step | 19.3 samples/s +Step 49160/50462 | Loss: 2.5272 | LR: 8.46e-07 | 3118.1ms/step | 19.2 samples/s +Step 49170/50462 | Loss: 2.5250 | LR: 8.33e-07 | 3117.1ms/step | 19.2 samples/s +Step 49180/50462 | Loss: 2.5151 | LR: 8.20e-07 | 3116.4ms/step | 19.3 samples/s +Step 49190/50462 | Loss: 2.5038 | LR: 8.07e-07 | 3116.9ms/step | 19.2 samples/s +Step 49200/50462 | Loss: 2.5575 | LR: 7.95e-07 | 3118.6ms/step | 19.2 samples/s +Step 49210/50462 | Loss: 2.5278 | LR: 7.82e-07 | 3115.7ms/step | 19.3 samples/s +Step 49220/50462 | Loss: 2.5376 | LR: 7.70e-07 | 3114.6ms/step | 19.3 samples/s +Step 49230/50462 | Loss: 2.5392 | LR: 7.57e-07 | 3114.4ms/step | 19.3 samples/s +Step 49240/50462 | Loss: 2.5300 | LR: 7.45e-07 | 3114.8ms/step | 19.3 samples/s +Step 49250/50462 | Loss: 2.5240 | LR: 7.33e-07 | 3112.0ms/step | 19.3 samples/s +Validation at step 49250: Loss = 2.5375, Perplexity = 12.65 +Step 49260/50462 | Loss: 2.4974 | LR: 7.21e-07 | 4776.6ms/step | 12.6 samples/s +Step 49270/50462 | Loss: 2.5166 | LR: 7.09e-07 | 3117.3ms/step | 19.2 samples/s +Step 49280/50462 | Loss: 2.4997 | LR: 6.97e-07 | 3117.5ms/step | 19.2 samples/s +Step 49290/50462 | Loss: 2.5211 | LR: 6.85e-07 | 3110.5ms/step | 19.3 samples/s +Step 49300/50462 | Loss: 2.5187 | LR: 6.74e-07 | 3114.9ms/step | 19.3 samples/s +Step 49310/50462 | Loss: 2.5251 | LR: 6.62e-07 | 3114.1ms/step | 19.3 samples/s +Step 49320/50462 | Loss: 2.5130 | LR: 6.51e-07 | 3117.4ms/step | 19.2 samples/s +Step 49330/50462 | Loss: 2.5225 | LR: 6.39e-07 | 3117.7ms/step | 19.2 samples/s +Step 49340/50462 | Loss: 2.5320 | LR: 6.28e-07 | 3114.4ms/step | 19.3 samples/s +Step 49350/50462 | Loss: 2.5348 | LR: 6.17e-07 | 3116.5ms/step | 19.3 samples/s +Step 49360/50462 | Loss: 2.5449 | LR: 6.06e-07 | 3115.2ms/step | 19.3 samples/s +Step 49370/50462 | Loss: 2.5264 | LR: 5.95e-07 | 3118.6ms/step | 19.2 samples/s +Step 49380/50462 | Loss: 2.5402 | LR: 5.84e-07 | 3114.4ms/step | 19.3 samples/s +Step 49390/50462 | Loss: 2.5394 | LR: 5.73e-07 | 3113.0ms/step | 19.3 samples/s +Step 49400/50462 | Loss: 2.5365 | LR: 5.63e-07 | 3114.7ms/step | 19.3 samples/s +Step 49410/50462 | Loss: 2.5299 | LR: 5.52e-07 | 3114.1ms/step | 19.3 samples/s +Step 49420/50462 | Loss: 2.5263 | LR: 5.42e-07 | 3115.6ms/step | 19.3 samples/s +Step 49430/50462 | Loss: 2.5065 | LR: 5.31e-07 | 3120.0ms/step | 19.2 samples/s +Step 49440/50462 | Loss: 2.5153 | LR: 5.21e-07 | 3117.8ms/step | 19.2 samples/s +Step 49450/50462 | Loss: 2.5030 | LR: 5.11e-07 | 3114.8ms/step | 19.3 samples/s +Step 49460/50462 | Loss: 2.5360 | LR: 5.01e-07 | 3116.0ms/step | 19.3 samples/s +Step 49470/50462 | Loss: 2.5374 | LR: 4.91e-07 | 3116.7ms/step | 19.3 samples/s +Step 49480/50462 | Loss: 2.5312 | LR: 4.81e-07 | 3119.2ms/step | 19.2 samples/s +Step 49490/50462 | Loss: 2.5121 | LR: 4.72e-07 | 3115.7ms/step | 19.3 samples/s +Step 49500/50462 | Loss: 2.5324 | LR: 4.62e-07 | 3116.6ms/step | 19.3 samples/s +Validation at step 49500: Loss = 2.5375, Perplexity = 12.65 +Step 49510/50462 | Loss: 2.5413 | LR: 4.52e-07 | 4773.8ms/step | 12.6 samples/s +Step 49520/50462 | Loss: 2.5216 | LR: 4.43e-07 | 3119.5ms/step | 19.2 samples/s +Step 49530/50462 | Loss: 2.5260 | LR: 4.34e-07 | 3120.0ms/step | 19.2 samples/s +Step 49540/50462 | Loss: 2.5171 | LR: 4.24e-07 | 3122.3ms/step | 19.2 samples/s +Step 49550/50462 | Loss: 2.5253 | LR: 4.15e-07 | 3115.9ms/step | 19.3 samples/s +Step 49560/50462 | Loss: 2.5402 | LR: 4.06e-07 | 3112.1ms/step | 19.3 samples/s +Step 49570/50462 | Loss: 2.5420 | LR: 3.97e-07 | 3117.6ms/step | 19.2 samples/s +Step 49580/50462 | Loss: 2.5418 | LR: 3.88e-07 | 3117.1ms/step | 19.2 samples/s +Step 49590/50462 | Loss: 2.5144 | LR: 3.79e-07 | 3116.9ms/step | 19.2 samples/s +Step 49600/50462 | Loss: 2.5464 | LR: 3.71e-07 | 3118.4ms/step | 19.2 samples/s +Step 49610/50462 | Loss: 2.5263 | LR: 3.62e-07 | 3116.7ms/step | 19.3 samples/s +Step 49620/50462 | Loss: 2.5246 | LR: 3.54e-07 | 3111.2ms/step | 19.3 samples/s +Step 49630/50462 | Loss: 2.5255 | LR: 3.45e-07 | 3116.4ms/step | 19.3 samples/s +Step 49640/50462 | Loss: 2.5275 | LR: 3.37e-07 | 3117.0ms/step | 19.2 samples/s +Step 49650/50462 | Loss: 2.5311 | LR: 3.29e-07 | 3114.5ms/step | 19.3 samples/s +Step 49660/50462 | Loss: 2.5137 | LR: 3.21e-07 | 3115.4ms/step | 19.3 samples/s +Step 49670/50462 | Loss: 2.5011 | LR: 3.13e-07 | 3115.9ms/step | 19.3 samples/s +Step 49680/50462 | Loss: 2.5455 | LR: 3.05e-07 | 3116.7ms/step | 19.3 samples/s +Step 49690/50462 | Loss: 2.5627 | LR: 2.97e-07 | 3117.6ms/step | 19.2 samples/s +Step 49700/50462 | Loss: 2.5350 | LR: 2.90e-07 | 3114.3ms/step | 19.3 samples/s +Step 49710/50462 | Loss: 2.5292 | LR: 2.82e-07 | 3613.0ms/step | 16.6 samples/s +Step 49720/50462 | Loss: 2.5194 | LR: 2.75e-07 | 3110.9ms/step | 19.3 samples/s +Step 49730/50462 | Loss: 2.5354 | LR: 2.67e-07 | 3112.4ms/step | 19.3 samples/s +Step 49740/50462 | Loss: 2.5554 | LR: 2.60e-07 | 3114.2ms/step | 19.3 samples/s +Switching attention mode for step 49750: dense (dense step interval) +Step 49750/50462 | Loss: 2.5255 | LR: 2.53e-07 | 3330.6ms/step | 18.0 samples/s +Validation at step 49750: Loss = 2.5199, Perplexity = 12.43 +Step 49760/50462 | Loss: 2.5271 | LR: 2.46e-07 | 7820.1ms/step | 7.7 samples/s +Step 49770/50462 | Loss: 2.5160 | LR: 2.39e-07 | 5253.2ms/step | 11.4 samples/s +Step 49780/50462 | Loss: 2.5007 | LR: 2.32e-07 | 5251.7ms/step | 11.4 samples/s +Step 49790/50462 | Loss: 2.4943 | LR: 2.25e-07 | 5244.7ms/step | 11.4 samples/s +Step 49800/50462 | Loss: 2.5369 | LR: 2.19e-07 | 5248.1ms/step | 11.4 samples/s +Step 49810/50462 | Loss: 2.5233 | LR: 2.12e-07 | 5245.5ms/step | 11.4 samples/s +Step 49820/50462 | Loss: 2.5209 | LR: 2.06e-07 | 5252.6ms/step | 11.4 samples/s +Step 49830/50462 | Loss: 2.5100 | LR: 1.99e-07 | 5247.0ms/step | 11.4 samples/s +Step 49840/50462 | Loss: 2.5245 | LR: 1.93e-07 | 5240.9ms/step | 11.4 samples/s +Step 49850/50462 | Loss: 2.5014 | LR: 1.87e-07 | 5249.8ms/step | 11.4 samples/s +Step 49860/50462 | Loss: 2.5007 | LR: 1.81e-07 | 5249.3ms/step | 11.4 samples/s +Step 49870/50462 | Loss: 2.5157 | LR: 1.75e-07 | 5249.1ms/step | 11.4 samples/s +Step 49880/50462 | Loss: 2.5009 | LR: 1.69e-07 | 5246.5ms/step | 11.4 samples/s +Step 49890/50462 | Loss: 2.5105 | LR: 1.63e-07 | 5251.9ms/step | 11.4 samples/s +Step 49900/50462 | Loss: 2.5227 | LR: 1.58e-07 | 5249.5ms/step | 11.4 samples/s +Step 49910/50462 | Loss: 2.5095 | LR: 1.52e-07 | 5248.1ms/step | 11.4 samples/s +Step 49920/50462 | Loss: 2.5233 | LR: 1.47e-07 | 5249.9ms/step | 11.4 samples/s +Step 49930/50462 | Loss: 2.4946 | LR: 1.41e-07 | 5247.1ms/step | 11.4 samples/s +Step 49940/50462 | Loss: 2.5065 | LR: 1.36e-07 | 5248.3ms/step | 11.4 samples/s +Step 49950/50462 | Loss: 2.4674 | LR: 1.31e-07 | 5250.3ms/step | 11.4 samples/s +Step 49960/50462 | Loss: 2.4964 | LR: 1.26e-07 | 5253.0ms/step | 11.4 samples/s +Step 49970/50462 | Loss: 2.4884 | LR: 1.21e-07 | 5254.1ms/step | 11.4 samples/s +Step 49980/50462 | Loss: 2.5035 | LR: 1.16e-07 | 5252.2ms/step | 11.4 samples/s +Step 49990/50462 | Loss: 2.4812 | LR: 1.11e-07 | 5245.6ms/step | 11.4 samples/s +Step 50000/50462 | Loss: 2.4924 | LR: 1.07e-07 | 5253.5ms/step | 11.4 samples/s +Validation at step 50000: Loss = 2.5199, Perplexity = 12.43 +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-50000 +Switching attention mode for step 50001: sparse_stochastic (configured sparse attention) +Step 50010/50462 | Loss: 2.5052 | LR: 1.02e-07 | 6581.4ms/step | 9.1 samples/s +Step 50020/50462 | Loss: 2.5132 | LR: 9.75e-08 | 3101.5ms/step | 19.3 samples/s +Step 50030/50462 | Loss: 2.5429 | LR: 9.32e-08 | 3109.7ms/step | 19.3 samples/s +Step 50040/50462 | Loss: 2.5730 | LR: 8.89e-08 | 3112.9ms/step | 19.3 samples/s +Step 50050/50462 | Loss: 2.5273 | LR: 8.47e-08 | 3114.6ms/step | 19.3 samples/s +Step 50060/50462 | Loss: 2.5204 | LR: 8.07e-08 | 3114.3ms/step | 19.3 samples/s +Step 50070/50462 | Loss: 2.5408 | LR: 7.67e-08 | 3110.6ms/step | 19.3 samples/s +Step 50080/50462 | Loss: 2.5118 | LR: 7.28e-08 | 3117.6ms/step | 19.2 samples/s +Step 50090/50462 | Loss: 2.5047 | LR: 6.91e-08 | 3117.0ms/step | 19.2 samples/s +Step 50100/50462 | Loss: 2.5111 | LR: 6.54e-08 | 3117.5ms/step | 19.2 samples/s +Step 50110/50462 | Loss: 2.5504 | LR: 6.19e-08 | 3116.7ms/step | 19.3 samples/s +Step 50120/50462 | Loss: 2.5323 | LR: 5.84e-08 | 3116.2ms/step | 19.3 samples/s +Step 50130/50462 | Loss: 2.5312 | LR: 5.50e-08 | 3115.8ms/step | 19.3 samples/s +Step 50140/50462 | Loss: 2.5497 | LR: 5.18e-08 | 3114.3ms/step | 19.3 samples/s +Step 50150/50462 | Loss: 2.5060 | LR: 4.86e-08 | 3114.2ms/step | 19.3 samples/s +Step 50160/50462 | Loss: 2.5210 | LR: 4.55e-08 | 3114.4ms/step | 19.3 samples/s +Step 50170/50462 | Loss: 2.4894 | LR: 4.26e-08 | 3111.3ms/step | 19.3 samples/s +Step 50180/50462 | Loss: 2.4937 | LR: 3.97e-08 | 3114.2ms/step | 19.3 samples/s +Step 50190/50462 | Loss: 2.5415 | LR: 3.69e-08 | 3114.0ms/step | 19.3 samples/s +Step 50200/50462 | Loss: 2.5109 | LR: 3.43e-08 | 3110.1ms/step | 19.3 samples/s +Step 50210/50462 | Loss: 2.5367 | LR: 3.17e-08 | 3112.5ms/step | 19.3 samples/s +Step 50220/50462 | Loss: 2.5311 | LR: 2.92e-08 | 3112.5ms/step | 19.3 samples/s +Step 50230/50462 | Loss: 2.5494 | LR: 2.69e-08 | 3113.3ms/step | 19.3 samples/s +Step 50240/50462 | Loss: 2.5136 | LR: 2.46e-08 | 3114.0ms/step | 19.3 samples/s +Step 50250/50462 | Loss: 2.5069 | LR: 2.24e-08 | 3113.2ms/step | 19.3 samples/s +Validation at step 50250: Loss = 2.5380, Perplexity = 12.65 +Step 50260/50462 | Loss: 2.5545 | LR: 2.04e-08 | 4771.1ms/step | 12.6 samples/s +Step 50270/50462 | Loss: 2.5662 | LR: 1.84e-08 | 3109.5ms/step | 19.3 samples/s +Step 50280/50462 | Loss: 2.5368 | LR: 1.65e-08 | 3111.9ms/step | 19.3 samples/s +Step 50290/50462 | Loss: 2.5513 | LR: 1.48e-08 | 3114.1ms/step | 19.3 samples/s +Step 50300/50462 | Loss: 2.5053 | LR: 1.31e-08 | 3112.5ms/step | 19.3 samples/s +Step 50310/50462 | Loss: 2.5010 | LR: 1.15e-08 | 3112.9ms/step | 19.3 samples/s +Step 50320/50462 | Loss: 2.5220 | LR: 1.01e-08 | 3115.5ms/step | 19.3 samples/s +Step 50330/50462 | Loss: 2.5502 | LR: 8.70e-09 | 3119.2ms/step | 19.2 samples/s +Step 50340/50462 | Loss: 2.5299 | LR: 7.43e-09 | 3114.8ms/step | 19.3 samples/s +Step 50350/50462 | Loss: 2.5110 | LR: 6.26e-09 | 3115.9ms/step | 19.3 samples/s +Step 50360/50462 | Loss: 2.5334 | LR: 5.19e-09 | 3111.5ms/step | 19.3 samples/s +Step 50370/50462 | Loss: 2.5096 | LR: 4.23e-09 | 3112.6ms/step | 19.3 samples/s +Step 50380/50462 | Loss: 2.5032 | LR: 3.36e-09 | 3113.3ms/step | 19.3 samples/s +Step 50390/50462 | Loss: 2.5418 | LR: 2.59e-09 | 3108.9ms/step | 19.3 samples/s +Step 50400/50462 | Loss: 2.5256 | LR: 1.92e-09 | 3106.9ms/step | 19.3 samples/s +Step 50410/50462 | Loss: 2.5153 | LR: 1.35e-09 | 3108.7ms/step | 19.3 samples/s +Step 50420/50462 | Loss: 2.5422 | LR: 8.81e-10 | 3111.2ms/step | 19.3 samples/s +Step 50430/50462 | Loss: 2.5327 | LR: 5.11e-10 | 3110.6ms/step | 19.3 samples/s +Step 50440/50462 | Loss: 2.5027 | LR: 2.42e-10 | 3110.0ms/step | 19.3 samples/s +Step 50450/50462 | Loss: 2.5436 | LR: 7.19e-11 | 3112.9ms/step | 19.3 samples/s +Step 50460/50462 | Loss: 2.5171 | LR: 2.00e-12 | 3111.7ms/step | 19.3 samples/s +Saved checkpoint to checkpoints/run_20260530_194538/checkpoint-50462 +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log; uploading config.yaml +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁█▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁██▁▁▁▁▁█ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▆▁▁▁▁▁▁▁▆▆▁▁▁▇▁▁▁▁▁▁▁▁█▁▁▁▁▁ +wandb: eval/dense_interval_start ▇▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▆▇▇▇▆▅▅▃▅▄▄▃▃▃▂▁▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ██▇▆▇▆▅▄▅▅▄▄▄▄▄▃▃▃▃▃▁▂▂▂▂▂▂▂▂▂▂▂▂▁▂▂▂▂▂▁ +wandb: eval/uses_dense_attention █▁▁▁▁▁▁▁█▁▁▁▁▁▁██▁▁▁▁▁▁▁█▁▁▁▁▁▁█▁▁▁▁▁▁▁█ +wandb: eval/uses_sparse_attention █████████████████████████▁█████████████▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▃▁▁▁▁▁▁▁▁▁▆▁█▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec █▆█████████▁███████████▆███████████▄████ +wandb: speed/steps_per_sec █▁████████████████████▂████████▁████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.53798 +wandb: eval/perplexity 12.6541 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 3111.70385 +wandb: speed/samples_per_sec 19.28204 +wandb: speed/steps_per_sec 0.32137 +wandb: +4 ... +wandb: +wandb: 🚀 View run sparse 1.2b at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ofounlqj +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260530_194904-ofounlqj/logs +Training complete! diff --git a/epfl_final/log_files/eval_sparse_64.log b/epfl_final/log_files/eval_sparse_64.log new file mode 100644 index 0000000000000000000000000000000000000000..259ffa111e6ee9d6160915f5a94d3c9dc7a44800 --- /dev/null +++ b/epfl_final/log_files/eval_sparse_64.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bd5473d0f688f6796cec17c573b34f0c4f7b8905670648a53b9b2cce8042111 +size 1247537736 diff --git a/epfl_final/log_files/pretrain.log b/epfl_final/log_files/pretrain.log new file mode 100644 index 0000000000000000000000000000000000000000..333461008d9d27336addd15ecc84071fd31f0da3 --- /dev/null +++ b/epfl_final/log_files/pretrain.log @@ -0,0 +1,2683 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-27000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense 1.2b", + "wandb_tags": [ + "qwen3", + "1.2b", + "dense" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "dense", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache", + "dense_mode": false +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-27000 +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: dense + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +=========================================================================================================================================================== +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +=========================================================================================================================================================== +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +=========================================================================================================================================================== +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +=========================================================================================================================================================== +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +=========================================================================================================================================================== + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-27000... +Resumed from checkpoint at step 27000 (epoch 0, micro_step 270000) +Resuming training from step 27000 +Resuming wandb run xj5iv7ks +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260525_182157-xj5iv7ks +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run dense 1.2b +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/xj5iv7ks +Starting training... +Validation at step 27000: Loss = 2.5575, Perplexity = 12.90 +wandb: WARNING Tried to log to step 27000 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27010/50462 | Loss: 2.5257 | LR: 2.28e-04 | 55911.3ms/step | 1.1 samples/s +wandb: WARNING Tried to log to step 27010 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27020/50462 | Loss: 2.5607 | LR: 2.28e-04 | 5150.1ms/step | 11.7 samples/s +wandb: WARNING Tried to log to step 27020 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27030/50462 | Loss: 2.5710 | LR: 2.28e-04 | 5173.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 27030 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27040/50462 | Loss: 2.5586 | LR: 2.27e-04 | 5166.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 27040 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27050/50462 | Loss: 2.5508 | LR: 2.27e-04 | 5146.8ms/step | 11.7 samples/s +wandb: WARNING Tried to log to step 27050 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27060/50462 | Loss: 2.5433 | LR: 2.27e-04 | 5172.2ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 27060 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27070/50462 | Loss: 2.5446 | LR: 2.27e-04 | 5208.2ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 27070 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27080/50462 | Loss: 2.5524 | LR: 2.27e-04 | 5192.4ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 27080 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27090/50462 | Loss: 2.5346 | LR: 2.27e-04 | 5313.1ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 27090 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27100/50462 | Loss: 2.5524 | LR: 2.26e-04 | 5157.5ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 27100 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27110/50462 | Loss: 2.5442 | LR: 2.26e-04 | 5160.9ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 27110 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27120/50462 | Loss: 2.5534 | LR: 2.26e-04 | 5192.7ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 27120 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27130/50462 | Loss: 2.5677 | LR: 2.26e-04 | 5203.4ms/step | 11.5 samples/s +wandb: WARNING Tried to log to step 27130 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27140/50462 | Loss: 2.5290 | LR: 2.26e-04 | 5193.8ms/step | 11.6 samples/s +wandb: WARNING Tried to log to step 27140 that is less than the current step 27141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 27150/50462 | Loss: 2.5350 | LR: 2.26e-04 | 5160.1ms/step | 11.6 samples/s +Step 27160/50462 | Loss: 2.5428 | LR: 2.25e-04 | 5184.5ms/step | 11.6 samples/s +Step 27170/50462 | Loss: 2.5740 | LR: 2.25e-04 | 5185.0ms/step | 11.6 samples/s +Step 27180/50462 | Loss: 2.5569 | LR: 2.25e-04 | 5187.2ms/step | 11.6 samples/s +Step 27190/50462 | Loss: 2.5625 | LR: 2.25e-04 | 5194.1ms/step | 11.6 samples/s +Step 27200/50462 | Loss: 2.5079 | LR: 2.25e-04 | 5155.5ms/step | 11.6 samples/s +Step 27210/50462 | Loss: 2.5471 | LR: 2.25e-04 | 5172.0ms/step | 11.6 samples/s +Step 27220/50462 | Loss: 2.5340 | LR: 2.25e-04 | 5194.6ms/step | 11.6 samples/s +Step 27230/50462 | Loss: 2.5773 | LR: 2.24e-04 | 5197.5ms/step | 11.5 samples/s +Step 27240/50462 | Loss: 2.5489 | LR: 2.24e-04 | 5161.9ms/step | 11.6 samples/s +Step 27250/50462 | Loss: 2.5631 | LR: 2.24e-04 | 5192.1ms/step | 11.6 samples/s +Validation at step 27250: Loss = 2.5556, Perplexity = 12.88 +Step 27260/50462 | Loss: 2.5505 | LR: 2.24e-04 | 7653.2ms/step | 7.8 samples/s +Step 27270/50462 | Loss: 2.5550 | LR: 2.24e-04 | 5124.4ms/step | 11.7 samples/s +Step 27280/50462 | Loss: 2.5277 | LR: 2.24e-04 | 5132.7ms/step | 11.7 samples/s +Step 27290/50462 | Loss: 2.5485 | LR: 2.23e-04 | 5124.5ms/step | 11.7 samples/s +Step 27300/50462 | Loss: 2.5095 | LR: 2.23e-04 | 5134.0ms/step | 11.7 samples/s +Step 27310/50462 | Loss: 2.5036 | LR: 2.23e-04 | 5110.2ms/step | 11.7 samples/s +Step 27320/50462 | Loss: 2.5464 | LR: 2.23e-04 | 5125.5ms/step | 11.7 samples/s +Step 27330/50462 | Loss: 2.5742 | LR: 2.23e-04 | 5146.4ms/step | 11.7 samples/s +Step 27340/50462 | Loss: 2.5359 | LR: 2.23e-04 | 5143.8ms/step | 11.7 samples/s +Step 27350/50462 | Loss: 2.5249 | LR: 2.23e-04 | 5133.0ms/step | 11.7 samples/s +Step 27360/50462 | Loss: 2.4962 | LR: 2.22e-04 | 5155.7ms/step | 11.6 samples/s +Step 27370/50462 | Loss: 2.5340 | LR: 2.22e-04 | 5145.4ms/step | 11.7 samples/s +Step 27380/50462 | Loss: 2.5267 | LR: 2.22e-04 | 5227.1ms/step | 11.5 samples/s +Step 27390/50462 | Loss: 2.5431 | LR: 2.22e-04 | 5150.0ms/step | 11.7 samples/s +Step 27400/50462 | Loss: 2.5583 | LR: 2.22e-04 | 5120.1ms/step | 11.7 samples/s +Step 27410/50462 | Loss: 2.5240 | LR: 2.22e-04 | 5191.6ms/step | 11.6 samples/s +Step 27420/50462 | Loss: 2.5264 | LR: 2.21e-04 | 5184.3ms/step | 11.6 samples/s +Step 27430/50462 | Loss: 2.5493 | LR: 2.21e-04 | 5188.7ms/step | 11.6 samples/s +Step 27440/50462 | Loss: 2.5339 | LR: 2.21e-04 | 5184.9ms/step | 11.6 samples/s +Step 27450/50462 | Loss: 2.5310 | LR: 2.21e-04 | 5184.9ms/step | 11.6 samples/s +Step 27460/50462 | Loss: 2.5676 | LR: 2.21e-04 | 5159.8ms/step | 11.6 samples/s +Step 27470/50462 | Loss: 2.5208 | LR: 2.21e-04 | 5182.2ms/step | 11.6 samples/s +Step 27480/50462 | Loss: 2.5388 | LR: 2.20e-04 | 5184.3ms/step | 11.6 samples/s +Step 27490/50462 | Loss: 2.5109 | LR: 2.20e-04 | 5183.0ms/step | 11.6 samples/s +Step 27500/50462 | Loss: 2.5480 | LR: 2.20e-04 | 5181.9ms/step | 11.6 samples/s +Validation at step 27500: Loss = 2.5541, Perplexity = 12.86 +Step 27510/50462 | Loss: 2.5401 | LR: 2.20e-04 | 7733.1ms/step | 7.8 samples/s +Step 27520/50462 | Loss: 2.5443 | LR: 2.20e-04 | 5126.9ms/step | 11.7 samples/s +Step 27530/50462 | Loss: 2.5662 | LR: 2.20e-04 | 5177.4ms/step | 11.6 samples/s +Step 27540/50462 | Loss: 2.5610 | LR: 2.20e-04 | 5119.7ms/step | 11.7 samples/s +Step 27550/50462 | Loss: 2.5388 | LR: 2.19e-04 | 5177.2ms/step | 11.6 samples/s +Step 27560/50462 | Loss: 2.5487 | LR: 2.19e-04 | 5158.9ms/step | 11.6 samples/s +Step 27570/50462 | Loss: 2.5420 | LR: 2.19e-04 | 5171.9ms/step | 11.6 samples/s +Step 27580/50462 | Loss: 2.5262 | LR: 2.19e-04 | 5140.2ms/step | 11.7 samples/s +Step 27590/50462 | Loss: 2.5495 | LR: 2.19e-04 | 5157.7ms/step | 11.6 samples/s +Step 27600/50462 | Loss: 2.5473 | LR: 2.19e-04 | 5126.4ms/step | 11.7 samples/s +Step 27610/50462 | Loss: 2.5321 | LR: 2.18e-04 | 5142.3ms/step | 11.7 samples/s +Step 27620/50462 | Loss: 2.5541 | LR: 2.18e-04 | 5162.8ms/step | 11.6 samples/s +Step 27630/50462 | Loss: 2.5314 | LR: 2.18e-04 | 5129.2ms/step | 11.7 samples/s +Step 27640/50462 | Loss: 2.5335 | LR: 2.18e-04 | 5147.1ms/step | 11.7 samples/s +Step 27650/50462 | Loss: 2.5214 | LR: 2.18e-04 | 5123.9ms/step | 11.7 samples/s +Step 27660/50462 | Loss: 2.5353 | LR: 2.18e-04 | 5159.0ms/step | 11.6 samples/s +Step 27670/50462 | Loss: 2.5412 | LR: 2.17e-04 | 5230.3ms/step | 11.5 samples/s +Step 27680/50462 | Loss: 2.5343 | LR: 2.17e-04 | 5169.5ms/step | 11.6 samples/s +Step 27690/50462 | Loss: 2.5243 | LR: 2.17e-04 | 5229.9ms/step | 11.5 samples/s +Step 27700/50462 | Loss: 2.5379 | LR: 2.17e-04 | 5209.2ms/step | 11.5 samples/s +Step 27710/50462 | Loss: 2.5373 | LR: 2.17e-04 | 5219.1ms/step | 11.5 samples/s +Step 27720/50462 | Loss: 2.5533 | LR: 2.17e-04 | 5140.2ms/step | 11.7 samples/s +Step 27730/50462 | Loss: 2.5537 | LR: 2.17e-04 | 5169.7ms/step | 11.6 samples/s +Step 27740/50462 | Loss: 2.5393 | LR: 2.16e-04 | 5114.1ms/step | 11.7 samples/s +Step 27750/50462 | Loss: 2.5467 | LR: 2.16e-04 | 5142.3ms/step | 11.7 samples/s +Validation at step 27750: Loss = 2.5519, Perplexity = 12.83 +Step 27760/50462 | Loss: 2.5333 | LR: 2.16e-04 | 7706.1ms/step | 7.8 samples/sStep 27770/50462 | Loss: 2.5192 | LR: 2.16e-04 | 5150.0ms/step | 11.7 samples/s +Step 27780/50462 | Loss: 2.5563 | LR: 2.16e-04 | 5155.4ms/step | 11.6 samples/s +Step 27790/50462 | Loss: 2.5081 | LR: 2.16e-04 | 5132.2ms/step | 11.7 samples/s +Step 27800/50462 | Loss: 2.5414 | LR: 2.15e-04 | 5132.5ms/step | 11.7 samples/s +Step 27810/50462 | Loss: 2.5474 | LR: 2.15e-04 | 5152.0ms/step | 11.6 samples/s +Step 27820/50462 | Loss: 2.5333 | LR: 2.15e-04 | 5161.1ms/step | 11.6 samples/s +Step 27830/50462 | Loss: 2.5467 | LR: 2.15e-04 | 5127.0ms/step | 11.7 samples/s +Step 27840/50462 | Loss: 2.5283 | LR: 2.15e-04 | 5135.6ms/step | 11.7 samples/s +Step 27850/50462 | Loss: 2.5328 | LR: 2.15e-04 | 5165.9ms/step | 11.6 samples/s +Step 27860/50462 | Loss: 2.5362 | LR: 2.15e-04 | 5172.5ms/step | 11.6 samples/s +Step 27870/50462 | Loss: 2.4938 | LR: 2.14e-04 | 5170.0ms/step | 11.6 samples/s +Step 27880/50462 | Loss: 2.5526 | LR: 2.14e-04 | 5173.5ms/step | 11.6 samples/s +Step 27890/50462 | Loss: 2.5218 | LR: 2.14e-04 | 5172.4ms/step | 11.6 samples/s +Step 27900/50462 | Loss: 2.5204 | LR: 2.14e-04 | 5178.7ms/step | 11.6 samples/s +Step 27910/50462 | Loss: 2.5919 | LR: 2.14e-04 | 5152.7ms/step | 11.6 samples/s +Step 27920/50462 | Loss: 2.5608 | LR: 2.14e-04 | 5166.8ms/step | 11.6 samples/s +Step 27930/50462 | Loss: 2.5295 | LR: 2.13e-04 | 5180.2ms/step | 11.6 samples/s +Step 27940/50462 | Loss: 2.5182 | LR: 2.13e-04 | 5170.0ms/step | 11.6 samples/s +Step 27950/50462 | Loss: 2.5641 | LR: 2.13e-04 | 5172.7ms/step | 11.6 samples/s +Step 27960/50462 | Loss: 2.5439 | LR: 2.13e-04 | 5249.9ms/step | 11.4 samples/s +Step 27970/50462 | Loss: 2.5477 | LR: 2.13e-04 | 5222.0ms/step | 11.5 samples/s +Step 27980/50462 | Loss: 2.5411 | LR: 2.13e-04 | 5175.8ms/step | 11.6 samples/s +Step 27990/50462 | Loss: 2.5476 | LR: 2.12e-04 | 5157.1ms/step | 11.6 samples/s +Step 28000/50462 | Loss: 2.5597 | LR: 2.12e-04 | 5174.0ms/step | 11.6 samples/s +Validation at step 28000: Loss = 2.5503, Perplexity = 12.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-28000 +Step 28010/50462 | Loss: 2.5517 | LR: 2.12e-04 | 8700.4ms/step | 6.9 samples/s +Step 28020/50462 | Loss: 2.5599 | LR: 2.12e-04 | 5142.8ms/step | 11.7 samples/s +Step 28030/50462 | Loss: 2.5624 | LR: 2.12e-04 | 5166.2ms/step | 11.6 samples/s +Step 28040/50462 | Loss: 2.5595 | LR: 2.12e-04 | 5164.0ms/step | 11.6 samples/s +Step 28050/50462 | Loss: 2.5215 | LR: 2.12e-04 | 5154.1ms/step | 11.6 samples/s +Step 28060/50462 | Loss: 2.5645 | LR: 2.11e-04 | 5148.6ms/step | 11.7 samples/s +Step 28070/50462 | Loss: 2.5395 | LR: 2.11e-04 | 5150.2ms/step | 11.6 samples/s +Step 28080/50462 | Loss: 2.5298 | LR: 2.11e-04 | 5180.7ms/step | 11.6 samples/s +Step 28090/50462 | Loss: 2.5441 | LR: 2.11e-04 | 5161.6ms/step | 11.6 samples/s +Step 28100/50462 | Loss: 2.5062 | LR: 2.11e-04 | 5136.8ms/step | 11.7 samples/s +Step 28110/50462 | Loss: 2.5427 | LR: 2.11e-04 | 5178.9ms/step | 11.6 samples/s +Step 28120/50462 | Loss: 2.5290 | LR: 2.10e-04 | 5134.0ms/step | 11.7 samples/s +Step 28130/50462 | Loss: 2.5604 | LR: 2.10e-04 | 5176.0ms/step | 11.6 samples/s +Step 28140/50462 | Loss: 2.5302 | LR: 2.10e-04 | 5175.1ms/step | 11.6 samples/s +Step 28150/50462 | Loss: 2.5515 | LR: 2.10e-04 | 5177.0ms/step | 11.6 samples/s +Step 28160/50462 | Loss: 2.5404 | LR: 2.10e-04 | 5138.2ms/step | 11.7 samples/s +Step 28170/50462 | Loss: 2.5182 | LR: 2.10e-04 | 5125.3ms/step | 11.7 samples/s +Step 28180/50462 | Loss: 2.5530 | LR: 2.10e-04 | 5137.8ms/step | 11.7 samples/s +Step 28190/50462 | Loss: 2.5404 | LR: 2.09e-04 | 5132.6ms/step | 11.7 samples/s +Step 28200/50462 | Loss: 2.5464 | LR: 2.09e-04 | 5150.6ms/step | 11.6 samples/s +Step 28210/50462 | Loss: 2.5341 | LR: 2.09e-04 | 5173.3ms/step | 11.6 samples/s +Step 28220/50462 | Loss: 2.5536 | LR: 2.09e-04 | 5151.7ms/step | 11.6 samples/s +Step 28230/50462 | Loss: 2.5238 | LR: 2.09e-04 | 5142.9ms/step | 11.7 samples/s +Step 28240/50462 | Loss: 2.5277 | LR: 2.09e-04 | 5199.3ms/step | 11.5 samples/s +Step 28250/50462 | Loss: 2.5464 | LR: 2.08e-04 | 5269.9ms/step | 11.4 samples/s +Validation at step 28250: Loss = 2.5491, Perplexity = 12.80 +Step 28260/50462 | Loss: 2.5211 | LR: 2.08e-04 | 7730.4ms/step | 7.8 samples/s +Step 28270/50462 | Loss: 2.5622 | LR: 2.08e-04 | 5181.2ms/step | 11.6 samples/s +Step 28280/50462 | Loss: 2.5128 | LR: 2.08e-04 | 5120.5ms/step | 11.7 samples/s +Step 28290/50462 | Loss: 2.5329 | LR: 2.08e-04 | 5151.9ms/step | 11.6 samples/s +Step 28300/50462 | Loss: 2.5232 | LR: 2.08e-04 | 5179.0ms/step | 11.6 samples/s +Step 28310/50462 | Loss: 2.5545 | LR: 2.07e-04 | 5131.7ms/step | 11.7 samples/s +Step 28320/50462 | Loss: 2.5457 | LR: 2.07e-04 | 5154.1ms/step | 11.6 samples/s +Step 28330/50462 | Loss: 2.5548 | LR: 2.07e-04 | 5128.2ms/step | 11.7 samples/s +Step 28340/50462 | Loss: 2.5375 | LR: 2.07e-04 | 5112.2ms/step | 11.7 samples/s +Step 28350/50462 | Loss: 2.5238 | LR: 2.07e-04 | 5136.2ms/step | 11.7 samples/s +Step 28360/50462 | Loss: 2.5266 | LR: 2.07e-04 | 5185.8ms/step | 11.6 samples/s +Step 28370/50462 | Loss: 2.5217 | LR: 2.07e-04 | 5149.7ms/step | 11.7 samples/s +Step 28380/50462 | Loss: 2.5291 | LR: 2.06e-04 | 5169.5ms/step | 11.6 samples/s +Step 28390/50462 | Loss: 2.5248 | LR: 2.06e-04 | 5141.0ms/step | 11.7 samples/s +Step 28400/50462 | Loss: 2.5578 | LR: 2.06e-04 | 5146.4ms/step | 11.7 samples/s +Step 28410/50462 | Loss: 2.5356 | LR: 2.06e-04 | 5149.7ms/step | 11.7 samples/s +Step 28420/50462 | Loss: 2.5568 | LR: 2.06e-04 | 5188.2ms/step | 11.6 samples/s +Step 28430/50462 | Loss: 2.5039 | LR: 2.06e-04 | 5167.4ms/step | 11.6 samples/s +Step 28440/50462 | Loss: 2.5427 | LR: 2.05e-04 | 5154.0ms/step | 11.6 samples/s +Step 28450/50462 | Loss: 2.5111 | LR: 2.05e-04 | 5158.4ms/step | 11.6 samples/s +Step 28460/50462 | Loss: 2.5406 | LR: 2.05e-04 | 5156.1ms/step | 11.6 samples/s +Step 28470/50462 | Loss: 2.5435 | LR: 2.05e-04 | 5169.6ms/step | 11.6 samples/s +Step 28480/50462 | Loss: 2.5335 | LR: 2.05e-04 | 5172.7ms/step | 11.6 samples/s +Step 28490/50462 | Loss: 2.5275 | LR: 2.05e-04 | 5166.3ms/step | 11.6 samples/s +Step 28500/50462 | Loss: 2.5376 | LR: 2.05e-04 | 5172.7ms/step | 11.6 samples/s +Validation at step 28500: Loss = 2.5474, Perplexity = 12.77 +Step 28510/50462 | Loss: 2.5251 | LR: 2.04e-04 | 7684.7ms/step | 7.8 samples/s +Step 28520/50462 | Loss: 2.5385 | LR: 2.04e-04 | 5198.6ms/step | 11.5 samples/s +Step 28530/50462 | Loss: 2.4882 | LR: 2.04e-04 | 5176.1ms/step | 11.6 samples/s +Step 28540/50462 | Loss: 2.5320 | LR: 2.04e-04 | 5239.6ms/step | 11.5 samples/s +Step 28550/50462 | Loss: 2.5834 | LR: 2.04e-04 | 5137.5ms/step | 11.7 samples/s +Step 28560/50462 | Loss: 2.5236 | LR: 2.04e-04 | 5144.2ms/step | 11.7 samples/s +Step 28570/50462 | Loss: 2.5389 | LR: 2.03e-04 | 5147.7ms/step | 11.7 samples/s +Step 28580/50462 | Loss: 2.5348 | LR: 2.03e-04 | 5150.8ms/step | 11.6 samples/s +Step 28590/50462 | Loss: 2.5514 | LR: 2.03e-04 | 5208.0ms/step | 11.5 samples/s +Step 28600/50462 | Loss: 2.5219 | LR: 2.03e-04 | 5203.1ms/step | 11.5 samples/s +Step 28610/50462 | Loss: 2.5372 | LR: 2.03e-04 | 5169.8ms/step | 11.6 samples/s +Step 28620/50462 | Loss: 2.5168 | LR: 2.03e-04 | 5187.3ms/step | 11.6 samples/s +Step 28630/50462 | Loss: 2.5321 | LR: 2.03e-04 | 5159.9ms/step | 11.6 samples/s +Step 28640/50462 | Loss: 2.5444 | LR: 2.02e-04 | 5175.4ms/step | 11.6 samples/s +Step 28650/50462 | Loss: 2.5337 | LR: 2.02e-04 | 5185.8ms/step | 11.6 samples/s +Step 28660/50462 | Loss: 2.5357 | LR: 2.02e-04 | 5216.0ms/step | 11.5 samples/s +Step 28670/50462 | Loss: 2.5334 | LR: 2.02e-04 | 5180.5ms/step | 11.6 samples/s +Step 28680/50462 | Loss: 2.5477 | LR: 2.02e-04 | 5176.9ms/step | 11.6 samples/s +Step 28690/50462 | Loss: 2.5507 | LR: 2.02e-04 | 5196.5ms/step | 11.5 samples/s +Step 28700/50462 | Loss: 2.5308 | LR: 2.01e-04 | 5196.6ms/step | 11.5 samples/s +Step 28710/50462 | Loss: 2.5150 | LR: 2.01e-04 | 5207.8ms/step | 11.5 samples/s +Step 28720/50462 | Loss: 2.5483 | LR: 2.01e-04 | 5198.0ms/step | 11.5 samples/s +Step 28730/50462 | Loss: 2.5192 | LR: 2.01e-04 | 5194.3ms/step | 11.6 samples/s +Step 28740/50462 | Loss: 2.5380 | LR: 2.01e-04 | 5194.2ms/step | 11.6 samples/s +Step 28750/50462 | Loss: 2.5237 | LR: 2.01e-04 | 5194.6ms/step | 11.6 samples/s +Validation at step 28750: Loss = 2.5453, Perplexity = 12.75 +Step 28760/50462 | Loss: 2.5763 | LR: 2.00e-04 | 7696.0ms/step | 7.8 samples/s +Step 28770/50462 | Loss: 2.5667 | LR: 2.00e-04 | 5137.3ms/step | 11.7 samples/s +Step 28780/50462 | Loss: 2.5363 | LR: 2.00e-04 | 5163.0ms/step | 11.6 samples/s +Step 28790/50462 | Loss: 2.5097 | LR: 2.00e-04 | 5135.3ms/step | 11.7 samples/s +Step 28800/50462 | Loss: 2.5718 | LR: 2.00e-04 | 5232.6ms/step | 11.5 samples/s +Step 28810/50462 | Loss: 2.5064 | LR: 2.00e-04 | 5177.1ms/step | 11.6 samples/s +Step 28820/50462 | Loss: 2.5199 | LR: 2.00e-04 | 5207.0ms/step | 11.5 samples/s +Step 28830/50462 | Loss: 2.5277 | LR: 1.99e-04 | 5260.7ms/step | 11.4 samples/s +Step 28840/50462 | Loss: 2.5366 | LR: 1.99e-04 | 5199.3ms/step | 11.5 samples/s +Step 28850/50462 | Loss: 2.5251 | LR: 1.99e-04 | 5175.4ms/step | 11.6 samples/s +Step 28860/50462 | Loss: 2.5186 | LR: 1.99e-04 | 5201.9ms/step | 11.5 samples/s +Step 28870/50462 | Loss: 2.5403 | LR: 1.99e-04 | 5186.7ms/step | 11.6 samples/s +Step 28880/50462 | Loss: 2.5603 | LR: 1.99e-04 | 5173.9ms/step | 11.6 samples/s +Step 28890/50462 | Loss: 2.5127 | LR: 1.98e-04 | 5163.6ms/step | 11.6 samples/s +Step 28900/50462 | Loss: 2.5277 | LR: 1.98e-04 | 5168.9ms/step | 11.6 samples/s +Step 28910/50462 | Loss: 2.5350 | LR: 1.98e-04 | 5163.9ms/step | 11.6 samples/s +Step 28920/50462 | Loss: 2.5235 | LR: 1.98e-04 | 5163.5ms/step | 11.6 samples/s +Step 28930/50462 | Loss: 2.5495 | LR: 1.98e-04 | 5150.8ms/step | 11.6 samples/s +Step 28940/50462 | Loss: 2.5252 | LR: 1.98e-04 | 5177.2ms/step | 11.6 samples/s +Step 28950/50462 | Loss: 2.5357 | LR: 1.98e-04 | 5191.4ms/step | 11.6 samples/s +Step 28960/50462 | Loss: 2.5372 | LR: 1.97e-04 | 5203.7ms/step | 11.5 samples/s +Step 28970/50462 | Loss: 2.5404 | LR: 1.97e-04 | 5181.1ms/step | 11.6 samples/s +Step 28980/50462 | Loss: 2.5328 | LR: 1.97e-04 | 5159.0ms/step | 11.6 samples/s +Step 28990/50462 | Loss: 2.5263 | LR: 1.97e-04 | 5173.4ms/step | 11.6 samples/s +Step 29000/50462 | Loss: 2.5319 | LR: 1.97e-04 | 5168.6ms/step | 11.6 samples/s +Validation at step 29000: Loss = 2.5442, Perplexity = 12.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-29000 +Step 29010/50462 | Loss: 2.5391 | LR: 1.97e-04 | 8647.3ms/step | 6.9 samples/s +Step 29020/50462 | Loss: 2.5629 | LR: 1.96e-04 | 5128.1ms/step | 11.7 samples/s +Step 29030/50462 | Loss: 2.5345 | LR: 1.96e-04 | 5122.6ms/step | 11.7 samples/s +Step 29040/50462 | Loss: 2.5134 | LR: 1.96e-04 | 5113.8ms/step | 11.7 samples/s +Step 29050/50462 | Loss: 2.5226 | LR: 1.96e-04 | 5153.2ms/step | 11.6 samples/s +Step 29060/50462 | Loss: 2.5532 | LR: 1.96e-04 | 5154.3ms/step | 11.6 samples/s +Step 29070/50462 | Loss: 2.5348 | LR: 1.96e-04 | 5148.0ms/step | 11.7 samples/s +Step 29080/50462 | Loss: 2.5449 | LR: 1.96e-04 | 5194.4ms/step | 11.6 samples/s +Step 29090/50462 | Loss: 2.5409 | LR: 1.95e-04 | 5127.5ms/step | 11.7 samples/s +Step 29100/50462 | Loss: 2.5020 | LR: 1.95e-04 | 5145.1ms/step | 11.7 samples/s +Step 29110/50462 | Loss: 2.5404 | LR: 1.95e-04 | 5126.9ms/step | 11.7 samples/s +Step 29120/50462 | Loss: 2.5488 | LR: 1.95e-04 | 5237.2ms/step | 11.5 samples/s +Step 29130/50462 | Loss: 2.5298 | LR: 1.95e-04 | 5162.7ms/step | 11.6 samples/s +Step 29140/50462 | Loss: 2.5284 | LR: 1.95e-04 | 5134.7ms/step | 11.7 samples/s +Step 29150/50462 | Loss: 2.5610 | LR: 1.94e-04 | 5166.0ms/step | 11.6 samples/s +Step 29160/50462 | Loss: 2.5472 | LR: 1.94e-04 | 5167.5ms/step | 11.6 samples/s +Step 29170/50462 | Loss: 2.5207 | LR: 1.94e-04 | 5126.0ms/step | 11.7 samples/s +Step 29180/50462 | Loss: 2.5217 | LR: 1.94e-04 | 5161.4ms/step | 11.6 samples/s +Step 29190/50462 | Loss: 2.5359 | LR: 1.94e-04 | 5186.7ms/step | 11.6 samples/s +Step 29200/50462 | Loss: 2.5129 | LR: 1.94e-04 | 5158.3ms/step | 11.6 samples/s +Step 29210/50462 | Loss: 2.5437 | LR: 1.94e-04 | 5178.3ms/step | 11.6 samples/s +Step 29220/50462 | Loss: 2.5559 | LR: 1.93e-04 | 5169.8ms/step | 11.6 samples/s +Step 29230/50462 | Loss: 2.5222 | LR: 1.93e-04 | 5160.0ms/step | 11.6 samples/s +Step 29240/50462 | Loss: 2.5411 | LR: 1.93e-04 | 5125.1ms/step | 11.7 samples/s +Step 29250/50462 | Loss: 2.5640 | LR: 1.93e-04 | 5151.2ms/step | 11.6 samples/s +Validation at step 29250: Loss = 2.5426, Perplexity = 12.71 +Step 29260/50462 | Loss: 2.5327 | LR: 1.93e-04 | 7671.4ms/step | 7.8 samples/s +Step 29270/50462 | Loss: 2.5352 | LR: 1.93e-04 | 5124.4ms/step | 11.7 samples/s +Step 29280/50462 | Loss: 2.5276 | LR: 1.92e-04 | 5135.4ms/step | 11.7 samples/s +Step 29290/50462 | Loss: 2.5485 | LR: 1.92e-04 | 5131.0ms/step | 11.7 samples/s +Step 29300/50462 | Loss: 2.5344 | LR: 1.92e-04 | 5114.9ms/step | 11.7 samples/s +Step 29310/50462 | Loss: 2.5355 | LR: 1.92e-04 | 5116.2ms/step | 11.7 samples/s +Step 29320/50462 | Loss: 2.5352 | LR: 1.92e-04 | 5129.5ms/step | 11.7 samples/s +Step 29330/50462 | Loss: 2.5230 | LR: 1.92e-04 | 5136.5ms/step | 11.7 samples/s +Step 29340/50462 | Loss: 2.5451 | LR: 1.92e-04 | 5136.7ms/step | 11.7 samples/s +Step 29350/50462 | Loss: 2.5565 | LR: 1.91e-04 | 5130.0ms/step | 11.7 samples/s +Step 29360/50462 | Loss: 2.5086 | LR: 1.91e-04 | 5170.9ms/step | 11.6 samples/s +Step 29370/50462 | Loss: 2.5557 | LR: 1.91e-04 | 5142.9ms/step | 11.7 samples/s +Step 29380/50462 | Loss: 2.5317 | LR: 1.91e-04 | 5141.0ms/step | 11.7 samples/s +Step 29390/50462 | Loss: 2.5249 | LR: 1.91e-04 | 5138.8ms/step | 11.7 samples/s +Step 29400/50462 | Loss: 2.5549 | LR: 1.91e-04 | 5156.7ms/step | 11.6 samples/s +Step 29410/50462 | Loss: 2.5326 | LR: 1.90e-04 | 5246.1ms/step | 11.4 samples/s +Step 29420/50462 | Loss: 2.5295 | LR: 1.90e-04 | 5176.5ms/step | 11.6 samples/s +Step 29430/50462 | Loss: 2.5007 | LR: 1.90e-04 | 5152.4ms/step | 11.6 samples/s +Step 29440/50462 | Loss: 2.5243 | LR: 1.90e-04 | 5178.0ms/step | 11.6 samples/s +Step 29450/50462 | Loss: 2.5104 | LR: 1.90e-04 | 5162.6ms/step | 11.6 samples/s +Step 29460/50462 | Loss: 2.5493 | LR: 1.90e-04 | 5150.6ms/step | 11.6 samples/s +Step 29470/50462 | Loss: 2.5393 | LR: 1.90e-04 | 5189.3ms/step | 11.6 samples/s +Step 29480/50462 | Loss: 2.5208 | LR: 1.89e-04 | 5190.7ms/step | 11.6 samples/s +Step 29490/50462 | Loss: 2.5020 | LR: 1.89e-04 | 5165.9ms/step | 11.6 samples/s +Step 29500/50462 | Loss: 2.5184 | LR: 1.89e-04 | 5187.0ms/step | 11.6 samples/s +Validation at step 29500: Loss = 2.5411, Perplexity = 12.69 +Step 29510/50462 | Loss: 2.5187 | LR: 1.89e-04 | 7729.4ms/step | 7.8 samples/s +Step 29520/50462 | Loss: 2.5437 | LR: 1.89e-04 | 5196.8ms/step | 11.5 samples/s +Step 29530/50462 | Loss: 2.5496 | LR: 1.89e-04 | 5190.1ms/step | 11.6 samples/s +Step 29540/50462 | Loss: 2.5268 | LR: 1.88e-04 | 5162.7ms/step | 11.6 samples/s +Step 29550/50462 | Loss: 2.4980 | LR: 1.88e-04 | 5145.1ms/step | 11.7 samples/s +Step 29560/50462 | Loss: 2.5233 | LR: 1.88e-04 | 5183.0ms/step | 11.6 samples/s +Step 29570/50462 | Loss: 2.5152 | LR: 1.88e-04 | 5185.9ms/step | 11.6 samples/s +Step 29580/50462 | Loss: 2.5710 | LR: 1.88e-04 | 5188.9ms/step | 11.6 samples/s +Step 29590/50462 | Loss: 2.5177 | LR: 1.88e-04 | 5195.4ms/step | 11.5 samples/s +Step 29600/50462 | Loss: 2.5470 | LR: 1.88e-04 | 5186.7ms/step | 11.6 samples/s +Step 29610/50462 | Loss: 2.5513 | LR: 1.87e-04 | 5193.1ms/step | 11.6 samples/s +Step 29620/50462 | Loss: 2.5243 | LR: 1.87e-04 | 5197.1ms/step | 11.5 samples/s +Step 29630/50462 | Loss: 2.5157 | LR: 1.87e-04 | 5252.3ms/step | 11.4 samples/s +Step 29640/50462 | Loss: 2.5314 | LR: 1.87e-04 | 5181.3ms/step | 11.6 samples/s +Step 29650/50462 | Loss: 2.5447 | LR: 1.87e-04 | 5199.2ms/step | 11.5 samples/s +Step 29660/50462 | Loss: 2.4961 | LR: 1.87e-04 | 5183.1ms/step | 11.6 samples/s +Step 29670/50462 | Loss: 2.5331 | LR: 1.86e-04 | 5176.8ms/step | 11.6 samples/s +Step 29680/50462 | Loss: 2.5350 | LR: 1.86e-04 | 5186.8ms/step | 11.6 samples/s +Step 29690/50462 | Loss: 2.5114 | LR: 1.86e-04 | 5189.4ms/step | 11.6 samples/s +Step 29700/50462 | Loss: 2.5426 | LR: 1.86e-04 | 5220.2ms/step | 11.5 samples/s +Step 29710/50462 | Loss: 2.5571 | LR: 1.86e-04 | 5166.2ms/step | 11.6 samples/s +Step 29720/50462 | Loss: 2.5351 | LR: 1.86e-04 | 5177.7ms/step | 11.6 samples/s +Step 29730/50462 | Loss: 2.5307 | LR: 1.86e-04 | 5169.9ms/step | 11.6 samples/s +Step 29740/50462 | Loss: 2.5430 | LR: 1.85e-04 | 5206.1ms/step | 11.5 samples/s +Step 29750/50462 | Loss: 2.5343 | LR: 1.85e-04 | 5181.3ms/step | 11.6 samples/s +Validation at step 29750: Loss = 2.5401, Perplexity = 12.68 +Step 29760/50462 | Loss: 2.5364 | LR: 1.85e-04 | 7711.2ms/step | 7.8 samples/s +Step 29770/50462 | Loss: 2.5342 | LR: 1.85e-04 | 5188.7ms/step | 11.6 samples/s +Step 29780/50462 | Loss: 2.5188 | LR: 1.85e-04 | 5181.9ms/step | 11.6 samples/s +Step 29790/50462 | Loss: 2.5304 | LR: 1.85e-04 | 5173.9ms/step | 11.6 samples/s +Step 29800/50462 | Loss: 2.5018 | LR: 1.85e-04 | 5189.9ms/step | 11.6 samples/s +Step 29810/50462 | Loss: 2.4979 | LR: 1.84e-04 | 5151.6ms/step | 11.6 samples/s +Step 29820/50462 | Loss: 2.5217 | LR: 1.84e-04 | 5182.5ms/step | 11.6 samples/s +Step 29830/50462 | Loss: 2.5371 | LR: 1.84e-04 | 5185.0ms/step | 11.6 samples/s +Step 29840/50462 | Loss: 2.5163 | LR: 1.84e-04 | 5178.0ms/step | 11.6 samples/s +Step 29850/50462 | Loss: 2.5191 | LR: 1.84e-04 | 5190.3ms/step | 11.6 samples/s +Step 29860/50462 | Loss: 2.5240 | LR: 1.84e-04 | 5157.6ms/step | 11.6 samples/s +Step 29870/50462 | Loss: 2.5302 | LR: 1.83e-04 | 5187.3ms/step | 11.6 samples/s +Step 29880/50462 | Loss: 2.5346 | LR: 1.83e-04 | 5177.6ms/step | 11.6 samples/s +Step 29890/50462 | Loss: 2.5328 | LR: 1.83e-04 | 5203.6ms/step | 11.5 samples/s +Step 29900/50462 | Loss: 2.5286 | LR: 1.83e-04 | 5178.2ms/step | 11.6 samples/s +Step 29910/50462 | Loss: 2.5134 | LR: 1.83e-04 | 5236.8ms/step | 11.5 samples/s +Step 29920/50462 | Loss: 2.5232 | LR: 1.83e-04 | 5151.1ms/step | 11.6 samples/s +Step 29930/50462 | Loss: 2.5464 | LR: 1.83e-04 | 5154.6ms/step | 11.6 samples/s +Step 29940/50462 | Loss: 2.5224 | LR: 1.82e-04 | 5138.9ms/step | 11.7 samples/s +Step 29950/50462 | Loss: 2.5645 | LR: 1.82e-04 | 5168.7ms/step | 11.6 samples/s +Step 29960/50462 | Loss: 2.5407 | LR: 1.82e-04 | 5135.0ms/step | 11.7 samples/s +Step 29970/50462 | Loss: 2.5582 | LR: 1.82e-04 | 5165.8ms/step | 11.6 samples/s +Step 29980/50462 | Loss: 2.5042 | LR: 1.82e-04 | 5250.9ms/step | 11.4 samples/s +Step 29990/50462 | Loss: 2.5003 | LR: 1.82e-04 | 5180.9ms/step | 11.6 samples/s +Step 30000/50462 | Loss: 2.4773 | LR: 1.81e-04 | 5158.5ms/step | 11.6 samples/s +Validation at step 30000: Loss = 2.5387, Perplexity = 12.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-30000 +Step 30010/50462 | Loss: 2.4957 | LR: 1.81e-04 | 8614.3ms/step | 7.0 samples/s +Step 30020/50462 | Loss: 2.5093 | LR: 1.81e-04 | 5155.0ms/step | 11.6 samples/s +Step 30030/50462 | Loss: 2.5281 | LR: 1.81e-04 | 5232.9ms/step | 11.5 samples/s +Step 30040/50462 | Loss: 2.5847 | LR: 1.81e-04 | 5247.3ms/step | 11.4 samples/s +Step 30050/50462 | Loss: 2.5351 | LR: 1.81e-04 | 5255.3ms/step | 11.4 samples/s +Step 30060/50462 | Loss: 2.5348 | LR: 1.81e-04 | 5255.4ms/step | 11.4 samples/s +Step 30070/50462 | Loss: 2.5410 | LR: 1.80e-04 | 5255.4ms/step | 11.4 samples/s +Step 30080/50462 | Loss: 2.5340 | LR: 1.80e-04 | 5250.9ms/step | 11.4 samples/s +Step 30090/50462 | Loss: 2.5411 | LR: 1.80e-04 | 5260.9ms/step | 11.4 samples/s +Step 30100/50462 | Loss: 2.5652 | LR: 1.80e-04 | 5251.9ms/step | 11.4 samples/s +Step 30110/50462 | Loss: 2.5571 | LR: 1.80e-04 | 5253.6ms/step | 11.4 samples/s +Step 30120/50462 | Loss: 2.5327 | LR: 1.80e-04 | 5251.8ms/step | 11.4 samples/s +Step 30130/50462 | Loss: 2.4932 | LR: 1.79e-04 | 5250.9ms/step | 11.4 samples/s +Step 30140/50462 | Loss: 2.5471 | LR: 1.79e-04 | 5257.1ms/step | 11.4 samples/s +Step 30150/50462 | Loss: 2.5374 | LR: 1.79e-04 | 5247.4ms/step | 11.4 samples/s +Step 30160/50462 | Loss: 2.5359 | LR: 1.79e-04 | 5243.5ms/step | 11.4 samples/s +Step 30170/50462 | Loss: 2.5159 | LR: 1.79e-04 | 5256.2ms/step | 11.4 samples/s +Step 30180/50462 | Loss: 2.5031 | LR: 1.79e-04 | 5290.4ms/step | 11.3 samples/s +Step 30190/50462 | Loss: 2.4999 | LR: 1.79e-04 | 5233.3ms/step | 11.5 samples/s +Step 30200/50462 | Loss: 2.5290 | LR: 1.78e-04 | 5225.7ms/step | 11.5 samples/s +Step 30210/50462 | Loss: 2.5501 | LR: 1.78e-04 | 5221.3ms/step | 11.5 samples/s +Step 30220/50462 | Loss: 2.5116 | LR: 1.78e-04 | 5252.2ms/step | 11.4 samples/s +Step 30230/50462 | Loss: 2.5258 | LR: 1.78e-04 | 5250.8ms/step | 11.4 samples/s +Step 30240/50462 | Loss: 2.5448 | LR: 1.78e-04 | 5257.3ms/step | 11.4 samples/s +Step 30250/50462 | Loss: 2.5075 | LR: 1.78e-04 | 5250.3ms/step | 11.4 samples/s +Validation at step 30250: Loss = 2.5379, Perplexity = 12.65 +Step 30260/50462 | Loss: 2.5282 | LR: 1.78e-04 | 7803.8ms/step | 7.7 samples/s +Step 30270/50462 | Loss: 2.5424 | LR: 1.77e-04 | 5314.2ms/step | 11.3 samples/s +Step 30280/50462 | Loss: 2.5388 | LR: 1.77e-04 | 5228.6ms/step | 11.5 samples/s +Step 30290/50462 | Loss: 2.5312 | LR: 1.77e-04 | 5217.3ms/step | 11.5 samples/s +Step 30300/50462 | Loss: 2.5330 | LR: 1.77e-04 | 5247.6ms/step | 11.4 samples/s +Step 30310/50462 | Loss: 2.5133 | LR: 1.77e-04 | 5255.8ms/step | 11.4 samples/s +Step 30320/50462 | Loss: 2.5189 | LR: 1.77e-04 | 5256.6ms/step | 11.4 samples/s +Step 30330/50462 | Loss: 2.5350 | LR: 1.76e-04 | 5250.3ms/step | 11.4 samples/s +Step 30340/50462 | Loss: 2.5077 | LR: 1.76e-04 | 5243.2ms/step | 11.4 samples/s +Step 30350/50462 | Loss: 2.5577 | LR: 1.76e-04 | 5244.2ms/step | 11.4 samples/s +Step 30360/50462 | Loss: 2.5382 | LR: 1.76e-04 | 5256.3ms/step | 11.4 samples/s +Step 30370/50462 | Loss: 2.5315 | LR: 1.76e-04 | 5245.5ms/step | 11.4 samples/s +Step 30380/50462 | Loss: 2.5276 | LR: 1.76e-04 | 5248.2ms/step | 11.4 samples/s +Step 30390/50462 | Loss: 2.5124 | LR: 1.76e-04 | 5237.3ms/step | 11.5 samples/s +Step 30400/50462 | Loss: 2.5006 | LR: 1.75e-04 | 5248.1ms/step | 11.4 samples/s +Step 30410/50462 | Loss: 2.5204 | LR: 1.75e-04 | 5251.3ms/step | 11.4 samples/s +Step 30420/50462 | Loss: 2.4814 | LR: 1.75e-04 | 5248.7ms/step | 11.4 samples/s +Step 30430/50462 | Loss: 2.5354 | LR: 1.75e-04 | 5242.8ms/step | 11.4 samples/s +Step 30440/50462 | Loss: 2.5369 | LR: 1.75e-04 | 5206.4ms/step | 11.5 samples/s +Step 30450/50462 | Loss: 2.5484 | LR: 1.75e-04 | 5232.2ms/step | 11.5 samples/s +Step 30460/50462 | Loss: 2.5014 | LR: 1.75e-04 | 5279.9ms/step | 11.4 samples/s +Step 30470/50462 | Loss: 2.5250 | LR: 1.74e-04 | 5199.9ms/step | 11.5 samples/s +Step 30480/50462 | Loss: 2.5450 | LR: 1.74e-04 | 5211.5ms/step | 11.5 samples/s +Step 30490/50462 | Loss: 2.5400 | LR: 1.74e-04 | 5242.4ms/step | 11.4 samples/s +Step 30500/50462 | Loss: 2.5200 | LR: 1.74e-04 | 5208.5ms/step | 11.5 samples/s +Validation at step 30500: Loss = 2.5363, Perplexity = 12.63 +Step 30510/50462 | Loss: 2.5185 | LR: 1.74e-04 | 7765.2ms/step | 7.7 samples/s +Step 30520/50462 | Loss: 2.5200 | LR: 1.74e-04 | 5238.8ms/step | 11.5 samples/s +Step 30530/50462 | Loss: 2.5416 | LR: 1.73e-04 | 5241.4ms/step | 11.4 samples/s +Step 30540/50462 | Loss: 2.4975 | LR: 1.73e-04 | 5230.7ms/step | 11.5 samples/s +Step 30550/50462 | Loss: 2.5459 | LR: 1.73e-04 | 5283.5ms/step | 11.4 samples/s +Step 30560/50462 | Loss: 2.4979 | LR: 1.73e-04 | 5252.6ms/step | 11.4 samples/s +Step 30570/50462 | Loss: 2.5314 | LR: 1.73e-04 | 5184.5ms/step | 11.6 samples/s +Step 30580/50462 | Loss: 2.5317 | LR: 1.73e-04 | 5233.7ms/step | 11.5 samples/s +Step 30590/50462 | Loss: 2.4992 | LR: 1.73e-04 | 5242.5ms/step | 11.4 samples/s +Step 30600/50462 | Loss: 2.5044 | LR: 1.72e-04 | 5249.3ms/step | 11.4 samples/s +Step 30610/50462 | Loss: 2.5508 | LR: 1.72e-04 | 5251.3ms/step | 11.4 samples/s +Step 30620/50462 | Loss: 2.5118 | LR: 1.72e-04 | 5251.7ms/step | 11.4 samples/s +Step 30630/50462 | Loss: 2.4976 | LR: 1.72e-04 | 5239.2ms/step | 11.5 samples/s +Step 30640/50462 | Loss: 2.5030 | LR: 1.72e-04 | 5240.8ms/step | 11.4 samples/s +Step 30650/50462 | Loss: 2.5328 | LR: 1.72e-04 | 5213.9ms/step | 11.5 samples/s +Step 30660/50462 | Loss: 2.4895 | LR: 1.72e-04 | 5220.1ms/step | 11.5 samples/s +Step 30670/50462 | Loss: 2.5095 | LR: 1.71e-04 | 5230.2ms/step | 11.5 samples/s +Step 30680/50462 | Loss: 2.5122 | LR: 1.71e-04 | 5222.8ms/step | 11.5 samples/s +Step 30690/50462 | Loss: 2.5101 | LR: 1.71e-04 | 5205.1ms/step | 11.5 samples/s +Step 30700/50462 | Loss: 2.5388 | LR: 1.71e-04 | 5223.2ms/step | 11.5 samples/s +Step 30710/50462 | Loss: 2.5207 | LR: 1.71e-04 | 5201.8ms/step | 11.5 samples/s +Step 30720/50462 | Loss: 2.5494 | LR: 1.71e-04 | 5220.7ms/step | 11.5 samples/s +Step 30730/50462 | Loss: 2.5221 | LR: 1.70e-04 | 5227.8ms/step | 11.5 samples/s +Step 30740/50462 | Loss: 2.5188 | LR: 1.70e-04 | 5262.7ms/step | 11.4 samples/s +Step 30750/50462 | Loss: 2.5124 | LR: 1.70e-04 | 5204.3ms/step | 11.5 samples/s +Validation at step 30750: Loss = 2.5348, Perplexity = 12.61 +Step 30760/50462 | Loss: 2.5344 | LR: 1.70e-04 | 7748.2ms/step | 7.7 samples/s +Step 30770/50462 | Loss: 2.5265 | LR: 1.70e-04 | 5201.6ms/step | 11.5 samples/s +Step 30780/50462 | Loss: 2.5204 | LR: 1.70e-04 | 5170.4ms/step | 11.6 samples/s +Step 30790/50462 | Loss: 2.5341 | LR: 1.70e-04 | 5191.5ms/step | 11.6 samples/s +Step 30800/50462 | Loss: 2.5031 | LR: 1.69e-04 | 5186.7ms/step | 11.6 samples/s +Step 30810/50462 | Loss: 2.5017 | LR: 1.69e-04 | 5220.0ms/step | 11.5 samples/s +Step 30820/50462 | Loss: 2.5319 | LR: 1.69e-04 | 5205.3ms/step | 11.5 samples/s +Step 30830/50462 | Loss: 2.5296 | LR: 1.69e-04 | 5204.6ms/step | 11.5 samples/s +Step 30840/50462 | Loss: 2.5060 | LR: 1.69e-04 | 5276.6ms/step | 11.4 samples/s +Step 30850/50462 | Loss: 2.5271 | LR: 1.69e-04 | 5206.6ms/step | 11.5 samples/s +Step 30860/50462 | Loss: 2.5410 | LR: 1.69e-04 | 5211.5ms/step | 11.5 samples/s +Step 30870/50462 | Loss: 2.5095 | LR: 1.68e-04 | 5220.5ms/step | 11.5 samples/s +Step 30880/50462 | Loss: 2.4850 | LR: 1.68e-04 | 5228.9ms/step | 11.5 samples/s +Step 30890/50462 | Loss: 2.5109 | LR: 1.68e-04 | 5224.3ms/step | 11.5 samples/s +Step 30900/50462 | Loss: 2.4966 | LR: 1.68e-04 | 5229.7ms/step | 11.5 samples/s +Step 30910/50462 | Loss: 2.4985 | LR: 1.68e-04 | 5221.3ms/step | 11.5 samples/s +Step 30920/50462 | Loss: 2.5238 | LR: 1.68e-04 | 5225.7ms/step | 11.5 samples/s +Step 30930/50462 | Loss: 2.5030 | LR: 1.67e-04 | 5189.6ms/step | 11.6 samples/s +Step 30940/50462 | Loss: 2.4803 | LR: 1.67e-04 | 5169.2ms/step | 11.6 samples/s +Step 30950/50462 | Loss: 2.5438 | LR: 1.67e-04 | 5213.2ms/step | 11.5 samples/s +Step 30960/50462 | Loss: 2.5183 | LR: 1.67e-04 | 5185.8ms/step | 11.6 samples/s +Step 30970/50462 | Loss: 2.5409 | LR: 1.67e-04 | 5221.1ms/step | 11.5 samples/s +Step 30980/50462 | Loss: 2.5377 | LR: 1.67e-04 | 5211.0ms/step | 11.5 samples/s +Step 30990/50462 | Loss: 2.5189 | LR: 1.67e-04 | 5217.4ms/step | 11.5 samples/s +Step 31000/50462 | Loss: 2.5129 | LR: 1.66e-04 | 5212.2ms/step | 11.5 samples/s +Validation at step 31000: Loss = 2.5341, Perplexity = 12.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-31000 +Step 31010/50462 | Loss: 2.5531 | LR: 1.66e-04 | 8763.7ms/step | 6.8 samples/s +Step 31020/50462 | Loss: 2.5197 | LR: 1.66e-04 | 5217.0ms/step | 11.5 samples/s +Step 31030/50462 | Loss: 2.5120 | LR: 1.66e-04 | 5227.4ms/step | 11.5 samples/s +Step 31040/50462 | Loss: 2.4898 | LR: 1.66e-04 | 5220.6ms/step | 11.5 samples/s +Step 31050/50462 | Loss: 2.5152 | LR: 1.66e-04 | 5219.8ms/step | 11.5 samples/s +Step 31060/50462 | Loss: 2.5153 | LR: 1.66e-04 | 5226.0ms/step | 11.5 samples/s +Step 31070/50462 | Loss: 2.5055 | LR: 1.65e-04 | 5248.2ms/step | 11.4 samples/s +Step 31080/50462 | Loss: 2.4825 | LR: 1.65e-04 | 5230.3ms/step | 11.5 samples/s +Step 31090/50462 | Loss: 2.5056 | LR: 1.65e-04 | 5236.5ms/step | 11.5 samples/s +Step 31100/50462 | Loss: 2.5440 | LR: 1.65e-04 | 5239.2ms/step | 11.5 samples/s +Step 31110/50462 | Loss: 2.5080 | LR: 1.65e-04 | 5233.7ms/step | 11.5 samples/s +Step 31120/50462 | Loss: 2.5452 | LR: 1.65e-04 | 5237.7ms/step | 11.5 samples/s +Step 31130/50462 | Loss: 2.5376 | LR: 1.64e-04 | 5293.9ms/step | 11.3 samples/s +Step 31140/50462 | Loss: 2.4881 | LR: 1.64e-04 | 5207.5ms/step | 11.5 samples/s +Step 31150/50462 | Loss: 2.5495 | LR: 1.64e-04 | 5206.5ms/step | 11.5 samples/s +Step 31160/50462 | Loss: 2.5122 | LR: 1.64e-04 | 5212.6ms/step | 11.5 samples/s +Step 31170/50462 | Loss: 2.5318 | LR: 1.64e-04 | 5212.8ms/step | 11.5 samples/s +Step 31180/50462 | Loss: 2.5142 | LR: 1.64e-04 | 5214.4ms/step | 11.5 samples/s +Step 31190/50462 | Loss: 2.5245 | LR: 1.64e-04 | 5243.1ms/step | 11.4 samples/s +Step 31200/50462 | Loss: 2.4963 | LR: 1.63e-04 | 5237.2ms/step | 11.5 samples/s +Step 31210/50462 | Loss: 2.5169 | LR: 1.63e-04 | 5231.6ms/step | 11.5 samples/s +Step 31220/50462 | Loss: 2.4999 | LR: 1.63e-04 | 5240.8ms/step | 11.4 samples/s +Step 31230/50462 | Loss: 2.5170 | LR: 1.63e-04 | 5226.9ms/step | 11.5 samples/s +Step 31240/50462 | Loss: 2.4962 | LR: 1.63e-04 | 5191.9ms/step | 11.6 samples/s +Step 31250/50462 | Loss: 2.5093 | LR: 1.63e-04 | 5219.8ms/step | 11.5 samples/s +Validation at step 31250: Loss = 2.5327, Perplexity = 12.59 +Step 31260/50462 | Loss: 2.4997 | LR: 1.63e-04 | 7762.5ms/step | 7.7 samples/s +Step 31270/50462 | Loss: 2.5099 | LR: 1.62e-04 | 5241.6ms/step | 11.4 samples/s +Step 31280/50462 | Loss: 2.5449 | LR: 1.62e-04 | 5253.0ms/step | 11.4 samples/s +Step 31290/50462 | Loss: 2.5404 | LR: 1.62e-04 | 5253.9ms/step | 11.4 samples/s +Step 31300/50462 | Loss: 2.5233 | LR: 1.62e-04 | 5202.3ms/step | 11.5 samples/s +Step 31310/50462 | Loss: 2.5217 | LR: 1.62e-04 | 5149.2ms/step | 11.7 samples/s +Step 31320/50462 | Loss: 2.5363 | LR: 1.62e-04 | 5170.0ms/step | 11.6 samples/s +Step 31330/50462 | Loss: 2.4753 | LR: 1.62e-04 | 5179.2ms/step | 11.6 samples/s +Step 31340/50462 | Loss: 2.5249 | LR: 1.61e-04 | 5180.8ms/step | 11.6 samples/s +Step 31350/50462 | Loss: 2.5435 | LR: 1.61e-04 | 5183.7ms/step | 11.6 samples/s +Step 31360/50462 | Loss: 2.5356 | LR: 1.61e-04 | 5158.8ms/step | 11.6 samples/s +Step 31370/50462 | Loss: 2.5131 | LR: 1.61e-04 | 5163.6ms/step | 11.6 samples/s +Step 31380/50462 | Loss: 2.4945 | LR: 1.61e-04 | 5152.3ms/step | 11.6 samples/s +Step 31390/50462 | Loss: 2.4865 | LR: 1.61e-04 | 5186.4ms/step | 11.6 samples/s +Step 31400/50462 | Loss: 2.5123 | LR: 1.60e-04 | 5149.0ms/step | 11.7 samples/s +Step 31410/50462 | Loss: 2.5225 | LR: 1.60e-04 | 5207.2ms/step | 11.5 samples/s +Step 31420/50462 | Loss: 2.5266 | LR: 1.60e-04 | 5208.4ms/step | 11.5 samples/s +Step 31430/50462 | Loss: 2.5275 | LR: 1.60e-04 | 5197.0ms/step | 11.5 samples/s +Step 31440/50462 | Loss: 2.5464 | LR: 1.60e-04 | 5183.0ms/step | 11.6 samples/s +Step 31450/50462 | Loss: 2.5116 | LR: 1.60e-04 | 5193.6ms/step | 11.6 samples/s +Step 31460/50462 | Loss: 2.5258 | LR: 1.60e-04 | 5174.1ms/step | 11.6 samples/s +Step 31470/50462 | Loss: 2.5020 | LR: 1.59e-04 | 5176.7ms/step | 11.6 samples/s +Step 31480/50462 | Loss: 2.5410 | LR: 1.59e-04 | 5189.8ms/step | 11.6 samples/s +Step 31490/50462 | Loss: 2.5492 | LR: 1.59e-04 | 5174.8ms/step | 11.6 samples/s +Step 31500/50462 | Loss: 2.5195 | LR: 1.59e-04 | 5167.4ms/step | 11.6 samples/s +Validation at step 31500: Loss = 2.5311, Perplexity = 12.57 +Step 31510/50462 | Loss: 2.5364 | LR: 1.59e-04 | 7686.8ms/step | 7.8 samples/s +Step 31520/50462 | Loss: 2.5651 | LR: 1.59e-04 | 5143.8ms/step | 11.7 samples/s +Step 31530/50462 | Loss: 2.4923 | LR: 1.59e-04 | 5179.7ms/step | 11.6 samples/s +Step 31540/50462 | Loss: 2.5079 | LR: 1.58e-04 | 5138.5ms/step | 11.7 samples/s +Step 31550/50462 | Loss: 2.5025 | LR: 1.58e-04 | 5130.6ms/step | 11.7 samples/s +Step 31560/50462 | Loss: 2.5342 | LR: 1.58e-04 | 5163.2ms/step | 11.6 samples/s +Step 31570/50462 | Loss: 2.5081 | LR: 1.58e-04 | 5236.6ms/step | 11.5 samples/s +Step 31580/50462 | Loss: 2.4892 | LR: 1.58e-04 | 5195.1ms/step | 11.5 samples/s +Step 31590/50462 | Loss: 2.5184 | LR: 1.58e-04 | 5180.2ms/step | 11.6 samples/s +Step 31600/50462 | Loss: 2.5122 | LR: 1.58e-04 | 5184.3ms/step | 11.6 samples/s +Step 31610/50462 | Loss: 2.4919 | LR: 1.57e-04 | 5180.8ms/step | 11.6 samples/s +Step 31620/50462 | Loss: 2.5419 | LR: 1.57e-04 | 5208.0ms/step | 11.5 samples/s +Step 31630/50462 | Loss: 2.5137 | LR: 1.57e-04 | 5184.4ms/step | 11.6 samples/s +Step 31640/50462 | Loss: 2.5366 | LR: 1.57e-04 | 5181.6ms/step | 11.6 samples/s +Step 31650/50462 | Loss: 2.5230 | LR: 1.57e-04 | 5186.9ms/step | 11.6 samples/s +Step 31660/50462 | Loss: 2.5023 | LR: 1.57e-04 | 5191.5ms/step | 11.6 samples/s +Step 31670/50462 | Loss: 2.5243 | LR: 1.57e-04 | 5188.6ms/step | 11.6 samples/s +Step 31680/50462 | Loss: 2.5273 | LR: 1.56e-04 | 5190.5ms/step | 11.6 samples/s +Step 31690/50462 | Loss: 2.5238 | LR: 1.56e-04 | 5196.1ms/step | 11.5 samples/s +Step 31700/50462 | Loss: 2.4788 | LR: 1.56e-04 | 5272.2ms/step | 11.4 samples/s +Step 31710/50462 | Loss: 2.4869 | LR: 1.56e-04 | 5182.7ms/step | 11.6 samples/s +Step 31720/50462 | Loss: 2.5322 | LR: 1.56e-04 | 5213.3ms/step | 11.5 samples/s +Step 31730/50462 | Loss: 2.5370 | LR: 1.56e-04 | 5214.0ms/step | 11.5 samples/s +Step 31740/50462 | Loss: 2.5316 | LR: 1.55e-04 | 5149.1ms/step | 11.7 samples/s +Step 31750/50462 | Loss: 2.5243 | LR: 1.55e-04 | 5179.4ms/step | 11.6 samples/s +Validation at step 31750: Loss = 2.5300, Perplexity = 12.55 +Step 31760/50462 | Loss: 2.5131 | LR: 1.55e-04 | 7745.3ms/step | 7.7 samples/s +Step 31770/50462 | Loss: 2.5351 | LR: 1.55e-04 | 5181.0ms/step | 11.6 samples/s +Step 31780/50462 | Loss: 2.5054 | LR: 1.55e-04 | 5235.9ms/step | 11.5 samples/s +Step 31790/50462 | Loss: 2.5284 | LR: 1.55e-04 | 5248.5ms/step | 11.4 samples/s +Step 31800/50462 | Loss: 2.5190 | LR: 1.55e-04 | 5244.3ms/step | 11.4 samples/s +Step 31810/50462 | Loss: 2.5110 | LR: 1.54e-04 | 5247.5ms/step | 11.4 samples/s +Step 31820/50462 | Loss: 2.5245 | LR: 1.54e-04 | 5247.2ms/step | 11.4 samples/s +Step 31830/50462 | Loss: 2.4988 | LR: 1.54e-04 | 5252.3ms/step | 11.4 samples/s +Step 31840/50462 | Loss: 2.5174 | LR: 1.54e-04 | 5300.0ms/step | 11.3 samples/s +Step 31850/50462 | Loss: 2.5112 | LR: 1.54e-04 | 5247.7ms/step | 11.4 samples/s +Step 31860/50462 | Loss: 2.5034 | LR: 1.54e-04 | 5247.1ms/step | 11.4 samples/s +Step 31870/50462 | Loss: 2.5191 | LR: 1.54e-04 | 5252.6ms/step | 11.4 samples/s +Step 31880/50462 | Loss: 2.5175 | LR: 1.53e-04 | 5247.7ms/step | 11.4 samples/s +Step 31890/50462 | Loss: 2.5180 | LR: 1.53e-04 | 5248.8ms/step | 11.4 samples/s +Step 31900/50462 | Loss: 2.5388 | LR: 1.53e-04 | 5246.4ms/step | 11.4 samples/s +Step 31910/50462 | Loss: 2.5209 | LR: 1.53e-04 | 5245.1ms/step | 11.4 samples/s +Step 31920/50462 | Loss: 2.4823 | LR: 1.53e-04 | 5225.7ms/step | 11.5 samples/s +Step 31930/50462 | Loss: 2.5027 | LR: 1.53e-04 | 5245.8ms/step | 11.4 samples/s +Step 31940/50462 | Loss: 2.5321 | LR: 1.53e-04 | 5237.4ms/step | 11.5 samples/s +Step 31950/50462 | Loss: 2.5368 | LR: 1.52e-04 | 5243.6ms/step | 11.4 samples/s +Step 31960/50462 | Loss: 2.5268 | LR: 1.52e-04 | 5230.3ms/step | 11.5 samples/s +Step 31970/50462 | Loss: 2.5172 | LR: 1.52e-04 | 5239.1ms/step | 11.5 samples/s +Step 31980/50462 | Loss: 2.5402 | LR: 1.52e-04 | 5239.1ms/step | 11.5 samples/s +Step 31990/50462 | Loss: 2.5394 | LR: 1.52e-04 | 5323.0ms/step | 11.3 samples/s +Step 32000/50462 | Loss: 2.5637 | LR: 1.52e-04 | 5222.8ms/step | 11.5 samples/s +Validation at step 32000: Loss = 2.5287, Perplexity = 12.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-32000 +Step 32010/50462 | Loss: 2.5463 | LR: 1.52e-04 | 8839.5ms/step | 6.8 samples/s +Step 32020/50462 | Loss: 2.4994 | LR: 1.51e-04 | 5222.8ms/step | 11.5 samples/s +Step 32030/50462 | Loss: 2.5023 | LR: 1.51e-04 | 5238.5ms/step | 11.5 samples/s +Step 32040/50462 | Loss: 2.5550 | LR: 1.51e-04 | 5233.0ms/step | 11.5 samples/s +Step 32050/50462 | Loss: 2.5180 | LR: 1.51e-04 | 5231.1ms/step | 11.5 samples/s +Step 32060/50462 | Loss: 2.5028 | LR: 1.51e-04 | 5169.5ms/step | 11.6 samples/s +Step 32070/50462 | Loss: 2.5213 | LR: 1.51e-04 | 5178.9ms/step | 11.6 samples/s +Step 32080/50462 | Loss: 2.5276 | LR: 1.51e-04 | 5121.7ms/step | 11.7 samples/s +Step 32090/50462 | Loss: 2.5138 | LR: 1.50e-04 | 5163.8ms/step | 11.6 samples/s +Step 32100/50462 | Loss: 2.5096 | LR: 1.50e-04 | 5161.4ms/step | 11.6 samples/s +Step 32110/50462 | Loss: 2.4952 | LR: 1.50e-04 | 5114.0ms/step | 11.7 samples/s +Step 32120/50462 | Loss: 2.5235 | LR: 1.50e-04 | 5200.2ms/step | 11.5 samples/s +Step 32130/50462 | Loss: 2.5104 | LR: 1.50e-04 | 5129.1ms/step | 11.7 samples/s +Step 32140/50462 | Loss: 2.5315 | LR: 1.50e-04 | 5165.2ms/step | 11.6 samples/s +Step 32150/50462 | Loss: 2.5504 | LR: 1.50e-04 | 5156.3ms/step | 11.6 samples/s +Step 32160/50462 | Loss: 2.5144 | LR: 1.49e-04 | 5159.3ms/step | 11.6 samples/s +Step 32170/50462 | Loss: 2.5136 | LR: 1.49e-04 | 5148.5ms/step | 11.7 samples/s +Step 32180/50462 | Loss: 2.5017 | LR: 1.49e-04 | 5155.2ms/step | 11.6 samples/s +Step 32190/50462 | Loss: 2.5285 | LR: 1.49e-04 | 5147.4ms/step | 11.7 samples/s +Step 32200/50462 | Loss: 2.5096 | LR: 1.49e-04 | 5151.6ms/step | 11.6 samples/s +Step 32210/50462 | Loss: 2.5297 | LR: 1.49e-04 | 5189.4ms/step | 11.6 samples/s +Step 32220/50462 | Loss: 2.5256 | LR: 1.49e-04 | 5119.8ms/step | 11.7 samples/s +Step 32230/50462 | Loss: 2.5163 | LR: 1.48e-04 | 5148.2ms/step | 11.7 samples/s +Step 32240/50462 | Loss: 2.5042 | LR: 1.48e-04 | 5134.8ms/step | 11.7 samples/s +Step 32250/50462 | Loss: 2.5185 | LR: 1.48e-04 | 5162.5ms/step | 11.6 samples/s +Validation at step 32250: Loss = 2.5280, Perplexity = 12.53 +Step 32260/50462 | Loss: 2.5099 | LR: 1.48e-04 | 7707.4ms/step | 7.8 samples/s +Step 32270/50462 | Loss: 2.5271 | LR: 1.48e-04 | 5197.8ms/step | 11.5 samples/s +Step 32280/50462 | Loss: 2.5143 | LR: 1.48e-04 | 5279.6ms/step | 11.4 samples/s +Step 32290/50462 | Loss: 2.5289 | LR: 1.48e-04 | 5181.2ms/step | 11.6 samples/s +Step 32300/50462 | Loss: 2.5351 | LR: 1.47e-04 | 5177.3ms/step | 11.6 samples/s +Step 32310/50462 | Loss: 2.5556 | LR: 1.47e-04 | 5182.8ms/step | 11.6 samples/s +Step 32320/50462 | Loss: 2.4933 | LR: 1.47e-04 | 5161.0ms/step | 11.6 samples/s +Step 32330/50462 | Loss: 2.5381 | LR: 1.47e-04 | 5145.2ms/step | 11.7 samples/s +Step 32340/50462 | Loss: 2.5673 | LR: 1.47e-04 | 5203.2ms/step | 11.5 samples/s +Step 32350/50462 | Loss: 2.5031 | LR: 1.47e-04 | 5143.6ms/step | 11.7 samples/s +Step 32360/50462 | Loss: 2.5165 | LR: 1.47e-04 | 5160.7ms/step | 11.6 samples/s +Step 32370/50462 | Loss: 2.5263 | LR: 1.46e-04 | 5160.3ms/step | 11.6 samples/s +Step 32380/50462 | Loss: 2.4771 | LR: 1.46e-04 | 5150.5ms/step | 11.6 samples/s +Step 32390/50462 | Loss: 2.5054 | LR: 1.46e-04 | 5156.6ms/step | 11.6 samples/s +Step 32400/50462 | Loss: 2.5357 | LR: 1.46e-04 | 5183.8ms/step | 11.6 samples/s +Step 32410/50462 | Loss: 2.5314 | LR: 1.46e-04 | 5127.5ms/step | 11.7 samples/s +Step 32420/50462 | Loss: 2.5080 | LR: 1.46e-04 | 5114.2ms/step | 11.7 samples/s +Step 32430/50462 | Loss: 2.5035 | LR: 1.45e-04 | 5111.2ms/step | 11.7 samples/s +Step 32440/50462 | Loss: 2.5093 | LR: 1.45e-04 | 5133.5ms/step | 11.7 samples/s +Step 32450/50462 | Loss: 2.5273 | LR: 1.45e-04 | 5131.1ms/step | 11.7 samples/s +Step 32460/50462 | Loss: 2.5062 | LR: 1.45e-04 | 5094.6ms/step | 11.8 samples/s +Step 32470/50462 | Loss: 2.5302 | LR: 1.45e-04 | 5148.0ms/step | 11.7 samples/s +Step 32480/50462 | Loss: 2.5346 | LR: 1.45e-04 | 5112.5ms/step | 11.7 samples/s +Step 32490/50462 | Loss: 2.5106 | LR: 1.45e-04 | 5123.5ms/step | 11.7 samples/s +Step 32500/50462 | Loss: 2.5590 | LR: 1.44e-04 | 5134.9ms/step | 11.7 samples/s +Validation at step 32500: Loss = 2.5271, Perplexity = 12.52 +Step 32510/50462 | Loss: 2.5133 | LR: 1.44e-04 | 7675.4ms/step | 7.8 samples/s +Step 32520/50462 | Loss: 2.5131 | LR: 1.44e-04 | 5195.0ms/step | 11.5 samples/s +Step 32530/50462 | Loss: 2.5175 | LR: 1.44e-04 | 5210.5ms/step | 11.5 samples/s +Step 32540/50462 | Loss: 2.5128 | LR: 1.44e-04 | 5229.5ms/step | 11.5 samples/s +Step 32550/50462 | Loss: 2.5159 | LR: 1.44e-04 | 5215.9ms/step | 11.5 samples/s +Step 32560/50462 | Loss: 2.4780 | LR: 1.44e-04 | 5260.6ms/step | 11.4 samples/s +Step 32570/50462 | Loss: 2.5034 | LR: 1.43e-04 | 5280.1ms/step | 11.4 samples/s +Step 32580/50462 | Loss: 2.4982 | LR: 1.43e-04 | 5227.8ms/step | 11.5 samples/s +Step 32590/50462 | Loss: 2.5108 | LR: 1.43e-04 | 5238.0ms/step | 11.5 samples/s +Step 32600/50462 | Loss: 2.5137 | LR: 1.43e-04 | 5238.3ms/step | 11.5 samples/s +Step 32610/50462 | Loss: 2.5067 | LR: 1.43e-04 | 5233.2ms/step | 11.5 samples/s +Step 32620/50462 | Loss: 2.5213 | LR: 1.43e-04 | 5230.5ms/step | 11.5 samples/s +Step 32630/50462 | Loss: 2.5056 | LR: 1.43e-04 | 5232.1ms/step | 11.5 samples/s +Step 32640/50462 | Loss: 2.5124 | LR: 1.42e-04 | 5234.4ms/step | 11.5 samples/s +Step 32650/50462 | Loss: 2.5124 | LR: 1.42e-04 | 5229.2ms/step | 11.5 samples/s +Step 32660/50462 | Loss: 2.5279 | LR: 1.42e-04 | 5231.9ms/step | 11.5 samples/s +Step 32670/50462 | Loss: 2.5308 | LR: 1.42e-04 | 5280.2ms/step | 11.4 samples/s +Step 32680/50462 | Loss: 2.5206 | LR: 1.42e-04 | 5249.2ms/step | 11.4 samples/s +Step 32690/50462 | Loss: 2.4884 | LR: 1.42e-04 | 5240.2ms/step | 11.4 samples/s +Step 32700/50462 | Loss: 2.4880 | LR: 1.42e-04 | 5253.9ms/step | 11.4 samples/s +Step 32710/50462 | Loss: 2.5360 | LR: 1.41e-04 | 5242.7ms/step | 11.4 samples/s +Step 32720/50462 | Loss: 2.5251 | LR: 1.41e-04 | 5246.4ms/step | 11.4 samples/s +Step 32730/50462 | Loss: 2.4999 | LR: 1.41e-04 | 5247.8ms/step | 11.4 samples/s +Step 32740/50462 | Loss: 2.5346 | LR: 1.41e-04 | 5239.2ms/step | 11.5 samples/s +Step 32750/50462 | Loss: 2.5235 | LR: 1.41e-04 | 5245.0ms/step | 11.4 samples/s +Validation at step 32750: Loss = 2.5262, Perplexity = 12.51 +Step 32760/50462 | Loss: 2.5318 | LR: 1.41e-04 | 7807.2ms/step | 7.7 samples/s +Step 32770/50462 | Loss: 2.5123 | LR: 1.41e-04 | 5238.3ms/step | 11.5 samples/s +Step 32780/50462 | Loss: 2.4972 | LR: 1.41e-04 | 5238.6ms/step | 11.5 samples/s +Step 32790/50462 | Loss: 2.5244 | LR: 1.40e-04 | 5238.3ms/step | 11.5 samples/s +Step 32800/50462 | Loss: 2.4823 | LR: 1.40e-04 | 5235.9ms/step | 11.5 samples/s +Step 32810/50462 | Loss: 2.5112 | LR: 1.40e-04 | 5237.2ms/step | 11.5 samples/s +Step 32820/50462 | Loss: 2.5279 | LR: 1.40e-04 | 5238.3ms/step | 11.5 samples/s +Step 32830/50462 | Loss: 2.5280 | LR: 1.40e-04 | 5245.2ms/step | 11.4 samples/s +Step 32840/50462 | Loss: 2.5416 | LR: 1.40e-04 | 5237.1ms/step | 11.5 samples/s +Step 32850/50462 | Loss: 2.5579 | LR: 1.40e-04 | 5279.9ms/step | 11.4 samples/s +Step 32860/50462 | Loss: 2.5221 | LR: 1.39e-04 | 5280.3ms/step | 11.4 samples/s +Step 32870/50462 | Loss: 2.5017 | LR: 1.39e-04 | 5234.2ms/step | 11.5 samples/s +Step 32880/50462 | Loss: 2.5054 | LR: 1.39e-04 | 5245.5ms/step | 11.4 samples/s +Step 32890/50462 | Loss: 2.5352 | LR: 1.39e-04 | 5242.1ms/step | 11.4 samples/s +Step 32900/50462 | Loss: 2.5169 | LR: 1.39e-04 | 5247.6ms/step | 11.4 samples/s +Step 32910/50462 | Loss: 2.5046 | LR: 1.39e-04 | 5251.6ms/step | 11.4 samples/s +Step 32920/50462 | Loss: 2.5207 | LR: 1.39e-04 | 5247.4ms/step | 11.4 samples/s +Step 32930/50462 | Loss: 2.5131 | LR: 1.38e-04 | 5244.4ms/step | 11.4 samples/s +Step 32940/50462 | Loss: 2.5083 | LR: 1.38e-04 | 5249.2ms/step | 11.4 samples/s +Step 32950/50462 | Loss: 2.4988 | LR: 1.38e-04 | 5282.4ms/step | 11.4 samples/s +Step 32960/50462 | Loss: 2.4840 | LR: 1.38e-04 | 5248.2ms/step | 11.4 samples/s +Step 32970/50462 | Loss: 2.5324 | LR: 1.38e-04 | 5244.4ms/step | 11.4 samples/s +Step 32980/50462 | Loss: 2.5002 | LR: 1.38e-04 | 5229.5ms/step | 11.5 samples/s +Step 32990/50462 | Loss: 2.4955 | LR: 1.38e-04 | 5243.5ms/step | 11.4 samples/s +Step 33000/50462 | Loss: 2.5095 | LR: 1.37e-04 | 5249.1ms/step | 11.4 samples/s +Validation at step 33000: Loss = 2.5252, Perplexity = 12.49 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-33000 +Step 33010/50462 | Loss: 2.5259 | LR: 1.37e-04 | 8719.0ms/step | 6.9 samples/s +Step 33020/50462 | Loss: 2.4987 | LR: 1.37e-04 | 5236.3ms/step | 11.5 samples/s +Step 33030/50462 | Loss: 2.5291 | LR: 1.37e-04 | 5246.4ms/step | 11.4 samples/s +Step 33040/50462 | Loss: 2.5190 | LR: 1.37e-04 | 5249.7ms/step | 11.4 samples/s +Step 33050/50462 | Loss: 2.4838 | LR: 1.37e-04 | 5245.1ms/step | 11.4 samples/s +Step 33060/50462 | Loss: 2.5148 | LR: 1.37e-04 | 5247.1ms/step | 11.4 samples/s +Step 33070/50462 | Loss: 2.5089 | LR: 1.36e-04 | 5246.9ms/step | 11.4 samples/s +Step 33080/50462 | Loss: 2.5126 | LR: 1.36e-04 | 5248.6ms/step | 11.4 samples/s +Step 33090/50462 | Loss: 2.5107 | LR: 1.36e-04 | 5239.5ms/step | 11.5 samples/s +Step 33100/50462 | Loss: 2.4977 | LR: 1.36e-04 | 5250.4ms/step | 11.4 samples/s +Step 33110/50462 | Loss: 2.4973 | LR: 1.36e-04 | 5241.3ms/step | 11.4 samples/s +Step 33120/50462 | Loss: 2.5063 | LR: 1.36e-04 | 5253.5ms/step | 11.4 samples/s +Step 33130/50462 | Loss: 2.5021 | LR: 1.36e-04 | 5242.6ms/step | 11.4 samples/s +Step 33140/50462 | Loss: 2.5122 | LR: 1.35e-04 | 5341.1ms/step | 11.2 samples/s +Step 33150/50462 | Loss: 2.5198 | LR: 1.35e-04 | 5250.6ms/step | 11.4 samples/s +Step 33160/50462 | Loss: 2.5411 | LR: 1.35e-04 | 5239.1ms/step | 11.5 samples/s +Step 33170/50462 | Loss: 2.5205 | LR: 1.35e-04 | 5242.0ms/step | 11.4 samples/s +Step 33180/50462 | Loss: 2.5044 | LR: 1.35e-04 | 5254.2ms/step | 11.4 samples/s +Step 33190/50462 | Loss: 2.4951 | LR: 1.35e-04 | 5243.0ms/step | 11.4 samples/s +Step 33200/50462 | Loss: 2.5175 | LR: 1.35e-04 | 5251.1ms/step | 11.4 samples/s +Step 33210/50462 | Loss: 2.5444 | LR: 1.34e-04 | 5247.8ms/step | 11.4 samples/s +Step 33220/50462 | Loss: 2.5202 | LR: 1.34e-04 | 5249.0ms/step | 11.4 samples/s +Step 33230/50462 | Loss: 2.5144 | LR: 1.34e-04 | 5281.8ms/step | 11.4 samples/s +Step 33240/50462 | Loss: 2.4664 | LR: 1.34e-04 | 5227.9ms/step | 11.5 samples/s +Step 33250/50462 | Loss: 2.5301 | LR: 1.34e-04 | 5244.4ms/step | 11.4 samples/s +Validation at step 33250: Loss = 2.5242, Perplexity = 12.48 +Step 33260/50462 | Loss: 2.4796 | LR: 1.34e-04 | 7806.2ms/step | 7.7 samples/s +Step 33270/50462 | Loss: 2.5266 | LR: 1.34e-04 | 5246.2ms/step | 11.4 samples/s +Step 33280/50462 | Loss: 2.5252 | LR: 1.33e-04 | 5251.8ms/step | 11.4 samples/s +Step 33290/50462 | Loss: 2.5087 | LR: 1.33e-04 | 5252.8ms/step | 11.4 samples/s +Step 33300/50462 | Loss: 2.5197 | LR: 1.33e-04 | 5238.6ms/step | 11.5 samples/s +Step 33310/50462 | Loss: 2.5364 | LR: 1.33e-04 | 5247.3ms/step | 11.4 samples/s +Step 33320/50462 | Loss: 2.5210 | LR: 1.33e-04 | 5247.1ms/step | 11.4 samples/s +Step 33330/50462 | Loss: 2.5436 | LR: 1.33e-04 | 5241.9ms/step | 11.4 samples/s +Step 33340/50462 | Loss: 2.5652 | LR: 1.33e-04 | 5232.8ms/step | 11.5 samples/s +Step 33350/50462 | Loss: 2.5172 | LR: 1.32e-04 | 5223.8ms/step | 11.5 samples/s +Step 33360/50462 | Loss: 2.4933 | LR: 1.32e-04 | 5222.7ms/step | 11.5 samples/s +Step 33370/50462 | Loss: 2.4872 | LR: 1.32e-04 | 5223.7ms/step | 11.5 samples/s +Step 33380/50462 | Loss: 2.5220 | LR: 1.32e-04 | 5227.1ms/step | 11.5 samples/s +Step 33390/50462 | Loss: 2.4954 | LR: 1.32e-04 | 5232.6ms/step | 11.5 samples/s +Step 33400/50462 | Loss: 2.4997 | LR: 1.32e-04 | 5216.8ms/step | 11.5 samples/s +Step 33410/50462 | Loss: 2.4978 | LR: 1.32e-04 | 5240.6ms/step | 11.4 samples/s +Step 33420/50462 | Loss: 2.5040 | LR: 1.32e-04 | 5236.7ms/step | 11.5 samples/s +Step 33430/50462 | Loss: 2.5176 | LR: 1.31e-04 | 5312.1ms/step | 11.3 samples/s +Step 33440/50462 | Loss: 2.5579 | LR: 1.31e-04 | 5225.2ms/step | 11.5 samples/s +Step 33450/50462 | Loss: 2.5244 | LR: 1.31e-04 | 5223.6ms/step | 11.5 samples/s +Step 33460/50462 | Loss: 2.5019 | LR: 1.31e-04 | 5225.4ms/step | 11.5 samples/s +Step 33470/50462 | Loss: 2.5027 | LR: 1.31e-04 | 5227.3ms/step | 11.5 samples/s +Step 33480/50462 | Loss: 2.4869 | LR: 1.31e-04 | 5224.4ms/step | 11.5 samples/s +Step 33490/50462 | Loss: 2.5263 | LR: 1.31e-04 | 5229.9ms/step | 11.5 samples/s +Step 33500/50462 | Loss: 2.5043 | LR: 1.30e-04 | 5279.8ms/step | 11.4 samples/s +Validation at step 33500: Loss = 2.5235, Perplexity = 12.47 +Step 33510/50462 | Loss: 2.5250 | LR: 1.30e-04 | 7803.7ms/step | 7.7 samples/s +Step 33520/50462 | Loss: 2.5078 | LR: 1.30e-04 | 5233.7ms/step | 11.5 samples/s +Step 33530/50462 | Loss: 2.4779 | LR: 1.30e-04 | 5235.8ms/step | 11.5 samples/s +Step 33540/50462 | Loss: 2.5124 | LR: 1.30e-04 | 5233.4ms/step | 11.5 samples/s +Step 33550/50462 | Loss: 2.5152 | LR: 1.30e-04 | 5233.8ms/step | 11.5 samples/s +Step 33560/50462 | Loss: 2.5007 | LR: 1.30e-04 | 5230.8ms/step | 11.5 samples/s +Step 33570/50462 | Loss: 2.4731 | LR: 1.29e-04 | 5239.0ms/step | 11.5 samples/s +Step 33580/50462 | Loss: 2.5119 | LR: 1.29e-04 | 5224.3ms/step | 11.5 samples/s +Step 33590/50462 | Loss: 2.5110 | LR: 1.29e-04 | 5234.0ms/step | 11.5 samples/s +Step 33600/50462 | Loss: 2.5049 | LR: 1.29e-04 | 5235.2ms/step | 11.5 samples/s +Step 33610/50462 | Loss: 2.5283 | LR: 1.29e-04 | 5245.3ms/step | 11.4 samples/s +Step 33620/50462 | Loss: 2.5121 | LR: 1.29e-04 | 5234.2ms/step | 11.5 samples/s +Step 33630/50462 | Loss: 2.4654 | LR: 1.29e-04 | 5252.7ms/step | 11.4 samples/s +Step 33640/50462 | Loss: 2.5145 | LR: 1.28e-04 | 5244.5ms/step | 11.4 samples/s +Step 33650/50462 | Loss: 2.5062 | LR: 1.28e-04 | 5246.0ms/step | 11.4 samples/s +Step 33660/50462 | Loss: 2.5060 | LR: 1.28e-04 | 5250.1ms/step | 11.4 samples/s +Step 33670/50462 | Loss: 2.4974 | LR: 1.28e-04 | 5234.0ms/step | 11.5 samples/s +Step 33680/50462 | Loss: 2.5220 | LR: 1.28e-04 | 5237.2ms/step | 11.5 samples/s +Step 33690/50462 | Loss: 2.5076 | LR: 1.28e-04 | 5246.3ms/step | 11.4 samples/s +Step 33700/50462 | Loss: 2.4916 | LR: 1.28e-04 | 5241.4ms/step | 11.4 samples/s +Step 33710/50462 | Loss: 2.5353 | LR: 1.27e-04 | 5244.5ms/step | 11.4 samples/s +Step 33720/50462 | Loss: 2.5187 | LR: 1.27e-04 | 5320.4ms/step | 11.3 samples/s +Step 33730/50462 | Loss: 2.4857 | LR: 1.27e-04 | 5238.6ms/step | 11.5 samples/s +Step 33740/50462 | Loss: 2.5049 | LR: 1.27e-04 | 5236.7ms/step | 11.5 samples/s +Step 33750/50462 | Loss: 2.5212 | LR: 1.27e-04 | 5250.3ms/step | 11.4 samples/s +Validation at step 33750: Loss = 2.5227, Perplexity = 12.46 +Step 33760/50462 | Loss: 2.4808 | LR: 1.27e-04 | 7814.1ms/step | 7.7 samples/s +Step 33770/50462 | Loss: 2.5041 | LR: 1.27e-04 | 5234.9ms/step | 11.5 samples/s +Step 33780/50462 | Loss: 2.5212 | LR: 1.27e-04 | 5278.7ms/step | 11.4 samples/s +Step 33790/50462 | Loss: 2.5229 | LR: 1.26e-04 | 5231.5ms/step | 11.5 samples/s +Step 33800/50462 | Loss: 2.5117 | LR: 1.26e-04 | 5235.9ms/step | 11.5 samples/s +Step 33810/50462 | Loss: 2.5183 | LR: 1.26e-04 | 5243.2ms/step | 11.4 samples/s +Step 33820/50462 | Loss: 2.4812 | LR: 1.26e-04 | 5244.7ms/step | 11.4 samples/s +Step 33830/50462 | Loss: 2.4768 | LR: 1.26e-04 | 5240.6ms/step | 11.4 samples/s +Step 33840/50462 | Loss: 2.5193 | LR: 1.26e-04 | 5233.6ms/step | 11.5 samples/s +Step 33850/50462 | Loss: 2.5054 | LR: 1.26e-04 | 5232.9ms/step | 11.5 samples/s +Step 33860/50462 | Loss: 2.5056 | LR: 1.25e-04 | 5253.3ms/step | 11.4 samples/s +Step 33870/50462 | Loss: 2.4909 | LR: 1.25e-04 | 5240.9ms/step | 11.4 samples/s +Step 33880/50462 | Loss: 2.5504 | LR: 1.25e-04 | 5240.3ms/step | 11.4 samples/s +Step 33890/50462 | Loss: 2.5153 | LR: 1.25e-04 | 5240.5ms/step | 11.4 samples/s +Step 33900/50462 | Loss: 2.5339 | LR: 1.25e-04 | 5240.7ms/step | 11.4 samples/s +Step 33910/50462 | Loss: 2.5031 | LR: 1.25e-04 | 5244.0ms/step | 11.4 samples/s +Step 33920/50462 | Loss: 2.5401 | LR: 1.25e-04 | 5246.1ms/step | 11.4 samples/s +Step 33930/50462 | Loss: 2.4884 | LR: 1.24e-04 | 5231.1ms/step | 11.5 samples/s +Step 33940/50462 | Loss: 2.5006 | LR: 1.24e-04 | 5218.7ms/step | 11.5 samples/s +Step 33950/50462 | Loss: 2.5113 | LR: 1.24e-04 | 5230.3ms/step | 11.5 samples/s +Step 33960/50462 | Loss: 2.5422 | LR: 1.24e-04 | 5249.3ms/step | 11.4 samples/s +Step 33970/50462 | Loss: 2.4752 | LR: 1.24e-04 | 5246.2ms/step | 11.4 samples/s +Step 33980/50462 | Loss: 2.5221 | LR: 1.24e-04 | 5237.5ms/step | 11.5 samples/s +Step 33990/50462 | Loss: 2.5138 | LR: 1.24e-04 | 5249.2ms/step | 11.4 samples/s +Step 34000/50462 | Loss: 2.4731 | LR: 1.24e-04 | 5286.7ms/step | 11.3 samples/s +Validation at step 34000: Loss = 2.5217, Perplexity = 12.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-34000 +Step 34010/50462 | Loss: 2.5026 | LR: 1.23e-04 | 8914.6ms/step | 6.7 samples/s +Step 34020/50462 | Loss: 2.5149 | LR: 1.23e-04 | 5235.4ms/step | 11.5 samples/s +Step 34030/50462 | Loss: 2.4842 | LR: 1.23e-04 | 5249.8ms/step | 11.4 samples/s +Step 34040/50462 | Loss: 2.5170 | LR: 1.23e-04 | 5249.6ms/step | 11.4 samples/s +Step 34050/50462 | Loss: 2.5289 | LR: 1.23e-04 | 5253.2ms/step | 11.4 samples/s +Step 34060/50462 | Loss: 2.5171 | LR: 1.23e-04 | 5292.8ms/step | 11.3 samples/s +Step 34070/50462 | Loss: 2.5049 | LR: 1.23e-04 | 5242.9ms/step | 11.4 samples/s +Step 34080/50462 | Loss: 2.5302 | LR: 1.22e-04 | 5250.4ms/step | 11.4 samples/s +Step 34090/50462 | Loss: 2.5270 | LR: 1.22e-04 | 5240.4ms/step | 11.4 samples/s +Step 34100/50462 | Loss: 2.5299 | LR: 1.22e-04 | 5236.2ms/step | 11.5 samples/s +Step 34110/50462 | Loss: 2.5134 | LR: 1.22e-04 | 5248.5ms/step | 11.4 samples/s +Step 34120/50462 | Loss: 2.5075 | LR: 1.22e-04 | 5245.5ms/step | 11.4 samples/s +Step 34130/50462 | Loss: 2.4899 | LR: 1.22e-04 | 5234.8ms/step | 11.5 samples/s +Step 34140/50462 | Loss: 2.5348 | LR: 1.22e-04 | 5236.7ms/step | 11.5 samples/s +Step 34150/50462 | Loss: 2.4878 | LR: 1.21e-04 | 5235.2ms/step | 11.5 samples/s +Step 34160/50462 | Loss: 2.5197 | LR: 1.21e-04 | 5246.9ms/step | 11.4 samples/s +Step 34170/50462 | Loss: 2.4960 | LR: 1.21e-04 | 5244.6ms/step | 11.4 samples/s +Step 34180/50462 | Loss: 2.5096 | LR: 1.21e-04 | 5237.0ms/step | 11.5 samples/s +Step 34190/50462 | Loss: 2.5008 | LR: 1.21e-04 | 5240.1ms/step | 11.5 samples/s +Step 34200/50462 | Loss: 2.4966 | LR: 1.21e-04 | 5246.1ms/step | 11.4 samples/s +Step 34210/50462 | Loss: 2.5079 | LR: 1.21e-04 | 5247.0ms/step | 11.4 samples/s +Step 34220/50462 | Loss: 2.5111 | LR: 1.21e-04 | 5241.1ms/step | 11.4 samples/s +Step 34230/50462 | Loss: 2.5210 | LR: 1.20e-04 | 5245.1ms/step | 11.4 samples/s +Step 34240/50462 | Loss: 2.4918 | LR: 1.20e-04 | 5241.2ms/step | 11.4 samples/s +Step 34250/50462 | Loss: 2.5011 | LR: 1.20e-04 | 5244.6ms/step | 11.4 samples/s +Validation at step 34250: Loss = 2.5209, Perplexity = 12.44 +Step 34260/50462 | Loss: 2.4599 | LR: 1.20e-04 | 7805.1ms/step | 7.7 samples/s +Step 34270/50462 | Loss: 2.5231 | LR: 1.20e-04 | 5236.9ms/step | 11.5 samples/s +Step 34280/50462 | Loss: 2.5064 | LR: 1.20e-04 | 5234.2ms/step | 11.5 samples/s +Step 34290/50462 | Loss: 2.5296 | LR: 1.20e-04 | 5271.4ms/step | 11.4 samples/s +Step 34300/50462 | Loss: 2.5268 | LR: 1.19e-04 | 5277.6ms/step | 11.4 samples/s +Step 34310/50462 | Loss: 2.5404 | LR: 1.19e-04 | 5245.6ms/step | 11.4 samples/s +Step 34320/50462 | Loss: 2.4890 | LR: 1.19e-04 | 5238.0ms/step | 11.5 samples/s +Step 34330/50462 | Loss: 2.4970 | LR: 1.19e-04 | 5252.2ms/step | 11.4 samples/s +Step 34340/50462 | Loss: 2.4885 | LR: 1.19e-04 | 5284.7ms/step | 11.4 samples/s +Step 34350/50462 | Loss: 2.5027 | LR: 1.19e-04 | 5239.1ms/step | 11.5 samples/s +Step 34360/50462 | Loss: 2.5130 | LR: 1.19e-04 | 5239.8ms/step | 11.5 samples/s +Step 34370/50462 | Loss: 2.5009 | LR: 1.19e-04 | 5248.8ms/step | 11.4 samples/s +Step 34380/50462 | Loss: 2.4820 | LR: 1.18e-04 | 5244.7ms/step | 11.4 samples/s +Step 34390/50462 | Loss: 2.5071 | LR: 1.18e-04 | 5244.7ms/step | 11.4 samples/s +Step 34400/50462 | Loss: 2.5162 | LR: 1.18e-04 | 5232.6ms/step | 11.5 samples/s +Step 34410/50462 | Loss: 2.5036 | LR: 1.18e-04 | 5232.1ms/step | 11.5 samples/s +Step 34420/50462 | Loss: 2.4941 | LR: 1.18e-04 | 5238.5ms/step | 11.5 samples/s +Step 34430/50462 | Loss: 2.5140 | LR: 1.18e-04 | 5239.5ms/step | 11.5 samples/s +Step 34440/50462 | Loss: 2.4959 | LR: 1.18e-04 | 5244.2ms/step | 11.4 samples/s +Step 34450/50462 | Loss: 2.5311 | LR: 1.17e-04 | 5246.3ms/step | 11.4 samples/s +Step 34460/50462 | Loss: 2.4911 | LR: 1.17e-04 | 5246.1ms/step | 11.4 samples/s +Step 34470/50462 | Loss: 2.5300 | LR: 1.17e-04 | 5238.8ms/step | 11.5 samples/s +Step 34480/50462 | Loss: 2.5333 | LR: 1.17e-04 | 5235.1ms/step | 11.5 samples/s +Step 34490/50462 | Loss: 2.5025 | LR: 1.17e-04 | 5237.1ms/step | 11.5 samples/s +Step 34500/50462 | Loss: 2.5137 | LR: 1.17e-04 | 5250.8ms/step | 11.4 samples/s +Validation at step 34500: Loss = 2.5203, Perplexity = 12.43 +Step 34510/50462 | Loss: 2.5103 | LR: 1.17e-04 | 7776.3ms/step | 7.7 samples/s +Step 34520/50462 | Loss: 2.4936 | LR: 1.16e-04 | 5209.2ms/step | 11.5 samples/s +Step 34530/50462 | Loss: 2.4998 | LR: 1.16e-04 | 5215.8ms/step | 11.5 samples/s +Step 34540/50462 | Loss: 2.5023 | LR: 1.16e-04 | 5246.6ms/step | 11.4 samples/s +Step 34550/50462 | Loss: 2.4850 | LR: 1.16e-04 | 5251.8ms/step | 11.4 samples/s +Step 34560/50462 | Loss: 2.5225 | LR: 1.16e-04 | 5246.4ms/step | 11.4 samples/s +Step 34570/50462 | Loss: 2.4923 | LR: 1.16e-04 | 5247.6ms/step | 11.4 samples/s +Step 34580/50462 | Loss: 2.5060 | LR: 1.16e-04 | 5273.4ms/step | 11.4 samples/s +Step 34590/50462 | Loss: 2.5105 | LR: 1.16e-04 | 5258.7ms/step | 11.4 samples/s +Step 34600/50462 | Loss: 2.4991 | LR: 1.15e-04 | 5218.8ms/step | 11.5 samples/s +Step 34610/50462 | Loss: 2.5089 | LR: 1.15e-04 | 5289.3ms/step | 11.3 samples/s +Step 34620/50462 | Loss: 2.5154 | LR: 1.15e-04 | 5245.8ms/step | 11.4 samples/s +Step 34630/50462 | Loss: 2.5022 | LR: 1.15e-04 | 5250.6ms/step | 11.4 samples/s +Step 34640/50462 | Loss: 2.5327 | LR: 1.15e-04 | 5251.1ms/step | 11.4 samples/s +Step 34650/50462 | Loss: 2.5179 | LR: 1.15e-04 | 5250.1ms/step | 11.4 samples/s +Step 34660/50462 | Loss: 2.5177 | LR: 1.15e-04 | 5253.4ms/step | 11.4 samples/s +Step 34670/50462 | Loss: 2.5330 | LR: 1.15e-04 | 5249.1ms/step | 11.4 samples/s +Step 34680/50462 | Loss: 2.5215 | LR: 1.14e-04 | 5245.4ms/step | 11.4 samples/s +Step 34690/50462 | Loss: 2.5190 | LR: 1.14e-04 | 5235.7ms/step | 11.5 samples/s +Step 34700/50462 | Loss: 2.5173 | LR: 1.14e-04 | 5239.4ms/step | 11.5 samples/s +Step 34710/50462 | Loss: 2.5081 | LR: 1.14e-04 | 5232.2ms/step | 11.5 samples/s +Step 34720/50462 | Loss: 2.5037 | LR: 1.14e-04 | 5246.6ms/step | 11.4 samples/s +Step 34730/50462 | Loss: 2.5049 | LR: 1.14e-04 | 5243.4ms/step | 11.4 samples/s +Step 34740/50462 | Loss: 2.5174 | LR: 1.14e-04 | 5252.8ms/step | 11.4 samples/s +Step 34750/50462 | Loss: 2.5182 | LR: 1.13e-04 | 5250.5ms/step | 11.4 samples/s +Validation at step 34750: Loss = 2.5195, Perplexity = 12.42 +Step 34760/50462 | Loss: 2.5064 | LR: 1.13e-04 | 7808.4ms/step | 7.7 samples/s +Step 34770/50462 | Loss: 2.4881 | LR: 1.13e-04 | 5248.5ms/step | 11.4 samples/s +Step 34780/50462 | Loss: 2.5040 | LR: 1.13e-04 | 5249.6ms/step | 11.4 samples/s +Step 34790/50462 | Loss: 2.4954 | LR: 1.13e-04 | 5250.1ms/step | 11.4 samples/s +Step 34800/50462 | Loss: 2.5096 | LR: 1.13e-04 | 5232.0ms/step | 11.5 samples/s +Step 34810/50462 | Loss: 2.5244 | LR: 1.13e-04 | 5248.6ms/step | 11.4 samples/s +Step 34820/50462 | Loss: 2.5167 | LR: 1.13e-04 | 5248.6ms/step | 11.4 samples/s +Step 34830/50462 | Loss: 2.5096 | LR: 1.12e-04 | 5241.1ms/step | 11.4 samples/s +Step 34840/50462 | Loss: 2.5223 | LR: 1.12e-04 | 5245.2ms/step | 11.4 samples/s +Step 34850/50462 | Loss: 2.5427 | LR: 1.12e-04 | 5249.6ms/step | 11.4 samples/s +Step 34860/50462 | Loss: 2.5240 | LR: 1.12e-04 | 5250.2ms/step | 11.4 samples/s +Step 34870/50462 | Loss: 2.4878 | LR: 1.12e-04 | 5327.5ms/step | 11.3 samples/s +Step 34880/50462 | Loss: 2.5111 | LR: 1.12e-04 | 5237.6ms/step | 11.5 samples/s +Step 34890/50462 | Loss: 2.4991 | LR: 1.12e-04 | 5277.8ms/step | 11.4 samples/s +Step 34900/50462 | Loss: 2.5094 | LR: 1.11e-04 | 5246.4ms/step | 11.4 samples/s +Step 34910/50462 | Loss: 2.5170 | LR: 1.11e-04 | 5242.5ms/step | 11.4 samples/s +Step 34920/50462 | Loss: 2.4548 | LR: 1.11e-04 | 5237.7ms/step | 11.5 samples/s +Step 34930/50462 | Loss: 2.4928 | LR: 1.11e-04 | 5247.4ms/step | 11.4 samples/s +Step 34940/50462 | Loss: 2.5082 | LR: 1.11e-04 | 5251.4ms/step | 11.4 samples/s +Step 34950/50462 | Loss: 2.5114 | LR: 1.11e-04 | 5244.8ms/step | 11.4 samples/s +Step 34960/50462 | Loss: 2.5125 | LR: 1.11e-04 | 5249.8ms/step | 11.4 samples/s +Step 34970/50462 | Loss: 2.4906 | LR: 1.11e-04 | 5245.0ms/step | 11.4 samples/s +Step 34980/50462 | Loss: 2.4957 | LR: 1.10e-04 | 5253.7ms/step | 11.4 samples/s +Step 34990/50462 | Loss: 2.5261 | LR: 1.10e-04 | 5240.1ms/step | 11.5 samples/s +Step 35000/50462 | Loss: 2.5361 | LR: 1.10e-04 | 5246.0ms/step | 11.4 samples/s +Validation at step 35000: Loss = 2.5189, Perplexity = 12.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-35000 +Step 35010/50462 | Loss: 2.5168 | LR: 1.10e-04 | 8772.6ms/step | 6.8 samples/s +Step 35020/50462 | Loss: 2.4990 | LR: 1.10e-04 | 5235.8ms/step | 11.5 samples/s +Step 35030/50462 | Loss: 2.5112 | LR: 1.10e-04 | 5236.8ms/step | 11.5 samples/s +Step 35040/50462 | Loss: 2.4965 | LR: 1.10e-04 | 5252.6ms/step | 11.4 samples/s +Step 35050/50462 | Loss: 2.4979 | LR: 1.09e-04 | 5251.3ms/step | 11.4 samples/s +Step 35060/50462 | Loss: 2.4910 | LR: 1.09e-04 | 5257.4ms/step | 11.4 samples/s +Step 35070/50462 | Loss: 2.4880 | LR: 1.09e-04 | 5245.9ms/step | 11.4 samples/s +Step 35080/50462 | Loss: 2.5255 | LR: 1.09e-04 | 5252.9ms/step | 11.4 samples/s +Step 35090/50462 | Loss: 2.4815 | LR: 1.09e-04 | 5245.7ms/step | 11.4 samples/s +Step 35100/50462 | Loss: 2.5140 | LR: 1.09e-04 | 5253.3ms/step | 11.4 samples/s +Step 35110/50462 | Loss: 2.5145 | LR: 1.09e-04 | 5250.8ms/step | 11.4 samples/s +Step 35120/50462 | Loss: 2.4971 | LR: 1.09e-04 | 5237.8ms/step | 11.5 samples/s +Step 35130/50462 | Loss: 2.4764 | LR: 1.08e-04 | 5242.2ms/step | 11.4 samples/s +Step 35140/50462 | Loss: 2.4866 | LR: 1.08e-04 | 5251.5ms/step | 11.4 samples/s +Step 35150/50462 | Loss: 2.5052 | LR: 1.08e-04 | 5247.4ms/step | 11.4 samples/s +Step 35160/50462 | Loss: 2.5172 | LR: 1.08e-04 | 5326.0ms/step | 11.3 samples/s +Step 35170/50462 | Loss: 2.5108 | LR: 1.08e-04 | 5271.7ms/step | 11.4 samples/s +Step 35180/50462 | Loss: 2.5023 | LR: 1.08e-04 | 5237.4ms/step | 11.5 samples/s +Step 35190/50462 | Loss: 2.5142 | LR: 1.08e-04 | 5235.7ms/step | 11.5 samples/s +Step 35200/50462 | Loss: 2.5063 | LR: 1.08e-04 | 5251.5ms/step | 11.4 samples/s +Step 35210/50462 | Loss: 2.5034 | LR: 1.07e-04 | 5245.2ms/step | 11.4 samples/s +Step 35220/50462 | Loss: 2.5084 | LR: 1.07e-04 | 5255.8ms/step | 11.4 samples/s +Step 35230/50462 | Loss: 2.5177 | LR: 1.07e-04 | 5252.9ms/step | 11.4 samples/s +Step 35240/50462 | Loss: 2.5102 | LR: 1.07e-04 | 5237.7ms/step | 11.5 samples/s +Step 35250/50462 | Loss: 2.4854 | LR: 1.07e-04 | 5255.9ms/step | 11.4 samples/s +Validation at step 35250: Loss = 2.5182, Perplexity = 12.41 +Step 35260/50462 | Loss: 2.4661 | LR: 1.07e-04 | 7811.6ms/step | 7.7 samples/s +Step 35270/50462 | Loss: 2.4925 | LR: 1.07e-04 | 5244.2ms/step | 11.4 samples/s +Step 35280/50462 | Loss: 2.4956 | LR: 1.07e-04 | 5249.5ms/step | 11.4 samples/s +Step 35290/50462 | Loss: 2.4785 | LR: 1.06e-04 | 5247.4ms/step | 11.4 samples/s +Step 35300/50462 | Loss: 2.4966 | LR: 1.06e-04 | 5246.1ms/step | 11.4 samples/s +Step 35310/50462 | Loss: 2.5216 | LR: 1.06e-04 | 5245.0ms/step | 11.4 samples/s +Step 35320/50462 | Loss: 2.5127 | LR: 1.06e-04 | 5244.6ms/step | 11.4 samples/s +Step 35330/50462 | Loss: 2.5257 | LR: 1.06e-04 | 5250.4ms/step | 11.4 samples/s +Step 35340/50462 | Loss: 2.5051 | LR: 1.06e-04 | 5249.4ms/step | 11.4 samples/s +Step 35350/50462 | Loss: 2.5155 | LR: 1.06e-04 | 5243.2ms/step | 11.4 samples/s +Step 35360/50462 | Loss: 2.5061 | LR: 1.05e-04 | 5246.2ms/step | 11.4 samples/s +Step 35370/50462 | Loss: 2.5183 | LR: 1.05e-04 | 5249.3ms/step | 11.4 samples/s +Step 35380/50462 | Loss: 2.5076 | LR: 1.05e-04 | 5250.1ms/step | 11.4 samples/s +Step 35390/50462 | Loss: 2.5102 | LR: 1.05e-04 | 5239.8ms/step | 11.5 samples/s +Step 35400/50462 | Loss: 2.5316 | LR: 1.05e-04 | 5261.4ms/step | 11.4 samples/s +Step 35410/50462 | Loss: 2.5110 | LR: 1.05e-04 | 5238.6ms/step | 11.5 samples/s +Step 35420/50462 | Loss: 2.5075 | LR: 1.05e-04 | 5239.0ms/step | 11.5 samples/s +Step 35430/50462 | Loss: 2.5062 | LR: 1.05e-04 | 5243.1ms/step | 11.4 samples/s +Step 35440/50462 | Loss: 2.5077 | LR: 1.04e-04 | 5244.9ms/step | 11.4 samples/s +Step 35450/50462 | Loss: 2.5035 | LR: 1.04e-04 | 5358.2ms/step | 11.2 samples/s +Step 35460/50462 | Loss: 2.5076 | LR: 1.04e-04 | 5239.5ms/step | 11.5 samples/s +Step 35470/50462 | Loss: 2.5009 | LR: 1.04e-04 | 5243.0ms/step | 11.4 samples/s +Step 35480/50462 | Loss: 2.4932 | LR: 1.04e-04 | 5247.9ms/step | 11.4 samples/s +Step 35490/50462 | Loss: 2.5061 | LR: 1.04e-04 | 5234.5ms/step | 11.5 samples/s +Step 35500/50462 | Loss: 2.5260 | LR: 1.04e-04 | 5256.3ms/step | 11.4 samples/s +Validation at step 35500: Loss = 2.5178, Perplexity = 12.40 +Step 35510/50462 | Loss: 2.5106 | LR: 1.04e-04 | 7808.2ms/step | 7.7 samples/s +Step 35520/50462 | Loss: 2.4913 | LR: 1.03e-04 | 5252.8ms/step | 11.4 samples/s +Step 35530/50462 | Loss: 2.5081 | LR: 1.03e-04 | 5245.2ms/step | 11.4 samples/s +Step 35540/50462 | Loss: 2.4664 | LR: 1.03e-04 | 5249.0ms/step | 11.4 samples/s +Step 35550/50462 | Loss: 2.5111 | LR: 1.03e-04 | 5245.6ms/step | 11.4 samples/s +Step 35560/50462 | Loss: 2.5134 | LR: 1.03e-04 | 5243.0ms/step | 11.4 samples/s +Step 35570/50462 | Loss: 2.5211 | LR: 1.03e-04 | 5244.7ms/step | 11.4 samples/s +Step 35580/50462 | Loss: 2.4970 | LR: 1.03e-04 | 5240.4ms/step | 11.4 samples/s +Step 35590/50462 | Loss: 2.5213 | LR: 1.03e-04 | 5243.8ms/step | 11.4 samples/s +Step 35600/50462 | Loss: 2.4738 | LR: 1.02e-04 | 5240.8ms/step | 11.4 samples/s +Step 35610/50462 | Loss: 2.4906 | LR: 1.02e-04 | 5245.6ms/step | 11.4 samples/s +Step 35620/50462 | Loss: 2.4971 | LR: 1.02e-04 | 5243.5ms/step | 11.4 samples/s +Step 35630/50462 | Loss: 2.5115 | LR: 1.02e-04 | 5248.3ms/step | 11.4 samples/s +Step 35640/50462 | Loss: 2.4822 | LR: 1.02e-04 | 5246.6ms/step | 11.4 samples/s +Step 35650/50462 | Loss: 2.5060 | LR: 1.02e-04 | 5239.0ms/step | 11.5 samples/s +Step 35660/50462 | Loss: 2.5020 | LR: 1.02e-04 | 5242.9ms/step | 11.4 samples/s +Step 35670/50462 | Loss: 2.4850 | LR: 1.02e-04 | 5248.3ms/step | 11.4 samples/s +Step 35680/50462 | Loss: 2.5228 | LR: 1.01e-04 | 5241.2ms/step | 11.4 samples/s +Step 35690/50462 | Loss: 2.4886 | LR: 1.01e-04 | 5248.8ms/step | 11.4 samples/s +Step 35700/50462 | Loss: 2.4992 | LR: 1.01e-04 | 5242.2ms/step | 11.4 samples/s +Step 35710/50462 | Loss: 2.5335 | LR: 1.01e-04 | 5231.8ms/step | 11.5 samples/s +Step 35720/50462 | Loss: 2.4705 | LR: 1.01e-04 | 5243.5ms/step | 11.4 samples/s +Step 35730/50462 | Loss: 2.5321 | LR: 1.01e-04 | 5313.5ms/step | 11.3 samples/s +Step 35740/50462 | Loss: 2.4857 | LR: 1.01e-04 | 5287.6ms/step | 11.3 samples/s +Step 35750/50462 | Loss: 2.5049 | LR: 1.00e-04 | 5248.3ms/step | 11.4 samples/s +Validation at step 35750: Loss = 2.5171, Perplexity = 12.39 +Step 35760/50462 | Loss: 2.5003 | LR: 1.00e-04 | 7815.1ms/step | 7.7 samples/s +Step 35770/50462 | Loss: 2.5062 | LR: 1.00e-04 | 5244.6ms/step | 11.4 samples/s +Step 35780/50462 | Loss: 2.4843 | LR: 1.00e-04 | 5247.9ms/step | 11.4 samples/s +Step 35790/50462 | Loss: 2.5054 | LR: 1.00e-04 | 5247.4ms/step | 11.4 samples/s +Step 35800/50462 | Loss: 2.4933 | LR: 9.99e-05 | 5239.9ms/step | 11.5 samples/s +Step 35810/50462 | Loss: 2.5155 | LR: 9.97e-05 | 5240.3ms/step | 11.4 samples/s +Step 35820/50462 | Loss: 2.5170 | LR: 9.96e-05 | 5240.1ms/step | 11.5 samples/s +Step 35830/50462 | Loss: 2.4979 | LR: 9.95e-05 | 5237.2ms/step | 11.5 samples/s +Step 35840/50462 | Loss: 2.5038 | LR: 9.94e-05 | 5234.4ms/step | 11.5 samples/s +Step 35850/50462 | Loss: 2.4955 | LR: 9.92e-05 | 5238.3ms/step | 11.5 samples/s +Step 35860/50462 | Loss: 2.4950 | LR: 9.91e-05 | 5242.2ms/step | 11.4 samples/s +Step 35870/50462 | Loss: 2.5080 | LR: 9.90e-05 | 5233.5ms/step | 11.5 samples/s +Step 35880/50462 | Loss: 2.5166 | LR: 9.88e-05 | 5228.7ms/step | 11.5 samples/s +Step 35890/50462 | Loss: 2.5105 | LR: 9.87e-05 | 5235.0ms/step | 11.5 samples/s +Step 35900/50462 | Loss: 2.4896 | LR: 9.86e-05 | 5245.1ms/step | 11.4 samples/s +Step 35910/50462 | Loss: 2.5245 | LR: 9.85e-05 | 5227.4ms/step | 11.5 samples/s +Step 35920/50462 | Loss: 2.5192 | LR: 9.83e-05 | 5237.4ms/step | 11.5 samples/s +Step 35930/50462 | Loss: 2.4965 | LR: 9.82e-05 | 5235.4ms/step | 11.5 samples/s +Step 35940/50462 | Loss: 2.4954 | LR: 9.81e-05 | 5251.0ms/step | 11.4 samples/s +Step 35950/50462 | Loss: 2.4783 | LR: 9.80e-05 | 5247.0ms/step | 11.4 samples/s +Step 35960/50462 | Loss: 2.5025 | LR: 9.78e-05 | 5237.2ms/step | 11.5 samples/s +Step 35970/50462 | Loss: 2.4951 | LR: 9.77e-05 | 5244.8ms/step | 11.4 samples/s +Step 35980/50462 | Loss: 2.4911 | LR: 9.76e-05 | 5249.9ms/step | 11.4 samples/s +Step 35990/50462 | Loss: 2.4846 | LR: 9.75e-05 | 5240.8ms/step | 11.4 samples/s +Step 36000/50462 | Loss: 2.4836 | LR: 9.73e-05 | 5286.5ms/step | 11.3 samples/s +Validation at step 36000: Loss = 2.5166, Perplexity = 12.39 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-36000 +Step 36010/50462 | Loss: 2.4801 | LR: 9.72e-05 | 8748.0ms/step | 6.9 samples/s +Step 36020/50462 | Loss: 2.5062 | LR: 9.71e-05 | 5271.1ms/step | 11.4 samples/s +Step 36030/50462 | Loss: 2.4829 | LR: 9.70e-05 | 5277.8ms/step | 11.4 samples/s +Step 36040/50462 | Loss: 2.5369 | LR: 9.68e-05 | 5248.6ms/step | 11.4 samples/s +Step 36050/50462 | Loss: 2.5072 | LR: 9.67e-05 | 5249.6ms/step | 11.4 samples/s +Step 36060/50462 | Loss: 2.5153 | LR: 9.66e-05 | 5243.3ms/step | 11.4 samples/s +Step 36070/50462 | Loss: 2.4932 | LR: 9.65e-05 | 5239.6ms/step | 11.5 samples/s +Step 36080/50462 | Loss: 2.5109 | LR: 9.63e-05 | 5245.0ms/step | 11.4 samples/s +Step 36090/50462 | Loss: 2.4893 | LR: 9.62e-05 | 5246.7ms/step | 11.4 samples/s +Step 36100/50462 | Loss: 2.5221 | LR: 9.61e-05 | 5241.6ms/step | 11.4 samples/s +Step 36110/50462 | Loss: 2.4814 | LR: 9.60e-05 | 5248.7ms/step | 11.4 samples/s +Step 36120/50462 | Loss: 2.5114 | LR: 9.58e-05 | 5236.9ms/step | 11.5 samples/s +Step 36130/50462 | Loss: 2.5138 | LR: 9.57e-05 | 5254.0ms/step | 11.4 samples/s +Step 36140/50462 | Loss: 2.5131 | LR: 9.56e-05 | 5239.6ms/step | 11.5 samples/s +Step 36150/50462 | Loss: 2.4859 | LR: 9.55e-05 | 5244.3ms/step | 11.4 samples/s +Step 36160/50462 | Loss: 2.4871 | LR: 9.53e-05 | 5248.6ms/step | 11.4 samples/s +Step 36170/50462 | Loss: 2.5040 | LR: 9.52e-05 | 5246.6ms/step | 11.4 samples/s +Step 36180/50462 | Loss: 2.5017 | LR: 9.51e-05 | 5233.6ms/step | 11.5 samples/s +Step 36190/50462 | Loss: 2.5116 | LR: 9.50e-05 | 5249.2ms/step | 11.4 samples/s +Step 36200/50462 | Loss: 2.5291 | LR: 9.49e-05 | 5241.3ms/step | 11.4 samples/s +Step 36210/50462 | Loss: 2.5203 | LR: 9.47e-05 | 5243.3ms/step | 11.4 samples/s +Step 36220/50462 | Loss: 2.4923 | LR: 9.46e-05 | 5250.9ms/step | 11.4 samples/s +Step 36230/50462 | Loss: 2.5066 | LR: 9.45e-05 | 5239.4ms/step | 11.5 samples/s +Step 36240/50462 | Loss: 2.5250 | LR: 9.44e-05 | 5240.6ms/step | 11.4 samples/s +Step 36250/50462 | Loss: 2.4920 | LR: 9.42e-05 | 5245.2ms/step | 11.4 samples/s +Validation at step 36250: Loss = 2.5162, Perplexity = 12.38 +Step 36260/50462 | Loss: 2.5111 | LR: 9.41e-05 | 7820.0ms/step | 7.7 samples/s +Step 36270/50462 | Loss: 2.4860 | LR: 9.40e-05 | 5236.4ms/step | 11.5 samples/s +Step 36280/50462 | Loss: 2.5071 | LR: 9.39e-05 | 5283.8ms/step | 11.4 samples/s +Step 36290/50462 | Loss: 2.4738 | LR: 9.37e-05 | 5235.2ms/step | 11.5 samples/s +Step 36300/50462 | Loss: 2.4630 | LR: 9.36e-05 | 5248.3ms/step | 11.4 samples/s +Step 36310/50462 | Loss: 2.4902 | LR: 9.35e-05 | 5288.8ms/step | 11.3 samples/s +Step 36320/50462 | Loss: 2.4975 | LR: 9.34e-05 | 5294.6ms/step | 11.3 samples/s +Step 36330/50462 | Loss: 2.5111 | LR: 9.32e-05 | 5256.4ms/step | 11.4 samples/s +Step 36340/50462 | Loss: 2.5102 | LR: 9.31e-05 | 5248.1ms/step | 11.4 samples/s +Step 36350/50462 | Loss: 2.5063 | LR: 9.30e-05 | 5232.2ms/step | 11.5 samples/s +Step 36360/50462 | Loss: 2.5130 | LR: 9.29e-05 | 5248.3ms/step | 11.4 samples/s +Step 36370/50462 | Loss: 2.5450 | LR: 9.28e-05 | 5246.8ms/step | 11.4 samples/s +Step 36380/50462 | Loss: 2.4822 | LR: 9.26e-05 | 5243.1ms/step | 11.4 samples/s +Step 36390/50462 | Loss: 2.4839 | LR: 9.25e-05 | 5236.4ms/step | 11.5 samples/s +Step 36400/50462 | Loss: 2.5187 | LR: 9.24e-05 | 5241.0ms/step | 11.4 samples/s +Step 36410/50462 | Loss: 2.4904 | LR: 9.23e-05 | 5248.1ms/step | 11.4 samples/s +Step 36420/50462 | Loss: 2.5220 | LR: 9.21e-05 | 5238.6ms/step | 11.5 samples/s +Step 36430/50462 | Loss: 2.5309 | LR: 9.20e-05 | 5245.5ms/step | 11.4 samples/s +Step 36440/50462 | Loss: 2.4834 | LR: 9.19e-05 | 5240.9ms/step | 11.4 samples/s +Step 36450/50462 | Loss: 2.4850 | LR: 9.18e-05 | 5242.2ms/step | 11.4 samples/s +Step 36460/50462 | Loss: 2.5090 | LR: 9.17e-05 | 5235.8ms/step | 11.5 samples/s +Step 36470/50462 | Loss: 2.5067 | LR: 9.15e-05 | 5242.7ms/step | 11.4 samples/s +Step 36480/50462 | Loss: 2.5047 | LR: 9.14e-05 | 5241.4ms/step | 11.4 samples/s +Step 36490/50462 | Loss: 2.5276 | LR: 9.13e-05 | 5222.2ms/step | 11.5 samples/s +Step 36500/50462 | Loss: 2.5263 | LR: 9.12e-05 | 5240.4ms/step | 11.4 samples/s +Validation at step 36500: Loss = 2.5155, Perplexity = 12.37 +Step 36510/50462 | Loss: 2.4924 | LR: 9.10e-05 | 7805.1ms/step | 7.7 samples/s +Step 36520/50462 | Loss: 2.4883 | LR: 9.09e-05 | 5249.9ms/step | 11.4 samples/s +Step 36530/50462 | Loss: 2.5390 | LR: 9.08e-05 | 5225.5ms/step | 11.5 samples/s +Step 36540/50462 | Loss: 2.5009 | LR: 9.07e-05 | 5245.4ms/step | 11.4 samples/s +Step 36550/50462 | Loss: 2.5029 | LR: 9.06e-05 | 5241.7ms/step | 11.4 samples/s +Step 36560/50462 | Loss: 2.4948 | LR: 9.04e-05 | 5295.1ms/step | 11.3 samples/s +Step 36570/50462 | Loss: 2.4888 | LR: 9.03e-05 | 5242.4ms/step | 11.4 samples/s +Step 36580/50462 | Loss: 2.4999 | LR: 9.02e-05 | 5245.4ms/step | 11.4 samples/s +Step 36590/50462 | Loss: 2.5176 | LR: 9.01e-05 | 5244.4ms/step | 11.4 samples/s +Step 36600/50462 | Loss: 2.4788 | LR: 8.99e-05 | 5287.4ms/step | 11.3 samples/s +Step 36610/50462 | Loss: 2.4898 | LR: 8.98e-05 | 5283.2ms/step | 11.4 samples/s +Step 36620/50462 | Loss: 2.4934 | LR: 8.97e-05 | 5255.5ms/step | 11.4 samples/s +Step 36630/50462 | Loss: 2.4833 | LR: 8.96e-05 | 5244.6ms/step | 11.4 samples/s +Step 36640/50462 | Loss: 2.5229 | LR: 8.95e-05 | 5237.9ms/step | 11.5 samples/s +Step 36650/50462 | Loss: 2.4826 | LR: 8.93e-05 | 5249.7ms/step | 11.4 samples/s +Step 36660/50462 | Loss: 2.5178 | LR: 8.92e-05 | 5237.6ms/step | 11.5 samples/s +Step 36670/50462 | Loss: 2.4847 | LR: 8.91e-05 | 5248.5ms/step | 11.4 samples/s +Step 36680/50462 | Loss: 2.4908 | LR: 8.90e-05 | 5258.5ms/step | 11.4 samples/s +Step 36690/50462 | Loss: 2.5344 | LR: 8.89e-05 | 5242.7ms/step | 11.4 samples/s +Step 36700/50462 | Loss: 2.4991 | LR: 8.87e-05 | 5240.5ms/step | 11.4 samples/s +Step 36710/50462 | Loss: 2.4826 | LR: 8.86e-05 | 5253.5ms/step | 11.4 samples/s +Step 36720/50462 | Loss: 2.4901 | LR: 8.85e-05 | 5254.5ms/step | 11.4 samples/s +Step 36730/50462 | Loss: 2.5311 | LR: 8.84e-05 | 5245.8ms/step | 11.4 samples/s +Step 36740/50462 | Loss: 2.5065 | LR: 8.83e-05 | 5251.7ms/step | 11.4 samples/s +Step 36750/50462 | Loss: 2.5052 | LR: 8.81e-05 | 5249.4ms/step | 11.4 samples/s +Validation at step 36750: Loss = 2.5148, Perplexity = 12.36 +Step 36760/50462 | Loss: 2.4822 | LR: 8.80e-05 | 7810.2ms/step | 7.7 samples/s +Step 36770/50462 | Loss: 2.5017 | LR: 8.79e-05 | 5244.1ms/step | 11.4 samples/s +Step 36780/50462 | Loss: 2.5304 | LR: 8.78e-05 | 5251.1ms/step | 11.4 samples/s +Step 36790/50462 | Loss: 2.4870 | LR: 8.77e-05 | 5251.5ms/step | 11.4 samples/s +Step 36800/50462 | Loss: 2.4930 | LR: 8.75e-05 | 5248.4ms/step | 11.4 samples/s +Step 36810/50462 | Loss: 2.5160 | LR: 8.74e-05 | 5250.0ms/step | 11.4 samples/s +Step 36820/50462 | Loss: 2.5185 | LR: 8.73e-05 | 5250.6ms/step | 11.4 samples/s +Step 36830/50462 | Loss: 2.5192 | LR: 8.72e-05 | 5295.5ms/step | 11.3 samples/s +Step 36840/50462 | Loss: 2.5157 | LR: 8.71e-05 | 5239.5ms/step | 11.5 samples/s +Step 36850/50462 | Loss: 2.5135 | LR: 8.69e-05 | 5250.9ms/step | 11.4 samples/s +Step 36860/50462 | Loss: 2.5221 | LR: 8.68e-05 | 5256.2ms/step | 11.4 samples/s +Step 36870/50462 | Loss: 2.4895 | LR: 8.67e-05 | 5238.3ms/step | 11.5 samples/s +Step 36880/50462 | Loss: 2.4869 | LR: 8.66e-05 | 5242.3ms/step | 11.4 samples/s +Step 36890/50462 | Loss: 2.4726 | LR: 8.65e-05 | 5309.5ms/step | 11.3 samples/s +Step 36900/50462 | Loss: 2.4813 | LR: 8.63e-05 | 5237.9ms/step | 11.5 samples/s +Step 36910/50462 | Loss: 2.5243 | LR: 8.62e-05 | 5244.1ms/step | 11.4 samples/s +Step 36920/50462 | Loss: 2.5027 | LR: 8.61e-05 | 5241.8ms/step | 11.4 samples/s +Step 36930/50462 | Loss: 2.5009 | LR: 8.60e-05 | 5241.2ms/step | 11.4 samples/s +Step 36940/50462 | Loss: 2.5039 | LR: 8.59e-05 | 5226.2ms/step | 11.5 samples/s +Step 36950/50462 | Loss: 2.5065 | LR: 8.57e-05 | 5235.8ms/step | 11.5 samples/s +Step 36960/50462 | Loss: 2.4853 | LR: 8.56e-05 | 5239.5ms/step | 11.5 samples/s +Step 36970/50462 | Loss: 2.4898 | LR: 8.55e-05 | 5227.0ms/step | 11.5 samples/s +Step 36980/50462 | Loss: 2.4673 | LR: 8.54e-05 | 5254.7ms/step | 11.4 samples/s +Step 36990/50462 | Loss: 2.4687 | LR: 8.53e-05 | 5246.3ms/step | 11.4 samples/s +Step 37000/50462 | Loss: 2.4911 | LR: 8.51e-05 | 5235.3ms/step | 11.5 samples/s +Validation at step 37000: Loss = 2.5144, Perplexity = 12.36 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-37000 +Step 37010/50462 | Loss: 2.5301 | LR: 8.50e-05 | 8723.0ms/step | 6.9 samples/s +Step 37020/50462 | Loss: 2.5014 | LR: 8.49e-05 | 5209.4ms/step | 11.5 samples/s +Step 37030/50462 | Loss: 2.4884 | LR: 8.48e-05 | 5220.3ms/step | 11.5 samples/s +Step 37040/50462 | Loss: 2.4588 | LR: 8.47e-05 | 5217.1ms/step | 11.5 samples/s +Step 37050/50462 | Loss: 2.5247 | LR: 8.46e-05 | 5237.0ms/step | 11.5 samples/s +Step 37060/50462 | Loss: 2.4900 | LR: 8.44e-05 | 5225.7ms/step | 11.5 samples/s +Step 37070/50462 | Loss: 2.4868 | LR: 8.43e-05 | 5237.9ms/step | 11.5 samples/s +Step 37080/50462 | Loss: 2.5312 | LR: 8.42e-05 | 5237.6ms/step | 11.5 samples/s +Step 37090/50462 | Loss: 2.4998 | LR: 8.41e-05 | 5232.8ms/step | 11.5 samples/s +Step 37100/50462 | Loss: 2.5017 | LR: 8.40e-05 | 5229.6ms/step | 11.5 samples/s +Step 37110/50462 | Loss: 2.4980 | LR: 8.38e-05 | 5285.3ms/step | 11.4 samples/s +Step 37120/50462 | Loss: 2.4839 | LR: 8.37e-05 | 5242.5ms/step | 11.4 samples/s +Step 37130/50462 | Loss: 2.5124 | LR: 8.36e-05 | 5242.4ms/step | 11.4 samples/s +Step 37140/50462 | Loss: 2.4942 | LR: 8.35e-05 | 5238.7ms/step | 11.5 samples/s +Step 37150/50462 | Loss: 2.5307 | LR: 8.34e-05 | 5234.0ms/step | 11.5 samples/s +Step 37160/50462 | Loss: 2.5078 | LR: 8.33e-05 | 5245.1ms/step | 11.4 samples/s +Step 37170/50462 | Loss: 2.5148 | LR: 8.31e-05 | 5240.0ms/step | 11.5 samples/s +Step 37180/50462 | Loss: 2.4887 | LR: 8.30e-05 | 5327.3ms/step | 11.3 samples/s +Step 37190/50462 | Loss: 2.5036 | LR: 8.29e-05 | 5235.5ms/step | 11.5 samples/s +Step 37200/50462 | Loss: 2.5047 | LR: 8.28e-05 | 5234.9ms/step | 11.5 samples/s +Step 37210/50462 | Loss: 2.4570 | LR: 8.27e-05 | 5247.0ms/step | 11.4 samples/s +Step 37220/50462 | Loss: 2.5005 | LR: 8.25e-05 | 5242.4ms/step | 11.4 samples/s +Step 37230/50462 | Loss: 2.4896 | LR: 8.24e-05 | 5238.7ms/step | 11.5 samples/s +Step 37240/50462 | Loss: 2.4902 | LR: 8.23e-05 | 5247.3ms/step | 11.4 samples/s +Step 37250/50462 | Loss: 2.4968 | LR: 8.22e-05 | 5244.4ms/step | 11.4 samples/s +Validation at step 37250: Loss = 2.5138, Perplexity = 12.35 +Step 37260/50462 | Loss: 2.5023 | LR: 8.21e-05 | 7809.8ms/step | 7.7 samples/s +Step 37270/50462 | Loss: 2.5171 | LR: 8.20e-05 | 5244.7ms/step | 11.4 samples/s +Step 37280/50462 | Loss: 2.5146 | LR: 8.18e-05 | 5246.2ms/step | 11.4 samples/s +Step 37290/50462 | Loss: 2.4902 | LR: 8.17e-05 | 5249.3ms/step | 11.4 samples/s +Step 37300/50462 | Loss: 2.5047 | LR: 8.16e-05 | 5246.5ms/step | 11.4 samples/s +Step 37310/50462 | Loss: 2.4837 | LR: 8.15e-05 | 5253.3ms/step | 11.4 samples/s +Step 37320/50462 | Loss: 2.5056 | LR: 8.14e-05 | 5244.4ms/step | 11.4 samples/s +Step 37330/50462 | Loss: 2.5109 | LR: 8.13e-05 | 5241.3ms/step | 11.4 samples/s +Step 37340/50462 | Loss: 2.4688 | LR: 8.11e-05 | 5249.0ms/step | 11.4 samples/s +Step 37350/50462 | Loss: 2.5129 | LR: 8.10e-05 | 5240.7ms/step | 11.4 samples/s +Step 37360/50462 | Loss: 2.5181 | LR: 8.09e-05 | 5241.9ms/step | 11.4 samples/s +Step 37370/50462 | Loss: 2.4883 | LR: 8.08e-05 | 5249.0ms/step | 11.4 samples/s +Step 37380/50462 | Loss: 2.5042 | LR: 8.07e-05 | 5253.7ms/step | 11.4 samples/s +Step 37390/50462 | Loss: 2.5135 | LR: 8.06e-05 | 5297.2ms/step | 11.3 samples/s +Step 37400/50462 | Loss: 2.5065 | LR: 8.04e-05 | 5248.1ms/step | 11.4 samples/s +Step 37410/50462 | Loss: 2.5012 | LR: 8.03e-05 | 5239.4ms/step | 11.5 samples/s +Step 37420/50462 | Loss: 2.5069 | LR: 8.02e-05 | 5241.6ms/step | 11.4 samples/s +Step 37430/50462 | Loss: 2.5102 | LR: 8.01e-05 | 5243.8ms/step | 11.4 samples/s +Step 37440/50462 | Loss: 2.4948 | LR: 8.00e-05 | 5244.7ms/step | 11.4 samples/s +Step 37450/50462 | Loss: 2.5309 | LR: 7.99e-05 | 5255.1ms/step | 11.4 samples/s +Step 37460/50462 | Loss: 2.4953 | LR: 7.98e-05 | 5288.3ms/step | 11.3 samples/s +Step 37470/50462 | Loss: 2.5176 | LR: 7.96e-05 | 5273.3ms/step | 11.4 samples/s +Step 37480/50462 | Loss: 2.4939 | LR: 7.95e-05 | 5230.7ms/step | 11.5 samples/s +Step 37490/50462 | Loss: 2.5097 | LR: 7.94e-05 | 5226.3ms/step | 11.5 samples/s +Step 37500/50462 | Loss: 2.4939 | LR: 7.93e-05 | 5235.9ms/step | 11.5 samples/s +Validation at step 37500: Loss = 2.5134, Perplexity = 12.35 +Step 37510/50462 | Loss: 2.5046 | LR: 7.92e-05 | 7816.7ms/step | 7.7 samples/s +Step 37520/50462 | Loss: 2.4823 | LR: 7.91e-05 | 5243.3ms/step | 11.4 samples/s +Step 37530/50462 | Loss: 2.5028 | LR: 7.89e-05 | 5241.3ms/step | 11.4 samples/s +Step 37540/50462 | Loss: 2.4992 | LR: 7.88e-05 | 5238.7ms/step | 11.5 samples/s +Step 37550/50462 | Loss: 2.5193 | LR: 7.87e-05 | 5225.4ms/step | 11.5 samples/s +Step 37560/50462 | Loss: 2.4932 | LR: 7.86e-05 | 5229.3ms/step | 11.5 samples/s +Step 37570/50462 | Loss: 2.5059 | LR: 7.85e-05 | 5228.2ms/step | 11.5 samples/s +Step 37580/50462 | Loss: 2.4966 | LR: 7.84e-05 | 5243.8ms/step | 11.4 samples/s +Step 37590/50462 | Loss: 2.4995 | LR: 7.83e-05 | 5247.1ms/step | 11.4 samples/s +Step 37600/50462 | Loss: 2.5011 | LR: 7.81e-05 | 5250.4ms/step | 11.4 samples/s +Step 37610/50462 | Loss: 2.5087 | LR: 7.80e-05 | 5246.0ms/step | 11.4 samples/s +Step 37620/50462 | Loss: 2.5123 | LR: 7.79e-05 | 5252.9ms/step | 11.4 samples/s +Step 37630/50462 | Loss: 2.4869 | LR: 7.78e-05 | 5243.1ms/step | 11.4 samples/s +Step 37640/50462 | Loss: 2.4982 | LR: 7.77e-05 | 5247.8ms/step | 11.4 samples/s +Step 37650/50462 | Loss: 2.4988 | LR: 7.76e-05 | 5250.1ms/step | 11.4 samples/s +Step 37660/50462 | Loss: 2.5044 | LR: 7.75e-05 | 5303.2ms/step | 11.3 samples/s +Step 37670/50462 | Loss: 2.4730 | LR: 7.73e-05 | 5238.0ms/step | 11.5 samples/s +Step 37680/50462 | Loss: 2.5010 | LR: 7.72e-05 | 5250.8ms/step | 11.4 samples/s +Step 37690/50462 | Loss: 2.5187 | LR: 7.71e-05 | 5254.9ms/step | 11.4 samples/s +Step 37700/50462 | Loss: 2.4886 | LR: 7.70e-05 | 5243.2ms/step | 11.4 samples/s +Step 37710/50462 | Loss: 2.5274 | LR: 7.69e-05 | 5251.9ms/step | 11.4 samples/s +Step 37720/50462 | Loss: 2.4631 | LR: 7.68e-05 | 5248.4ms/step | 11.4 samples/s +Step 37730/50462 | Loss: 2.5106 | LR: 7.67e-05 | 5250.0ms/step | 11.4 samples/s +Step 37740/50462 | Loss: 2.5229 | LR: 7.65e-05 | 5246.5ms/step | 11.4 samples/s +Step 37750/50462 | Loss: 2.4777 | LR: 7.64e-05 | 5290.5ms/step | 11.3 samples/s +Validation at step 37750: Loss = 2.5131, Perplexity = 12.34 +Step 37760/50462 | Loss: 2.5043 | LR: 7.63e-05 | 7855.5ms/step | 7.6 samples/s +Step 37770/50462 | Loss: 2.5061 | LR: 7.62e-05 | 5247.6ms/step | 11.4 samples/s +Step 37780/50462 | Loss: 2.4817 | LR: 7.61e-05 | 5249.1ms/step | 11.4 samples/s +Step 37790/50462 | Loss: 2.4642 | LR: 7.60e-05 | 5249.1ms/step | 11.4 samples/s +Step 37800/50462 | Loss: 2.4812 | LR: 7.59e-05 | 5251.2ms/step | 11.4 samples/s +Step 37810/50462 | Loss: 2.4940 | LR: 7.57e-05 | 5246.0ms/step | 11.4 samples/s +Step 37820/50462 | Loss: 2.5098 | LR: 7.56e-05 | 5244.4ms/step | 11.4 samples/s +Step 37830/50462 | Loss: 2.5117 | LR: 7.55e-05 | 5243.9ms/step | 11.4 samples/s +Step 37840/50462 | Loss: 2.5189 | LR: 7.54e-05 | 5249.0ms/step | 11.4 samples/s +Step 37850/50462 | Loss: 2.5177 | LR: 7.53e-05 | 5249.7ms/step | 11.4 samples/s +Step 37860/50462 | Loss: 2.5210 | LR: 7.52e-05 | 5259.0ms/step | 11.4 samples/s +Step 37870/50462 | Loss: 2.4959 | LR: 7.51e-05 | 5251.2ms/step | 11.4 samples/s +Step 37880/50462 | Loss: 2.5274 | LR: 7.50e-05 | 5246.6ms/step | 11.4 samples/s +Step 37890/50462 | Loss: 2.5062 | LR: 7.48e-05 | 5255.0ms/step | 11.4 samples/s +Step 37900/50462 | Loss: 2.4765 | LR: 7.47e-05 | 5251.2ms/step | 11.4 samples/s +Step 37910/50462 | Loss: 2.5044 | LR: 7.46e-05 | 5247.9ms/step | 11.4 samples/s +Step 37920/50462 | Loss: 2.4748 | LR: 7.45e-05 | 5254.3ms/step | 11.4 samples/s +Step 37930/50462 | Loss: 2.4846 | LR: 7.44e-05 | 5256.4ms/step | 11.4 samples/s +Step 37940/50462 | Loss: 2.4975 | LR: 7.43e-05 | 5292.9ms/step | 11.3 samples/s +Step 37950/50462 | Loss: 2.5117 | LR: 7.42e-05 | 5246.5ms/step | 11.4 samples/s +Step 37960/50462 | Loss: 2.5135 | LR: 7.41e-05 | 5249.2ms/step | 11.4 samples/s +Step 37970/50462 | Loss: 2.5168 | LR: 7.39e-05 | 5256.8ms/step | 11.4 samples/s +Step 37980/50462 | Loss: 2.4841 | LR: 7.38e-05 | 5249.6ms/step | 11.4 samples/s +Step 37990/50462 | Loss: 2.4739 | LR: 7.37e-05 | 5257.4ms/step | 11.4 samples/s +Step 38000/50462 | Loss: 2.4971 | LR: 7.36e-05 | 5248.1ms/step | 11.4 samples/s +Validation at step 38000: Loss = 2.5126, Perplexity = 12.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-38000 +Step 38010/50462 | Loss: 2.5011 | LR: 7.35e-05 | 8799.1ms/step | 6.8 samples/s +Step 38020/50462 | Loss: 2.4915 | LR: 7.34e-05 | 5254.2ms/step | 11.4 samples/s +Step 38030/50462 | Loss: 2.5175 | LR: 7.33e-05 | 5248.2ms/step | 11.4 samples/s +Step 38040/50462 | Loss: 2.4849 | LR: 7.32e-05 | 5285.0ms/step | 11.4 samples/s +Step 38050/50462 | Loss: 2.5055 | LR: 7.30e-05 | 5281.5ms/step | 11.4 samples/s +Step 38060/50462 | Loss: 2.5009 | LR: 7.29e-05 | 5249.5ms/step | 11.4 samples/s +Step 38070/50462 | Loss: 2.5411 | LR: 7.28e-05 | 5255.1ms/step | 11.4 samples/s +Step 38080/50462 | Loss: 2.4994 | LR: 7.27e-05 | 5248.9ms/step | 11.4 samples/s +Step 38090/50462 | Loss: 2.4986 | LR: 7.26e-05 | 5250.9ms/step | 11.4 samples/s +Step 38100/50462 | Loss: 2.5216 | LR: 7.25e-05 | 5247.8ms/step | 11.4 samples/s +Step 38110/50462 | Loss: 2.4554 | LR: 7.24e-05 | 5240.7ms/step | 11.4 samples/s +Step 38120/50462 | Loss: 2.5071 | LR: 7.23e-05 | 5239.8ms/step | 11.5 samples/s +Step 38130/50462 | Loss: 2.5051 | LR: 7.22e-05 | 5242.4ms/step | 11.4 samples/s +Step 38140/50462 | Loss: 2.5084 | LR: 7.20e-05 | 5248.0ms/step | 11.4 samples/s +Step 38150/50462 | Loss: 2.5280 | LR: 7.19e-05 | 5246.2ms/step | 11.4 samples/s +Step 38160/50462 | Loss: 2.4841 | LR: 7.18e-05 | 5250.5ms/step | 11.4 samples/s +Step 38170/50462 | Loss: 2.5145 | LR: 7.17e-05 | 5242.9ms/step | 11.4 samples/s +Step 38180/50462 | Loss: 2.5104 | LR: 7.16e-05 | 5256.0ms/step | 11.4 samples/s +Step 38190/50462 | Loss: 2.4967 | LR: 7.15e-05 | 5242.3ms/step | 11.4 samples/s +Step 38200/50462 | Loss: 2.5318 | LR: 7.14e-05 | 5260.7ms/step | 11.4 samples/s +Step 38210/50462 | Loss: 2.5218 | LR: 7.13e-05 | 5235.5ms/step | 11.5 samples/s +Step 38220/50462 | Loss: 2.4912 | LR: 7.12e-05 | 5290.6ms/step | 11.3 samples/s +Step 38230/50462 | Loss: 2.5065 | LR: 7.10e-05 | 5237.9ms/step | 11.5 samples/s +Step 38240/50462 | Loss: 2.5053 | LR: 7.09e-05 | 5246.6ms/step | 11.4 samples/s +Step 38250/50462 | Loss: 2.5091 | LR: 7.08e-05 | 5258.1ms/step | 11.4 samples/s +Validation at step 38250: Loss = 2.5122, Perplexity = 12.33 +Step 38260/50462 | Loss: 2.4919 | LR: 7.07e-05 | 7822.7ms/step | 7.7 samples/s +Step 38270/50462 | Loss: 2.5290 | LR: 7.06e-05 | 5251.2ms/step | 11.4 samples/s +Step 38280/50462 | Loss: 2.4717 | LR: 7.05e-05 | 5244.9ms/step | 11.4 samples/s +Step 38290/50462 | Loss: 2.4936 | LR: 7.04e-05 | 5250.5ms/step | 11.4 samples/s +Step 38300/50462 | Loss: 2.5015 | LR: 7.03e-05 | 5242.9ms/step | 11.4 samples/s +Step 38310/50462 | Loss: 2.5279 | LR: 7.02e-05 | 5252.0ms/step | 11.4 samples/s +Step 38320/50462 | Loss: 2.5341 | LR: 7.01e-05 | 5244.1ms/step | 11.4 samples/s +Step 38330/50462 | Loss: 2.4925 | LR: 6.99e-05 | 5285.3ms/step | 11.4 samples/s +Step 38340/50462 | Loss: 2.4944 | LR: 6.98e-05 | 5289.7ms/step | 11.3 samples/s +Step 38350/50462 | Loss: 2.5147 | LR: 6.97e-05 | 5200.7ms/step | 11.5 samples/s +Step 38360/50462 | Loss: 2.5115 | LR: 6.96e-05 | 5223.3ms/step | 11.5 samples/s +Step 38370/50462 | Loss: 2.5079 | LR: 6.95e-05 | 5222.5ms/step | 11.5 samples/s +Step 38380/50462 | Loss: 2.4883 | LR: 6.94e-05 | 5232.0ms/step | 11.5 samples/s +Step 38390/50462 | Loss: 2.5050 | LR: 6.93e-05 | 5240.9ms/step | 11.4 samples/s +Step 38400/50462 | Loss: 2.4987 | LR: 6.92e-05 | 5167.1ms/step | 11.6 samples/s +Step 38410/50462 | Loss: 2.5020 | LR: 6.91e-05 | 5238.6ms/step | 11.5 samples/s +Step 38420/50462 | Loss: 2.5158 | LR: 6.90e-05 | 5246.3ms/step | 11.4 samples/s +Step 38430/50462 | Loss: 2.5004 | LR: 6.89e-05 | 5250.5ms/step | 11.4 samples/s +Step 38440/50462 | Loss: 2.4902 | LR: 6.87e-05 | 5251.6ms/step | 11.4 samples/s +Step 38450/50462 | Loss: 2.5118 | LR: 6.86e-05 | 5169.5ms/step | 11.6 samples/s +Step 38460/50462 | Loss: 2.5000 | LR: 6.85e-05 | 5211.0ms/step | 11.5 samples/s +Step 38470/50462 | Loss: 2.4992 | LR: 6.84e-05 | 5242.5ms/step | 11.4 samples/s +Step 38480/50462 | Loss: 2.4879 | LR: 6.83e-05 | 5239.8ms/step | 11.5 samples/s +Step 38490/50462 | Loss: 2.5251 | LR: 6.82e-05 | 5298.6ms/step | 11.3 samples/s +Step 38500/50462 | Loss: 2.5037 | LR: 6.81e-05 | 5231.7ms/step | 11.5 samples/s +Validation at step 38500: Loss = 2.5120, Perplexity = 12.33 +Step 38510/50462 | Loss: 2.4996 | LR: 6.80e-05 | 7794.4ms/step | 7.7 samples/s +Step 38520/50462 | Loss: 2.4885 | LR: 6.79e-05 | 5246.2ms/step | 11.4 samples/s +Step 38530/50462 | Loss: 2.4932 | LR: 6.78e-05 | 5247.3ms/step | 11.4 samples/s +Step 38540/50462 | Loss: 2.4729 | LR: 6.77e-05 | 5255.5ms/step | 11.4 samples/s +Step 38550/50462 | Loss: 2.5092 | LR: 6.76e-05 | 5243.6ms/step | 11.4 samples/s +Step 38560/50462 | Loss: 2.4824 | LR: 6.74e-05 | 5238.3ms/step | 11.5 samples/s +Step 38570/50462 | Loss: 2.4865 | LR: 6.73e-05 | 5251.7ms/step | 11.4 samples/s +Step 38580/50462 | Loss: 2.4977 | LR: 6.72e-05 | 5253.2ms/step | 11.4 samples/s +Step 38590/50462 | Loss: 2.4975 | LR: 6.71e-05 | 5251.7ms/step | 11.4 samples/s +Step 38600/50462 | Loss: 2.5036 | LR: 6.70e-05 | 5243.1ms/step | 11.4 samples/s +Step 38610/50462 | Loss: 2.4785 | LR: 6.69e-05 | 5237.8ms/step | 11.5 samples/s +Step 38620/50462 | Loss: 2.4860 | LR: 6.68e-05 | 5314.2ms/step | 11.3 samples/s +Step 38630/50462 | Loss: 2.5111 | LR: 6.67e-05 | 5244.9ms/step | 11.4 samples/s +Step 38640/50462 | Loss: 2.4875 | LR: 6.66e-05 | 5242.3ms/step | 11.4 samples/s +Step 38650/50462 | Loss: 2.5166 | LR: 6.65e-05 | 5193.7ms/step | 11.6 samples/s +Step 38660/50462 | Loss: 2.4848 | LR: 6.64e-05 | 5218.9ms/step | 11.5 samples/s +Step 38670/50462 | Loss: 2.4919 | LR: 6.63e-05 | 5227.5ms/step | 11.5 samples/s +Step 38680/50462 | Loss: 2.4834 | LR: 6.62e-05 | 5239.3ms/step | 11.5 samples/s +Step 38690/50462 | Loss: 2.4611 | LR: 6.60e-05 | 5249.1ms/step | 11.4 samples/s +Step 38700/50462 | Loss: 2.5135 | LR: 6.59e-05 | 5243.8ms/step | 11.4 samples/s +Step 38710/50462 | Loss: 2.4876 | LR: 6.58e-05 | 5223.1ms/step | 11.5 samples/s +Step 38720/50462 | Loss: 2.4881 | LR: 6.57e-05 | 5234.8ms/step | 11.5 samples/s +Step 38730/50462 | Loss: 2.5305 | LR: 6.56e-05 | 5245.3ms/step | 11.4 samples/s +Step 38740/50462 | Loss: 2.4976 | LR: 6.55e-05 | 5247.2ms/step | 11.4 samples/s +Step 38750/50462 | Loss: 2.5023 | LR: 6.54e-05 | 5240.4ms/step | 11.4 samples/s +Validation at step 38750: Loss = 2.5116, Perplexity = 12.32 +Step 38760/50462 | Loss: 2.5034 | LR: 6.53e-05 | 7747.5ms/step | 7.7 samples/s +Step 38770/50462 | Loss: 2.5184 | LR: 6.52e-05 | 5264.1ms/step | 11.4 samples/s +Step 38780/50462 | Loss: 2.4901 | LR: 6.51e-05 | 5230.4ms/step | 11.5 samples/s +Step 38790/50462 | Loss: 2.5415 | LR: 6.50e-05 | 5249.2ms/step | 11.4 samples/s +Step 38800/50462 | Loss: 2.5068 | LR: 6.49e-05 | 5209.7ms/step | 11.5 samples/s +Step 38810/50462 | Loss: 2.4905 | LR: 6.48e-05 | 5223.7ms/step | 11.5 samples/s +Step 38820/50462 | Loss: 2.4947 | LR: 6.47e-05 | 5249.9ms/step | 11.4 samples/s +Step 38830/50462 | Loss: 2.5039 | LR: 6.46e-05 | 5251.1ms/step | 11.4 samples/s +Step 38840/50462 | Loss: 2.5146 | LR: 6.45e-05 | 5243.7ms/step | 11.4 samples/s +Step 38850/50462 | Loss: 2.5229 | LR: 6.43e-05 | 5224.2ms/step | 11.5 samples/s +Step 38860/50462 | Loss: 2.4862 | LR: 6.42e-05 | 5196.0ms/step | 11.5 samples/s +Step 38870/50462 | Loss: 2.5027 | LR: 6.41e-05 | 5241.5ms/step | 11.4 samples/s +Step 38880/50462 | Loss: 2.4974 | LR: 6.40e-05 | 5241.2ms/step | 11.4 samples/s +Step 38890/50462 | Loss: 2.5079 | LR: 6.39e-05 | 5246.8ms/step | 11.4 samples/s +Step 38900/50462 | Loss: 2.5048 | LR: 6.38e-05 | 5192.7ms/step | 11.6 samples/s +Step 38910/50462 | Loss: 2.5066 | LR: 6.37e-05 | 5281.1ms/step | 11.4 samples/s +Step 38920/50462 | Loss: 2.4975 | LR: 6.36e-05 | 5221.1ms/step | 11.5 samples/s +Step 38930/50462 | Loss: 2.5129 | LR: 6.35e-05 | 5240.3ms/step | 11.4 samples/s +Step 38940/50462 | Loss: 2.5048 | LR: 6.34e-05 | 5245.0ms/step | 11.4 samples/s +Step 38950/50462 | Loss: 2.5119 | LR: 6.33e-05 | 5209.9ms/step | 11.5 samples/s +Step 38960/50462 | Loss: 2.4948 | LR: 6.32e-05 | 5198.9ms/step | 11.5 samples/s +Step 38970/50462 | Loss: 2.4903 | LR: 6.31e-05 | 5226.6ms/step | 11.5 samples/s +Step 38980/50462 | Loss: 2.4982 | LR: 6.30e-05 | 5240.8ms/step | 11.4 samples/s +Step 38990/50462 | Loss: 2.4738 | LR: 6.29e-05 | 5243.0ms/step | 11.4 samples/s +Step 39000/50462 | Loss: 2.5061 | LR: 6.28e-05 | 5209.8ms/step | 11.5 samples/s +Validation at step 39000: Loss = 2.5113, Perplexity = 12.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-39000 +Step 39010/50462 | Loss: 2.5227 | LR: 6.27e-05 | 8698.5ms/step | 6.9 samples/s +Step 39020/50462 | Loss: 2.4755 | LR: 6.26e-05 | 5203.1ms/step | 11.5 samples/s +Step 39030/50462 | Loss: 2.5053 | LR: 6.25e-05 | 5220.4ms/step | 11.5 samples/s +Step 39040/50462 | Loss: 2.4954 | LR: 6.23e-05 | 5242.0ms/step | 11.4 samples/s +Step 39050/50462 | Loss: 2.4731 | LR: 6.22e-05 | 5284.3ms/step | 11.4 samples/s +Step 39060/50462 | Loss: 2.4851 | LR: 6.21e-05 | 5236.8ms/step | 11.5 samples/s +Step 39070/50462 | Loss: 2.5069 | LR: 6.20e-05 | 5236.1ms/step | 11.5 samples/s +Step 39080/50462 | Loss: 2.5218 | LR: 6.19e-05 | 5252.4ms/step | 11.4 samples/s +Step 39090/50462 | Loss: 2.4800 | LR: 6.18e-05 | 5243.1ms/step | 11.4 samples/s +Step 39100/50462 | Loss: 2.5032 | LR: 6.17e-05 | 5200.3ms/step | 11.5 samples/s +Step 39110/50462 | Loss: 2.4768 | LR: 6.16e-05 | 5244.5ms/step | 11.4 samples/s +Step 39120/50462 | Loss: 2.4830 | LR: 6.15e-05 | 5245.8ms/step | 11.4 samples/s +Step 39130/50462 | Loss: 2.5201 | LR: 6.14e-05 | 5248.9ms/step | 11.4 samples/s +Step 39140/50462 | Loss: 2.4935 | LR: 6.13e-05 | 5253.4ms/step | 11.4 samples/s +Step 39150/50462 | Loss: 2.4974 | LR: 6.12e-05 | 5209.5ms/step | 11.5 samples/s +Step 39160/50462 | Loss: 2.4851 | LR: 6.11e-05 | 5242.5ms/step | 11.4 samples/s +Step 39170/50462 | Loss: 2.5116 | LR: 6.10e-05 | 5245.7ms/step | 11.4 samples/s +Step 39180/50462 | Loss: 2.5245 | LR: 6.09e-05 | 5244.7ms/step | 11.4 samples/s +Step 39190/50462 | Loss: 2.5057 | LR: 6.08e-05 | 5296.7ms/step | 11.3 samples/s +Step 39200/50462 | Loss: 2.4753 | LR: 6.07e-05 | 5242.3ms/step | 11.4 samples/s +Step 39210/50462 | Loss: 2.4985 | LR: 6.06e-05 | 5242.9ms/step | 11.4 samples/s +Step 39220/50462 | Loss: 2.4728 | LR: 6.05e-05 | 5237.5ms/step | 11.5 samples/s +Step 39230/50462 | Loss: 2.5170 | LR: 6.04e-05 | 5249.9ms/step | 11.4 samples/s +Step 39240/50462 | Loss: 2.4961 | LR: 6.03e-05 | 5248.5ms/step | 11.4 samples/s +Step 39250/50462 | Loss: 2.5160 | LR: 6.02e-05 | 5177.3ms/step | 11.6 samples/s +Validation at step 39250: Loss = 2.5110, Perplexity = 12.32 +Step 39260/50462 | Loss: 2.4998 | LR: 6.01e-05 | 7794.2ms/step | 7.7 samples/s +Step 39270/50462 | Loss: 2.5016 | LR: 6.00e-05 | 5230.5ms/step | 11.5 samples/s +Step 39280/50462 | Loss: 2.5155 | LR: 5.99e-05 | 5238.3ms/step | 11.5 samples/s +Step 39290/50462 | Loss: 2.4940 | LR: 5.98e-05 | 5238.0ms/step | 11.5 samples/s +Step 39300/50462 | Loss: 2.5130 | LR: 5.97e-05 | 5219.1ms/step | 11.5 samples/s +Step 39310/50462 | Loss: 2.5006 | LR: 5.96e-05 | 5234.6ms/step | 11.5 samples/s +Step 39320/50462 | Loss: 2.4547 | LR: 5.95e-05 | 5249.5ms/step | 11.4 samples/s +Step 39330/50462 | Loss: 2.5008 | LR: 5.94e-05 | 5286.3ms/step | 11.4 samples/s +Step 39340/50462 | Loss: 2.5065 | LR: 5.93e-05 | 5240.9ms/step | 11.4 samples/s +Step 39350/50462 | Loss: 2.4988 | LR: 5.91e-05 | 5218.1ms/step | 11.5 samples/s +Step 39360/50462 | Loss: 2.4741 | LR: 5.90e-05 | 5232.9ms/step | 11.5 samples/s +Step 39370/50462 | Loss: 2.5015 | LR: 5.89e-05 | 5237.5ms/step | 11.5 samples/s +Step 39380/50462 | Loss: 2.4912 | LR: 5.88e-05 | 5249.3ms/step | 11.4 samples/s +Step 39390/50462 | Loss: 2.5059 | LR: 5.87e-05 | 5257.4ms/step | 11.4 samples/s +Step 39400/50462 | Loss: 2.5123 | LR: 5.86e-05 | 5243.5ms/step | 11.4 samples/s +Step 39410/50462 | Loss: 2.4617 | LR: 5.85e-05 | 5240.5ms/step | 11.4 samples/s +Step 39420/50462 | Loss: 2.4801 | LR: 5.84e-05 | 5249.7ms/step | 11.4 samples/s +Step 39430/50462 | Loss: 2.4775 | LR: 5.83e-05 | 5257.2ms/step | 11.4 samples/s +Step 39440/50462 | Loss: 2.4984 | LR: 5.82e-05 | 5254.1ms/step | 11.4 samples/s +Step 39450/50462 | Loss: 2.4873 | LR: 5.81e-05 | 5234.6ms/step | 11.5 samples/s +Step 39460/50462 | Loss: 2.5003 | LR: 5.80e-05 | 5239.4ms/step | 11.5 samples/s +Step 39470/50462 | Loss: 2.5370 | LR: 5.79e-05 | 5247.6ms/step | 11.4 samples/s +Step 39480/50462 | Loss: 2.5208 | LR: 5.78e-05 | 5292.0ms/step | 11.3 samples/s +Step 39490/50462 | Loss: 2.4905 | LR: 5.77e-05 | 5296.5ms/step | 11.3 samples/s +Step 39500/50462 | Loss: 2.5064 | LR: 5.76e-05 | 5223.2ms/step | 11.5 samples/s +Validation at step 39500: Loss = 2.5107, Perplexity = 12.31 +Step 39510/50462 | Loss: 2.5015 | LR: 5.75e-05 | 7799.4ms/step | 7.7 samples/s +Step 39520/50462 | Loss: 2.5130 | LR: 5.74e-05 | 5242.4ms/step | 11.4 samples/s +Step 39530/50462 | Loss: 2.5027 | LR: 5.73e-05 | 5260.9ms/step | 11.4 samples/s +Step 39540/50462 | Loss: 2.5024 | LR: 5.72e-05 | 5258.4ms/step | 11.4 samples/s +Step 39550/50462 | Loss: 2.5042 | LR: 5.71e-05 | 5211.8ms/step | 11.5 samples/s +Step 39560/50462 | Loss: 2.4938 | LR: 5.70e-05 | 5244.4ms/step | 11.4 samples/s +Step 39570/50462 | Loss: 2.5033 | LR: 5.69e-05 | 5251.9ms/step | 11.4 samples/s +Step 39580/50462 | Loss: 2.4855 | LR: 5.68e-05 | 5245.3ms/step | 11.4 samples/s +Step 39590/50462 | Loss: 2.5106 | LR: 5.67e-05 | 5253.0ms/step | 11.4 samples/s +Step 39600/50462 | Loss: 2.4942 | LR: 5.66e-05 | 5270.7ms/step | 11.4 samples/s +Step 39610/50462 | Loss: 2.4857 | LR: 5.65e-05 | 5224.6ms/step | 11.5 samples/s +Step 39620/50462 | Loss: 2.4810 | LR: 5.64e-05 | 5219.4ms/step | 11.5 samples/s +Step 39630/50462 | Loss: 2.4727 | LR: 5.63e-05 | 5245.5ms/step | 11.4 samples/s +Step 39640/50462 | Loss: 2.5268 | LR: 5.62e-05 | 5256.1ms/step | 11.4 samples/s +Step 39650/50462 | Loss: 2.5005 | LR: 5.61e-05 | 5209.0ms/step | 11.5 samples/s +Step 39660/50462 | Loss: 2.5012 | LR: 5.60e-05 | 5203.3ms/step | 11.5 samples/s +Step 39670/50462 | Loss: 2.5000 | LR: 5.59e-05 | 5218.0ms/step | 11.5 samples/s +Step 39680/50462 | Loss: 2.5091 | LR: 5.58e-05 | 5242.7ms/step | 11.4 samples/s +Step 39690/50462 | Loss: 2.4957 | LR: 5.57e-05 | 5246.9ms/step | 11.4 samples/s +Step 39700/50462 | Loss: 2.4920 | LR: 5.56e-05 | 5222.8ms/step | 11.5 samples/s +Step 39710/50462 | Loss: 2.4821 | LR: 5.55e-05 | 5224.4ms/step | 11.5 samples/s +Step 39720/50462 | Loss: 2.4718 | LR: 5.54e-05 | 5239.9ms/step | 11.5 samples/s +Step 39730/50462 | Loss: 2.4752 | LR: 5.53e-05 | 5250.9ms/step | 11.4 samples/s +Step 39740/50462 | Loss: 2.4956 | LR: 5.52e-05 | 5249.7ms/step | 11.4 samples/s +Step 39750/50462 | Loss: 2.5065 | LR: 5.51e-05 | 5227.0ms/step | 11.5 samples/s +Validation at step 39750: Loss = 2.5105, Perplexity = 12.31 +Step 39760/50462 | Loss: 2.4934 | LR: 5.50e-05 | 7795.2ms/step | 7.7 samples/s +Step 39770/50462 | Loss: 2.5181 | LR: 5.49e-05 | 5278.1ms/step | 11.4 samples/s +Step 39780/50462 | Loss: 2.5158 | LR: 5.48e-05 | 5294.9ms/step | 11.3 samples/s +Step 39790/50462 | Loss: 2.4941 | LR: 5.47e-05 | 5249.9ms/step | 11.4 samples/s +Step 39800/50462 | Loss: 2.4888 | LR: 5.46e-05 | 5199.8ms/step | 11.5 samples/s +Step 39810/50462 | Loss: 2.5008 | LR: 5.45e-05 | 5199.7ms/step | 11.5 samples/s +Step 39820/50462 | Loss: 2.5219 | LR: 5.44e-05 | 5229.5ms/step | 11.5 samples/s +Step 39830/50462 | Loss: 2.4938 | LR: 5.43e-05 | 5254.3ms/step | 11.4 samples/s +Step 39840/50462 | Loss: 2.5047 | LR: 5.42e-05 | 5249.6ms/step | 11.4 samples/s +Step 39850/50462 | Loss: 2.5101 | LR: 5.41e-05 | 5231.6ms/step | 11.5 samples/s +Step 39860/50462 | Loss: 2.4728 | LR: 5.40e-05 | 5234.6ms/step | 11.5 samples/s +Step 39870/50462 | Loss: 2.4996 | LR: 5.39e-05 | 5243.6ms/step | 11.4 samples/s +Step 39880/50462 | Loss: 2.4992 | LR: 5.38e-05 | 5293.0ms/step | 11.3 samples/s +Step 39890/50462 | Loss: 2.5106 | LR: 5.38e-05 | 5254.5ms/step | 11.4 samples/s +Step 39900/50462 | Loss: 2.4905 | LR: 5.37e-05 | 5250.0ms/step | 11.4 samples/s +Step 39910/50462 | Loss: 2.4969 | LR: 5.36e-05 | 5228.9ms/step | 11.5 samples/s +Step 39920/50462 | Loss: 2.4976 | LR: 5.35e-05 | 5250.9ms/step | 11.4 samples/s +Step 39930/50462 | Loss: 2.5017 | LR: 5.34e-05 | 5245.9ms/step | 11.4 samples/s +Step 39940/50462 | Loss: 2.4921 | LR: 5.33e-05 | 5258.5ms/step | 11.4 samples/s +Step 39950/50462 | Loss: 2.4827 | LR: 5.32e-05 | 5262.9ms/step | 11.4 samples/s +Step 39960/50462 | Loss: 2.4996 | LR: 5.31e-05 | 5210.1ms/step | 11.5 samples/s +Step 39970/50462 | Loss: 2.5227 | LR: 5.30e-05 | 5243.6ms/step | 11.4 samples/s +Step 39980/50462 | Loss: 2.4994 | LR: 5.29e-05 | 5243.2ms/step | 11.4 samples/s +Step 39990/50462 | Loss: 2.5332 | LR: 5.28e-05 | 5244.5ms/step | 11.4 samples/s +Step 40000/50462 | Loss: 2.4721 | LR: 5.27e-05 | 5254.9ms/step | 11.4 samples/s +Validation at step 40000: Loss = 2.5103, Perplexity = 12.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-40000 +Step 40010/50462 | Loss: 2.5161 | LR: 5.26e-05 | 8755.4ms/step | 6.9 samples/s +Step 40020/50462 | Loss: 2.5366 | LR: 5.25e-05 | 5240.7ms/step | 11.4 samples/s +Step 40030/50462 | Loss: 2.4807 | LR: 5.24e-05 | 5244.9ms/step | 11.4 samples/s +Step 40040/50462 | Loss: 2.5136 | LR: 5.23e-05 | 5249.0ms/step | 11.4 samples/s +Step 40050/50462 | Loss: 2.5012 | LR: 5.22e-05 | 5241.8ms/step | 11.4 samples/s +Step 40060/50462 | Loss: 2.4955 | LR: 5.21e-05 | 5264.2ms/step | 11.4 samples/s +Step 40070/50462 | Loss: 2.5052 | LR: 5.20e-05 | 5279.1ms/step | 11.4 samples/s +Step 40080/50462 | Loss: 2.4803 | LR: 5.19e-05 | 5253.1ms/step | 11.4 samples/s +Step 40090/50462 | Loss: 2.5148 | LR: 5.18e-05 | 5253.8ms/step | 11.4 samples/s +Step 40100/50462 | Loss: 2.5096 | LR: 5.17e-05 | 5244.9ms/step | 11.4 samples/s +Step 40110/50462 | Loss: 2.4985 | LR: 5.16e-05 | 5220.4ms/step | 11.5 samples/s +Step 40120/50462 | Loss: 2.5110 | LR: 5.15e-05 | 5242.0ms/step | 11.4 samples/s +Step 40130/50462 | Loss: 2.4917 | LR: 5.14e-05 | 5232.0ms/step | 11.5 samples/s +Step 40140/50462 | Loss: 2.4918 | LR: 5.13e-05 | 5245.2ms/step | 11.4 samples/s +Step 40150/50462 | Loss: 2.4687 | LR: 5.12e-05 | 5248.2ms/step | 11.4 samples/s +Step 40160/50462 | Loss: 2.4662 | LR: 5.11e-05 | 5269.5ms/step | 11.4 samples/s +Step 40170/50462 | Loss: 2.5050 | LR: 5.10e-05 | 5229.4ms/step | 11.5 samples/s +Step 40180/50462 | Loss: 2.5058 | LR: 5.09e-05 | 5240.7ms/step | 11.4 samples/s +Step 40190/50462 | Loss: 2.5023 | LR: 5.09e-05 | 5240.6ms/step | 11.4 samples/s +Step 40200/50462 | Loss: 2.5118 | LR: 5.08e-05 | 5248.8ms/step | 11.4 samples/s +Step 40210/50462 | Loss: 2.5160 | LR: 5.07e-05 | 5226.8ms/step | 11.5 samples/s +Step 40220/50462 | Loss: 2.5168 | LR: 5.06e-05 | 5218.0ms/step | 11.5 samples/s +Step 40230/50462 | Loss: 2.5082 | LR: 5.05e-05 | 5240.6ms/step | 11.4 samples/s +Step 40240/50462 | Loss: 2.5378 | LR: 5.04e-05 | 5248.3ms/step | 11.4 samples/s +Step 40250/50462 | Loss: 2.4991 | LR: 5.03e-05 | 5255.4ms/step | 11.4 samples/s +Validation at step 40250: Loss = 2.5100, Perplexity = 12.31 +Step 40260/50462 | Loss: 2.5045 | LR: 5.02e-05 | 7784.2ms/step | 7.7 samples/s +Step 40270/50462 | Loss: 2.4806 | LR: 5.01e-05 | 5232.0ms/step | 11.5 samples/s +Step 40280/50462 | Loss: 2.5041 | LR: 5.00e-05 | 5237.4ms/step | 11.5 samples/s +Step 40290/50462 | Loss: 2.5020 | LR: 4.99e-05 | 5241.0ms/step | 11.4 samples/s +Step 40300/50462 | Loss: 2.4980 | LR: 4.98e-05 | 5250.2ms/step | 11.4 samples/s +Step 40310/50462 | Loss: 2.4833 | LR: 4.97e-05 | 5246.0ms/step | 11.4 samples/s +Step 40320/50462 | Loss: 2.5017 | LR: 4.96e-05 | 5245.8ms/step | 11.4 samples/s +Step 40330/50462 | Loss: 2.5246 | LR: 4.95e-05 | 5236.2ms/step | 11.5 samples/s +Step 40340/50462 | Loss: 2.4833 | LR: 4.94e-05 | 5243.9ms/step | 11.4 samples/s +Step 40350/50462 | Loss: 2.4746 | LR: 4.93e-05 | 5286.4ms/step | 11.3 samples/s +Step 40360/50462 | Loss: 2.5047 | LR: 4.92e-05 | 5284.1ms/step | 11.4 samples/s +Step 40370/50462 | Loss: 2.5168 | LR: 4.91e-05 | 5199.2ms/step | 11.5 samples/s +Step 40380/50462 | Loss: 2.5031 | LR: 4.90e-05 | 5224.4ms/step | 11.5 samples/s +Step 40390/50462 | Loss: 2.4988 | LR: 4.90e-05 | 5243.3ms/step | 11.4 samples/s +Step 40400/50462 | Loss: 2.5099 | LR: 4.89e-05 | 5249.6ms/step | 11.4 samples/s +Step 40410/50462 | Loss: 2.5248 | LR: 4.88e-05 | 5257.1ms/step | 11.4 samples/s +Step 40420/50462 | Loss: 2.5226 | LR: 4.87e-05 | 5201.0ms/step | 11.5 samples/s +Step 40430/50462 | Loss: 2.4907 | LR: 4.86e-05 | 5228.4ms/step | 11.5 samples/s +Step 40440/50462 | Loss: 2.5134 | LR: 4.85e-05 | 5278.5ms/step | 11.4 samples/s +Step 40450/50462 | Loss: 2.4983 | LR: 4.84e-05 | 5245.8ms/step | 11.4 samples/s +Step 40460/50462 | Loss: 2.5325 | LR: 4.83e-05 | 5253.8ms/step | 11.4 samples/s +Step 40470/50462 | Loss: 2.4849 | LR: 4.82e-05 | 5247.9ms/step | 11.4 samples/s +Step 40480/50462 | Loss: 2.4968 | LR: 4.81e-05 | 5224.8ms/step | 11.5 samples/s +Step 40490/50462 | Loss: 2.5130 | LR: 4.80e-05 | 5237.5ms/step | 11.5 samples/s +Step 40500/50462 | Loss: 2.5023 | LR: 4.79e-05 | 5239.9ms/step | 11.5 samples/s +Validation at step 40500: Loss = 2.5099, Perplexity = 12.30 +Step 40510/50462 | Loss: 2.5003 | LR: 4.78e-05 | 7822.8ms/step | 7.7 samples/s +Step 40520/50462 | Loss: 2.4878 | LR: 4.77e-05 | 5229.6ms/step | 11.5 samples/s +Step 40530/50462 | Loss: 2.4904 | LR: 4.76e-05 | 5232.6ms/step | 11.5 samples/s +Step 40540/50462 | Loss: 2.4631 | LR: 4.76e-05 | 5252.0ms/step | 11.4 samples/s +Step 40550/50462 | Loss: 2.5199 | LR: 4.75e-05 | 5245.6ms/step | 11.4 samples/s +Step 40560/50462 | Loss: 2.5106 | LR: 4.74e-05 | 5254.9ms/step | 11.4 samples/s +Step 40570/50462 | Loss: 2.4849 | LR: 4.73e-05 | 5241.3ms/step | 11.4 samples/s +Step 40580/50462 | Loss: 2.5131 | LR: 4.72e-05 | 5237.9ms/step | 11.5 samples/s +Step 40590/50462 | Loss: 2.4972 | LR: 4.71e-05 | 5248.4ms/step | 11.4 samples/s +Step 40600/50462 | Loss: 2.4983 | LR: 4.70e-05 | 5249.9ms/step | 11.4 samples/s +Step 40610/50462 | Loss: 2.5166 | LR: 4.69e-05 | 5266.5ms/step | 11.4 samples/s +Step 40620/50462 | Loss: 2.4731 | LR: 4.68e-05 | 5237.1ms/step | 11.5 samples/s +Step 40630/50462 | Loss: 2.4954 | LR: 4.67e-05 | 5268.5ms/step | 11.4 samples/s +Step 40640/50462 | Loss: 2.5074 | LR: 4.66e-05 | 5228.3ms/step | 11.5 samples/s +Step 40650/50462 | Loss: 2.5047 | LR: 4.65e-05 | 5285.1ms/step | 11.4 samples/s +Step 40660/50462 | Loss: 2.5157 | LR: 4.64e-05 | 5244.7ms/step | 11.4 samples/s +Step 40670/50462 | Loss: 2.4941 | LR: 4.64e-05 | 5249.7ms/step | 11.4 samples/s +Step 40680/50462 | Loss: 2.4976 | LR: 4.63e-05 | 5190.2ms/step | 11.6 samples/s +Step 40690/50462 | Loss: 2.4942 | LR: 4.62e-05 | 5222.1ms/step | 11.5 samples/s +Step 40700/50462 | Loss: 2.4668 | LR: 4.61e-05 | 5253.7ms/step | 11.4 samples/s +Step 40710/50462 | Loss: 2.5057 | LR: 4.60e-05 | 5294.0ms/step | 11.3 samples/s +Step 40720/50462 | Loss: 2.4730 | LR: 4.59e-05 | 5248.4ms/step | 11.4 samples/s +Step 40730/50462 | Loss: 2.4852 | LR: 4.58e-05 | 5189.8ms/step | 11.6 samples/s +Step 40740/50462 | Loss: 2.4799 | LR: 4.57e-05 | 5237.2ms/step | 11.5 samples/s +Step 40750/50462 | Loss: 2.4913 | LR: 4.56e-05 | 5242.9ms/step | 11.4 samples/s +Validation at step 40750: Loss = 2.5096, Perplexity = 12.30 +Step 40760/50462 | Loss: 2.5156 | LR: 4.55e-05 | 7809.5ms/step | 7.7 samples/s +Step 40770/50462 | Loss: 2.4971 | LR: 4.54e-05 | 5252.1ms/step | 11.4 samples/s +Step 40780/50462 | Loss: 2.5128 | LR: 4.54e-05 | 5189.1ms/step | 11.6 samples/s +Step 40790/50462 | Loss: 2.4983 | LR: 4.53e-05 | 5233.4ms/step | 11.5 samples/s +Step 40800/50462 | Loss: 2.5060 | LR: 4.52e-05 | 5234.8ms/step | 11.5 samples/s +Step 40810/50462 | Loss: 2.4634 | LR: 4.51e-05 | 5255.1ms/step | 11.4 samples/s +Step 40820/50462 | Loss: 2.5385 | LR: 4.50e-05 | 5243.8ms/step | 11.4 samples/s +Step 40830/50462 | Loss: 2.4502 | LR: 4.49e-05 | 5210.6ms/step | 11.5 samples/s +Step 40840/50462 | Loss: 2.4750 | LR: 4.48e-05 | 5211.3ms/step | 11.5 samples/s +Step 40850/50462 | Loss: 2.5187 | LR: 4.47e-05 | 5223.6ms/step | 11.5 samples/s +Step 40860/50462 | Loss: 2.4650 | LR: 4.46e-05 | 5233.4ms/step | 11.5 samples/s +Step 40870/50462 | Loss: 2.5206 | LR: 4.45e-05 | 5221.2ms/step | 11.5 samples/s +Step 40880/50462 | Loss: 2.5268 | LR: 4.45e-05 | 5237.7ms/step | 11.5 samples/s +Step 40890/50462 | Loss: 2.4599 | LR: 4.44e-05 | 5195.4ms/step | 11.5 samples/s +Step 40900/50462 | Loss: 2.4853 | LR: 4.43e-05 | 5219.1ms/step | 11.5 samples/s +Step 40910/50462 | Loss: 2.5053 | LR: 4.42e-05 | 5215.7ms/step | 11.5 samples/s +Step 40920/50462 | Loss: 2.5143 | LR: 4.41e-05 | 5275.4ms/step | 11.4 samples/s +Step 40930/50462 | Loss: 2.5056 | LR: 4.40e-05 | 5283.7ms/step | 11.4 samples/s +Step 40940/50462 | Loss: 2.5224 | LR: 4.39e-05 | 5232.0ms/step | 11.5 samples/s +Step 40950/50462 | Loss: 2.4765 | LR: 4.38e-05 | 5250.2ms/step | 11.4 samples/s +Step 40960/50462 | Loss: 2.5072 | LR: 4.37e-05 | 5247.1ms/step | 11.4 samples/s +Step 40970/50462 | Loss: 2.4956 | LR: 4.36e-05 | 5260.5ms/step | 11.4 samples/s +Step 40980/50462 | Loss: 2.4961 | LR: 4.36e-05 | 5251.4ms/step | 11.4 samples/s +Step 40990/50462 | Loss: 2.4658 | LR: 4.35e-05 | 5291.8ms/step | 11.3 samples/s +Step 41000/50462 | Loss: 2.5109 | LR: 4.34e-05 | 5207.3ms/step | 11.5 samples/s +Validation at step 41000: Loss = 2.5095, Perplexity = 12.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-41000 +Step 41010/50462 | Loss: 2.5177 | LR: 4.33e-05 | 8737.4ms/step | 6.9 samples/s +Step 41020/50462 | Loss: 2.5114 | LR: 4.32e-05 | 5240.4ms/step | 11.4 samples/s +Step 41030/50462 | Loss: 2.5140 | LR: 4.31e-05 | 5245.7ms/step | 11.4 samples/s +Step 41040/50462 | Loss: 2.4772 | LR: 4.30e-05 | 5243.1ms/step | 11.4 samples/s +Step 41050/50462 | Loss: 2.5086 | LR: 4.29e-05 | 5237.2ms/step | 11.5 samples/s +Step 41060/50462 | Loss: 2.5129 | LR: 4.28e-05 | 5244.0ms/step | 11.4 samples/s +Step 41070/50462 | Loss: 2.4926 | LR: 4.28e-05 | 5249.9ms/step | 11.4 samples/s +Step 41080/50462 | Loss: 2.4846 | LR: 4.27e-05 | 5253.4ms/step | 11.4 samples/s +Step 41090/50462 | Loss: 2.4639 | LR: 4.26e-05 | 5254.5ms/step | 11.4 samples/s +Step 41100/50462 | Loss: 2.4977 | LR: 4.25e-05 | 5229.2ms/step | 11.5 samples/s +Step 41110/50462 | Loss: 2.5029 | LR: 4.24e-05 | 5239.9ms/step | 11.5 samples/s +Step 41120/50462 | Loss: 2.4914 | LR: 4.23e-05 | 5246.8ms/step | 11.4 samples/s +Step 41130/50462 | Loss: 2.5168 | LR: 4.22e-05 | 5250.2ms/step | 11.4 samples/s +Step 41140/50462 | Loss: 2.4890 | LR: 4.21e-05 | 5249.0ms/step | 11.4 samples/s +Step 41150/50462 | Loss: 2.5034 | LR: 4.21e-05 | 5232.3ms/step | 11.5 samples/s +Step 41160/50462 | Loss: 2.4996 | LR: 4.20e-05 | 5240.3ms/step | 11.4 samples/s +Step 41170/50462 | Loss: 2.5047 | LR: 4.19e-05 | 5248.3ms/step | 11.4 samples/s +Step 41180/50462 | Loss: 2.4910 | LR: 4.18e-05 | 5254.5ms/step | 11.4 samples/s +Step 41190/50462 | Loss: 2.4497 | LR: 4.17e-05 | 5262.5ms/step | 11.4 samples/s +Step 41200/50462 | Loss: 2.4859 | LR: 4.16e-05 | 5254.9ms/step | 11.4 samples/s +Step 41210/50462 | Loss: 2.4631 | LR: 4.15e-05 | 5276.3ms/step | 11.4 samples/s +Step 41220/50462 | Loss: 2.4796 | LR: 4.14e-05 | 5291.5ms/step | 11.3 samples/s +Step 41230/50462 | Loss: 2.5015 | LR: 4.14e-05 | 5246.5ms/step | 11.4 samples/s +Step 41240/50462 | Loss: 2.4909 | LR: 4.13e-05 | 5246.4ms/step | 11.4 samples/s +Step 41250/50462 | Loss: 2.5076 | LR: 4.12e-05 | 5256.1ms/step | 11.4 samples/s +Validation at step 41250: Loss = 2.5093, Perplexity = 12.30 +Step 41260/50462 | Loss: 2.5021 | LR: 4.11e-05 | 7781.1ms/step | 7.7 samples/s +Step 41270/50462 | Loss: 2.4759 | LR: 4.10e-05 | 5283.0ms/step | 11.4 samples/s +Step 41280/50462 | Loss: 2.4528 | LR: 4.09e-05 | 5242.3ms/step | 11.4 samples/s +Step 41290/50462 | Loss: 2.5171 | LR: 4.08e-05 | 5239.9ms/step | 11.5 samples/s +Step 41300/50462 | Loss: 2.5063 | LR: 4.07e-05 | 5245.6ms/step | 11.4 samples/s +Step 41310/50462 | Loss: 2.4663 | LR: 4.07e-05 | 5207.3ms/step | 11.5 samples/s +Step 41320/50462 | Loss: 2.5505 | LR: 4.06e-05 | 5199.2ms/step | 11.5 samples/s +Step 41330/50462 | Loss: 2.4849 | LR: 4.05e-05 | 5228.2ms/step | 11.5 samples/s +Step 41340/50462 | Loss: 2.4794 | LR: 4.04e-05 | 5242.7ms/step | 11.4 samples/s +Step 41350/50462 | Loss: 2.4994 | LR: 4.03e-05 | 5245.5ms/step | 11.4 samples/s +Step 41360/50462 | Loss: 2.5266 | LR: 4.02e-05 | 5236.7ms/step | 11.5 samples/s +Step 41370/50462 | Loss: 2.4994 | LR: 4.01e-05 | 5204.4ms/step | 11.5 samples/s +Step 41380/50462 | Loss: 2.4910 | LR: 4.01e-05 | 5242.4ms/step | 11.4 samples/s +Step 41390/50462 | Loss: 2.4663 | LR: 4.00e-05 | 5251.1ms/step | 11.4 samples/s +Step 41400/50462 | Loss: 2.5066 | LR: 3.99e-05 | 5254.6ms/step | 11.4 samples/s +Step 41410/50462 | Loss: 2.5122 | LR: 3.98e-05 | 5254.5ms/step | 11.4 samples/s +Step 41420/50462 | Loss: 2.4771 | LR: 3.97e-05 | 5209.4ms/step | 11.5 samples/s +Step 41430/50462 | Loss: 2.5091 | LR: 3.96e-05 | 5213.2ms/step | 11.5 samples/s +Step 41440/50462 | Loss: 2.4765 | LR: 3.95e-05 | 5248.0ms/step | 11.4 samples/s +Step 41450/50462 | Loss: 2.4973 | LR: 3.95e-05 | 5246.3ms/step | 11.4 samples/s +Step 41460/50462 | Loss: 2.4776 | LR: 3.94e-05 | 5248.6ms/step | 11.4 samples/s +Step 41470/50462 | Loss: 2.4587 | LR: 3.93e-05 | 5227.5ms/step | 11.5 samples/s +Step 41480/50462 | Loss: 2.5166 | LR: 3.92e-05 | 5202.1ms/step | 11.5 samples/s +Step 41490/50462 | Loss: 2.4701 | LR: 3.91e-05 | 5256.0ms/step | 11.4 samples/s +Step 41500/50462 | Loss: 2.5114 | LR: 3.90e-05 | 5218.9ms/step | 11.5 samples/s +Validation at step 41500: Loss = 2.5091, Perplexity = 12.29 +Step 41510/50462 | Loss: 2.4819 | LR: 3.90e-05 | 7845.4ms/step | 7.6 samples/s +Step 41520/50462 | Loss: 2.4966 | LR: 3.89e-05 | 5228.2ms/step | 11.5 samples/s +Step 41530/50462 | Loss: 2.5091 | LR: 3.88e-05 | 5212.9ms/step | 11.5 samples/s +Step 41540/50462 | Loss: 2.4769 | LR: 3.87e-05 | 5245.5ms/step | 11.4 samples/s +Step 41550/50462 | Loss: 2.5041 | LR: 3.86e-05 | 5278.6ms/step | 11.4 samples/s +Step 41560/50462 | Loss: 2.5144 | LR: 3.85e-05 | 5252.3ms/step | 11.4 samples/s +Step 41570/50462 | Loss: 2.5010 | LR: 3.84e-05 | 5244.6ms/step | 11.4 samples/s +Step 41580/50462 | Loss: 2.5089 | LR: 3.84e-05 | 5225.2ms/step | 11.5 samples/s +Step 41590/50462 | Loss: 2.4905 | LR: 3.83e-05 | 5239.6ms/step | 11.5 samples/s +Step 41600/50462 | Loss: 2.5041 | LR: 3.82e-05 | 5248.1ms/step | 11.4 samples/s +Step 41610/50462 | Loss: 2.4862 | LR: 3.81e-05 | 5257.5ms/step | 11.4 samples/s +Step 41620/50462 | Loss: 2.4916 | LR: 3.80e-05 | 5249.2ms/step | 11.4 samples/s +Step 41630/50462 | Loss: 2.4736 | LR: 3.79e-05 | 5240.0ms/step | 11.5 samples/s +Step 41640/50462 | Loss: 2.4905 | LR: 3.79e-05 | 5205.1ms/step | 11.5 samples/s +Step 41650/50462 | Loss: 2.4751 | LR: 3.78e-05 | 5231.5ms/step | 11.5 samples/s +Step 41660/50462 | Loss: 2.5046 | LR: 3.77e-05 | 5243.2ms/step | 11.4 samples/s +Step 41670/50462 | Loss: 2.4959 | LR: 3.76e-05 | 5249.1ms/step | 11.4 samples/s +Step 41680/50462 | Loss: 2.4805 | LR: 3.75e-05 | 5256.8ms/step | 11.4 samples/s +Step 41690/50462 | Loss: 2.4791 | LR: 3.74e-05 | 5232.6ms/step | 11.5 samples/s +Step 41700/50462 | Loss: 2.4894 | LR: 3.74e-05 | 5223.6ms/step | 11.5 samples/s +Step 41710/50462 | Loss: 2.4827 | LR: 3.73e-05 | 5243.1ms/step | 11.4 samples/s +Step 41720/50462 | Loss: 2.4791 | LR: 3.72e-05 | 5251.0ms/step | 11.4 samples/s +Step 41730/50462 | Loss: 2.4853 | LR: 3.71e-05 | 5245.3ms/step | 11.4 samples/s +Step 41740/50462 | Loss: 2.4790 | LR: 3.70e-05 | 5252.4ms/step | 11.4 samples/s +Step 41750/50462 | Loss: 2.4847 | LR: 3.69e-05 | 5254.5ms/step | 11.4 samples/s +Validation at step 41750: Loss = 2.5090, Perplexity = 12.29 +Step 41760/50462 | Loss: 2.4971 | LR: 3.69e-05 | 7835.6ms/step | 7.7 samples/s +Step 41770/50462 | Loss: 2.5044 | LR: 3.68e-05 | 5252.1ms/step | 11.4 samples/s +Step 41780/50462 | Loss: 2.4909 | LR: 3.67e-05 | 5287.7ms/step | 11.3 samples/s +Step 41790/50462 | Loss: 2.4763 | LR: 3.66e-05 | 5248.5ms/step | 11.4 samples/s +Step 41800/50462 | Loss: 2.4808 | LR: 3.65e-05 | 5268.2ms/step | 11.4 samples/s +Step 41810/50462 | Loss: 2.5153 | LR: 3.64e-05 | 5233.6ms/step | 11.5 samples/s +Step 41820/50462 | Loss: 2.4407 | LR: 3.64e-05 | 5282.5ms/step | 11.4 samples/s +Step 41830/50462 | Loss: 2.4888 | LR: 3.63e-05 | 5253.5ms/step | 11.4 samples/s +Step 41840/50462 | Loss: 2.4621 | LR: 3.62e-05 | 5251.6ms/step | 11.4 samples/s +Step 41850/50462 | Loss: 2.5050 | LR: 3.61e-05 | 5234.8ms/step | 11.5 samples/s +Step 41860/50462 | Loss: 2.4909 | LR: 3.60e-05 | 5217.6ms/step | 11.5 samples/s +Step 41870/50462 | Loss: 2.4889 | LR: 3.60e-05 | 5241.1ms/step | 11.4 samples/s +Step 41880/50462 | Loss: 2.4998 | LR: 3.59e-05 | 5239.1ms/step | 11.5 samples/s +Step 41890/50462 | Loss: 2.4930 | LR: 3.58e-05 | 5248.8ms/step | 11.4 samples/s +Step 41900/50462 | Loss: 2.5172 | LR: 3.57e-05 | 5245.9ms/step | 11.4 samples/s +Step 41910/50462 | Loss: 2.5082 | LR: 3.56e-05 | 5237.3ms/step | 11.5 samples/s +Step 41920/50462 | Loss: 2.4993 | LR: 3.55e-05 | 5239.0ms/step | 11.5 samples/s +Step 41930/50462 | Loss: 2.4780 | LR: 3.55e-05 | 5229.8ms/step | 11.5 samples/s +Step 41940/50462 | Loss: 2.5066 | LR: 3.54e-05 | 5234.3ms/step | 11.5 samples/s +Step 41950/50462 | Loss: 2.4788 | LR: 3.53e-05 | 5246.6ms/step | 11.4 samples/s +Step 41960/50462 | Loss: 2.5055 | LR: 3.52e-05 | 5242.1ms/step | 11.4 samples/s +Step 41970/50462 | Loss: 2.4931 | LR: 3.51e-05 | 5218.4ms/step | 11.5 samples/s +Step 41980/50462 | Loss: 2.4909 | LR: 3.51e-05 | 5230.4ms/step | 11.5 samples/s +Step 41990/50462 | Loss: 2.5239 | LR: 3.50e-05 | 5248.2ms/step | 11.4 samples/s +Step 42000/50462 | Loss: 2.5017 | LR: 3.49e-05 | 5240.1ms/step | 11.5 samples/s +Validation at step 42000: Loss = 2.5089, Perplexity = 12.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-42000 +Step 42010/50462 | Loss: 2.4704 | LR: 3.48e-05 | 8732.4ms/step | 6.9 samples/s +Step 42020/50462 | Loss: 2.5120 | LR: 3.47e-05 | 5206.4ms/step | 11.5 samples/s +Step 42030/50462 | Loss: 2.5308 | LR: 3.47e-05 | 5239.0ms/step | 11.5 samples/s +Step 42040/50462 | Loss: 2.4961 | LR: 3.46e-05 | 5239.4ms/step | 11.5 samples/s +Step 42050/50462 | Loss: 2.4988 | LR: 3.45e-05 | 5248.4ms/step | 11.4 samples/s +Step 42060/50462 | Loss: 2.5308 | LR: 3.44e-05 | 5242.9ms/step | 11.4 samples/s +Step 42070/50462 | Loss: 2.5126 | LR: 3.43e-05 | 5252.7ms/step | 11.4 samples/s +Step 42080/50462 | Loss: 2.4841 | LR: 3.43e-05 | 5219.3ms/step | 11.5 samples/s +Step 42090/50462 | Loss: 2.5156 | LR: 3.42e-05 | 5259.2ms/step | 11.4 samples/s +Step 42100/50462 | Loss: 2.4956 | LR: 3.41e-05 | 5271.3ms/step | 11.4 samples/s +Step 42110/50462 | Loss: 2.4637 | LR: 3.40e-05 | 5248.0ms/step | 11.4 samples/s +Step 42120/50462 | Loss: 2.4986 | LR: 3.39e-05 | 5240.8ms/step | 11.4 samples/s +Step 42130/50462 | Loss: 2.5120 | LR: 3.39e-05 | 5170.7ms/step | 11.6 samples/s +Step 42140/50462 | Loss: 2.5074 | LR: 3.38e-05 | 5208.1ms/step | 11.5 samples/s +Step 42150/50462 | Loss: 2.5044 | LR: 3.37e-05 | 5197.7ms/step | 11.5 samples/s +Step 42160/50462 | Loss: 2.4621 | LR: 3.36e-05 | 5220.1ms/step | 11.5 samples/s +Step 42170/50462 | Loss: 2.4891 | LR: 3.35e-05 | 5233.6ms/step | 11.5 samples/s +Step 42180/50462 | Loss: 2.4949 | LR: 3.35e-05 | 5243.9ms/step | 11.4 samples/s +Step 42190/50462 | Loss: 2.5264 | LR: 3.34e-05 | 5228.5ms/step | 11.5 samples/s +Step 42200/50462 | Loss: 2.4813 | LR: 3.33e-05 | 5238.7ms/step | 11.5 samples/s +Step 42210/50462 | Loss: 2.4866 | LR: 3.32e-05 | 5247.0ms/step | 11.4 samples/s +Step 42220/50462 | Loss: 2.5138 | LR: 3.32e-05 | 5241.8ms/step | 11.4 samples/s +Step 42230/50462 | Loss: 2.5011 | LR: 3.31e-05 | 5241.2ms/step | 11.4 samples/s +Step 42240/50462 | Loss: 2.4547 | LR: 3.30e-05 | 5215.6ms/step | 11.5 samples/s +Step 42250/50462 | Loss: 2.5062 | LR: 3.29e-05 | 5218.0ms/step | 11.5 samples/s +Validation at step 42250: Loss = 2.5088, Perplexity = 12.29 +Step 42260/50462 | Loss: 2.5140 | LR: 3.28e-05 | 7782.4ms/step | 7.7 samples/s +Step 42270/50462 | Loss: 2.5241 | LR: 3.28e-05 | 5251.5ms/step | 11.4 samples/s +Step 42280/50462 | Loss: 2.4944 | LR: 3.27e-05 | 5251.3ms/step | 11.4 samples/s +Step 42290/50462 | Loss: 2.5083 | LR: 3.26e-05 | 5211.0ms/step | 11.5 samples/s +Step 42300/50462 | Loss: 2.4938 | LR: 3.25e-05 | 5195.0ms/step | 11.5 samples/s +Step 42310/50462 | Loss: 2.4867 | LR: 3.24e-05 | 5220.1ms/step | 11.5 samples/s +Step 42320/50462 | Loss: 2.4678 | LR: 3.24e-05 | 5238.6ms/step | 11.5 samples/s +Step 42330/50462 | Loss: 2.5169 | LR: 3.23e-05 | 5252.7ms/step | 11.4 samples/s +Step 42340/50462 | Loss: 2.5232 | LR: 3.22e-05 | 5242.0ms/step | 11.4 samples/s +Step 42350/50462 | Loss: 2.4709 | LR: 3.21e-05 | 5214.3ms/step | 11.5 samples/s +Step 42360/50462 | Loss: 2.4812 | LR: 3.21e-05 | 5261.5ms/step | 11.4 samples/s +Step 42370/50462 | Loss: 2.4950 | LR: 3.20e-05 | 5241.5ms/step | 11.4 samples/s +Step 42380/50462 | Loss: 2.4802 | LR: 3.19e-05 | 5326.9ms/step | 11.3 samples/s +Step 42390/50462 | Loss: 2.5087 | LR: 3.18e-05 | 5250.6ms/step | 11.4 samples/s +Step 42400/50462 | Loss: 2.5158 | LR: 3.18e-05 | 5242.1ms/step | 11.4 samples/s +Step 42410/50462 | Loss: 2.5087 | LR: 3.17e-05 | 5173.7ms/step | 11.6 samples/s +Step 42420/50462 | Loss: 2.5022 | LR: 3.16e-05 | 5225.4ms/step | 11.5 samples/s +Step 42430/50462 | Loss: 2.4903 | LR: 3.15e-05 | 5229.7ms/step | 11.5 samples/s +Step 42440/50462 | Loss: 2.4882 | LR: 3.14e-05 | 5229.9ms/step | 11.5 samples/s +Step 42450/50462 | Loss: 2.5076 | LR: 3.14e-05 | 5235.0ms/step | 11.5 samples/s +Step 42460/50462 | Loss: 2.4882 | LR: 3.13e-05 | 5215.4ms/step | 11.5 samples/s +Step 42470/50462 | Loss: 2.5000 | LR: 3.12e-05 | 5215.2ms/step | 11.5 samples/s +Step 42480/50462 | Loss: 2.4559 | LR: 3.11e-05 | 5229.5ms/step | 11.5 samples/s +Step 42490/50462 | Loss: 2.5025 | LR: 3.11e-05 | 5243.2ms/step | 11.4 samples/s +Step 42500/50462 | Loss: 2.5119 | LR: 3.10e-05 | 5250.4ms/step | 11.4 samples/s +Validation at step 42500: Loss = 2.5087, Perplexity = 12.29 +Step 42510/50462 | Loss: 2.5079 | LR: 3.09e-05 | 7816.6ms/step | 7.7 samples/s +Step 42520/50462 | Loss: 2.4938 | LR: 3.08e-05 | 5255.2ms/step | 11.4 samples/s +Step 42530/50462 | Loss: 2.4860 | LR: 3.08e-05 | 5260.8ms/step | 11.4 samples/s +Step 42540/50462 | Loss: 2.4932 | LR: 3.07e-05 | 5254.7ms/step | 11.4 samples/s +Step 42550/50462 | Loss: 2.5169 | LR: 3.06e-05 | 5251.5ms/step | 11.4 samples/s +Step 42560/50462 | Loss: 2.5743 | LR: 3.05e-05 | 5250.2ms/step | 11.4 samples/s +Step 42570/50462 | Loss: 2.5056 | LR: 3.05e-05 | 5248.3ms/step | 11.4 samples/s +Step 42580/50462 | Loss: 2.4980 | LR: 3.04e-05 | 5249.0ms/step | 11.4 samples/s +Step 42590/50462 | Loss: 2.4929 | LR: 3.03e-05 | 5243.0ms/step | 11.4 samples/s +Step 42600/50462 | Loss: 2.5410 | LR: 3.02e-05 | 5201.5ms/step | 11.5 samples/s +Step 42610/50462 | Loss: 2.4890 | LR: 3.02e-05 | 5241.0ms/step | 11.4 samples/s +Step 42620/50462 | Loss: 2.4822 | LR: 3.01e-05 | 5240.8ms/step | 11.4 samples/s +Step 42630/50462 | Loss: 2.5141 | LR: 3.00e-05 | 5256.4ms/step | 11.4 samples/s +Step 42640/50462 | Loss: 2.5399 | LR: 2.99e-05 | 5266.4ms/step | 11.4 samples/s +Step 42650/50462 | Loss: 2.5098 | LR: 2.99e-05 | 5272.6ms/step | 11.4 samples/s +Step 42660/50462 | Loss: 2.5109 | LR: 2.98e-05 | 5264.7ms/step | 11.4 samples/s +Step 42670/50462 | Loss: 2.4803 | LR: 2.97e-05 | 5236.2ms/step | 11.5 samples/s +Step 42680/50462 | Loss: 2.4809 | LR: 2.96e-05 | 5244.5ms/step | 11.4 samples/s +Step 42690/50462 | Loss: 2.5020 | LR: 2.96e-05 | 5240.0ms/step | 11.5 samples/s +Step 42700/50462 | Loss: 2.5087 | LR: 2.95e-05 | 5241.8ms/step | 11.4 samples/s +Step 42710/50462 | Loss: 2.4914 | LR: 2.94e-05 | 5238.0ms/step | 11.5 samples/s +Step 42720/50462 | Loss: 2.4917 | LR: 2.93e-05 | 5248.8ms/step | 11.4 samples/s +Step 42730/50462 | Loss: 2.4934 | LR: 2.93e-05 | 5233.3ms/step | 11.5 samples/s +Step 42740/50462 | Loss: 2.4984 | LR: 2.92e-05 | 5245.6ms/step | 11.4 samples/s +Step 42750/50462 | Loss: 2.4935 | LR: 2.91e-05 | 5247.6ms/step | 11.4 samples/s +Validation at step 42750: Loss = 2.5085, Perplexity = 12.29 +Step 42760/50462 | Loss: 2.5077 | LR: 2.90e-05 | 7817.5ms/step | 7.7 samples/s +Step 42770/50462 | Loss: 2.4983 | LR: 2.90e-05 | 5243.8ms/step | 11.4 samples/s +Step 42780/50462 | Loss: 2.4962 | LR: 2.89e-05 | 5241.1ms/step | 11.4 samples/s +Step 42790/50462 | Loss: 2.5134 | LR: 2.88e-05 | 5239.8ms/step | 11.5 samples/s +Step 42800/50462 | Loss: 2.5135 | LR: 2.87e-05 | 5240.3ms/step | 11.4 samples/s +Step 42810/50462 | Loss: 2.5169 | LR: 2.87e-05 | 5244.5ms/step | 11.4 samples/s +Step 42820/50462 | Loss: 2.4905 | LR: 2.86e-05 | 5246.5ms/step | 11.4 samples/s +Step 42830/50462 | Loss: 2.4823 | LR: 2.85e-05 | 5250.8ms/step | 11.4 samples/s +Step 42840/50462 | Loss: 2.5113 | LR: 2.84e-05 | 5250.3ms/step | 11.4 samples/s +Step 42850/50462 | Loss: 2.4992 | LR: 2.84e-05 | 5247.1ms/step | 11.4 samples/s +Step 42860/50462 | Loss: 2.5275 | LR: 2.83e-05 | 5248.1ms/step | 11.4 samples/s +Step 42870/50462 | Loss: 2.5105 | LR: 2.82e-05 | 5247.7ms/step | 11.4 samples/s +Step 42880/50462 | Loss: 2.5053 | LR: 2.82e-05 | 5255.9ms/step | 11.4 samples/s +Step 42890/50462 | Loss: 2.4895 | LR: 2.81e-05 | 5248.4ms/step | 11.4 samples/s +Step 42900/50462 | Loss: 2.4975 | LR: 2.80e-05 | 5247.6ms/step | 11.4 samples/s +Step 42910/50462 | Loss: 2.5283 | LR: 2.79e-05 | 5249.1ms/step | 11.4 samples/s +Step 42920/50462 | Loss: 2.4872 | LR: 2.79e-05 | 5250.0ms/step | 11.4 samples/s +Step 42930/50462 | Loss: 2.4861 | LR: 2.78e-05 | 5327.5ms/step | 11.3 samples/s +Step 42940/50462 | Loss: 2.4765 | LR: 2.77e-05 | 5226.9ms/step | 11.5 samples/s +Step 42950/50462 | Loss: 2.5259 | LR: 2.76e-05 | 5269.7ms/step | 11.4 samples/s +Step 42960/50462 | Loss: 2.5097 | LR: 2.76e-05 | 5214.7ms/step | 11.5 samples/s +Step 42970/50462 | Loss: 2.4800 | LR: 2.75e-05 | 5228.4ms/step | 11.5 samples/s +Step 42980/50462 | Loss: 2.4947 | LR: 2.74e-05 | 5249.6ms/step | 11.4 samples/s +Step 42990/50462 | Loss: 2.5014 | LR: 2.74e-05 | 5246.2ms/step | 11.4 samples/s +Step 43000/50462 | Loss: 2.4799 | LR: 2.73e-05 | 5243.8ms/step | 11.4 samples/s +Validation at step 43000: Loss = 2.5085, Perplexity = 12.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-43000 +Step 43010/50462 | Loss: 2.5206 | LR: 2.72e-05 | 8759.2ms/step | 6.8 samples/s +Step 43020/50462 | Loss: 2.4934 | LR: 2.71e-05 | 5214.4ms/step | 11.5 samples/s +Step 43030/50462 | Loss: 2.4994 | LR: 2.71e-05 | 5233.7ms/step | 11.5 samples/s +Step 43040/50462 | Loss: 2.4859 | LR: 2.70e-05 | 5242.7ms/step | 11.4 samples/s +Step 43050/50462 | Loss: 2.5082 | LR: 2.69e-05 | 5244.3ms/step | 11.4 samples/s +Step 43060/50462 | Loss: 2.4819 | LR: 2.69e-05 | 5246.3ms/step | 11.4 samples/s +Step 43070/50462 | Loss: 2.4997 | LR: 2.68e-05 | 5247.4ms/step | 11.4 samples/s +Step 43080/50462 | Loss: 2.4787 | LR: 2.67e-05 | 5238.1ms/step | 11.5 samples/s +Step 43090/50462 | Loss: 2.4747 | LR: 2.66e-05 | 5249.8ms/step | 11.4 samples/s +Step 43100/50462 | Loss: 2.4753 | LR: 2.66e-05 | 5250.4ms/step | 11.4 samples/s +Step 43110/50462 | Loss: 2.4823 | LR: 2.65e-05 | 5251.2ms/step | 11.4 samples/s +Step 43120/50462 | Loss: 2.4973 | LR: 2.64e-05 | 5250.8ms/step | 11.4 samples/s +Step 43130/50462 | Loss: 2.5085 | LR: 2.64e-05 | 5243.1ms/step | 11.4 samples/s +Step 43140/50462 | Loss: 2.4776 | LR: 2.63e-05 | 5249.6ms/step | 11.4 samples/s +Step 43150/50462 | Loss: 2.4795 | LR: 2.62e-05 | 5249.8ms/step | 11.4 samples/s +Step 43160/50462 | Loss: 2.5182 | LR: 2.61e-05 | 5242.3ms/step | 11.4 samples/s +Step 43170/50462 | Loss: 2.5110 | LR: 2.61e-05 | 5256.2ms/step | 11.4 samples/s +Step 43180/50462 | Loss: 2.4640 | LR: 2.60e-05 | 5254.7ms/step | 11.4 samples/s +Step 43190/50462 | Loss: 2.5195 | LR: 2.59e-05 | 5252.4ms/step | 11.4 samples/s +Step 43200/50462 | Loss: 2.4972 | LR: 2.59e-05 | 5301.2ms/step | 11.3 samples/s +Step 43210/50462 | Loss: 2.5023 | LR: 2.58e-05 | 5255.2ms/step | 11.4 samples/s +Step 43220/50462 | Loss: 2.5099 | LR: 2.57e-05 | 5285.4ms/step | 11.4 samples/s +Step 43230/50462 | Loss: 2.4953 | LR: 2.57e-05 | 5235.1ms/step | 11.5 samples/s +Step 43240/50462 | Loss: 2.5166 | LR: 2.56e-05 | 5276.3ms/step | 11.4 samples/s +Step 43250/50462 | Loss: 2.4662 | LR: 2.55e-05 | 5255.1ms/step | 11.4 samples/s +Validation at step 43250: Loss = 2.5084, Perplexity = 12.29 +Step 43260/50462 | Loss: 2.4683 | LR: 2.54e-05 | 7822.8ms/step | 7.7 samples/s +Step 43270/50462 | Loss: 2.4814 | LR: 2.54e-05 | 5252.9ms/step | 11.4 samples/s +Step 43280/50462 | Loss: 2.5014 | LR: 2.53e-05 | 5250.8ms/step | 11.4 samples/s +Step 43290/50462 | Loss: 2.4808 | LR: 2.52e-05 | 5257.9ms/step | 11.4 samples/s +Step 43300/50462 | Loss: 2.5235 | LR: 2.52e-05 | 5254.3ms/step | 11.4 samples/s +Step 43310/50462 | Loss: 2.4887 | LR: 2.51e-05 | 5248.4ms/step | 11.4 samples/s +Step 43320/50462 | Loss: 2.4928 | LR: 2.50e-05 | 5246.5ms/step | 11.4 samples/s +Step 43330/50462 | Loss: 2.4792 | LR: 2.50e-05 | 5254.1ms/step | 11.4 samples/s +Step 43340/50462 | Loss: 2.4901 | LR: 2.49e-05 | 5243.6ms/step | 11.4 samples/s +Step 43350/50462 | Loss: 2.4762 | LR: 2.48e-05 | 5247.6ms/step | 11.4 samples/s +Step 43360/50462 | Loss: 2.5097 | LR: 2.48e-05 | 5239.9ms/step | 11.5 samples/s +Step 43370/50462 | Loss: 2.5015 | LR: 2.47e-05 | 5243.5ms/step | 11.4 samples/s +Step 43380/50462 | Loss: 2.4802 | LR: 2.46e-05 | 5241.3ms/step | 11.4 samples/s +Step 43390/50462 | Loss: 2.5093 | LR: 2.46e-05 | 5247.2ms/step | 11.4 samples/s +Step 43400/50462 | Loss: 2.4917 | LR: 2.45e-05 | 5233.4ms/step | 11.5 samples/s +Step 43410/50462 | Loss: 2.5118 | LR: 2.44e-05 | 5246.1ms/step | 11.4 samples/s +Step 43420/50462 | Loss: 2.4909 | LR: 2.44e-05 | 5253.3ms/step | 11.4 samples/s +Step 43430/50462 | Loss: 2.4953 | LR: 2.43e-05 | 5243.0ms/step | 11.4 samples/s +Step 43440/50462 | Loss: 2.4926 | LR: 2.42e-05 | 5242.9ms/step | 11.4 samples/s +Step 43450/50462 | Loss: 2.5182 | LR: 2.41e-05 | 5245.4ms/step | 11.4 samples/s +Step 43460/50462 | Loss: 2.5049 | LR: 2.41e-05 | 5245.2ms/step | 11.4 samples/s +Step 43470/50462 | Loss: 2.5200 | LR: 2.40e-05 | 5256.2ms/step | 11.4 samples/s +Step 43480/50462 | Loss: 2.5197 | LR: 2.39e-05 | 5280.5ms/step | 11.4 samples/s +Step 43490/50462 | Loss: 2.4874 | LR: 2.39e-05 | 5244.4ms/step | 11.4 samples/s +Step 43500/50462 | Loss: 2.5057 | LR: 2.38e-05 | 5240.2ms/step | 11.4 samples/s +Validation at step 43500: Loss = 2.5084, Perplexity = 12.28 +Step 43510/50462 | Loss: 2.5065 | LR: 2.37e-05 | 7854.1ms/step | 7.6 samples/s +Step 43520/50462 | Loss: 2.4703 | LR: 2.37e-05 | 5249.0ms/step | 11.4 samples/s +Step 43530/50462 | Loss: 2.4916 | LR: 2.36e-05 | 5295.0ms/step | 11.3 samples/s +Step 43540/50462 | Loss: 2.4889 | LR: 2.35e-05 | 5242.2ms/step | 11.4 samples/s +Step 43550/50462 | Loss: 2.4977 | LR: 2.35e-05 | 5252.1ms/step | 11.4 samples/s +Step 43560/50462 | Loss: 2.4769 | LR: 2.34e-05 | 5250.0ms/step | 11.4 samples/s +Step 43570/50462 | Loss: 2.4982 | LR: 2.33e-05 | 5241.2ms/step | 11.4 samples/s +Step 43580/50462 | Loss: 2.5255 | LR: 2.33e-05 | 5259.3ms/step | 11.4 samples/s +Step 43590/50462 | Loss: 2.4817 | LR: 2.32e-05 | 5254.9ms/step | 11.4 samples/s +Step 43600/50462 | Loss: 2.5166 | LR: 2.31e-05 | 5250.2ms/step | 11.4 samples/s +Step 43610/50462 | Loss: 2.5038 | LR: 2.31e-05 | 5258.7ms/step | 11.4 samples/s +Step 43620/50462 | Loss: 2.4861 | LR: 2.30e-05 | 5248.0ms/step | 11.4 samples/s +Step 43630/50462 | Loss: 2.5242 | LR: 2.29e-05 | 5248.6ms/step | 11.4 samples/s +Step 43640/50462 | Loss: 2.4862 | LR: 2.29e-05 | 5248.7ms/step | 11.4 samples/s +Step 43650/50462 | Loss: 2.5057 | LR: 2.28e-05 | 5255.9ms/step | 11.4 samples/s +Step 43660/50462 | Loss: 2.5186 | LR: 2.27e-05 | 5250.5ms/step | 11.4 samples/s +Step 43670/50462 | Loss: 2.5054 | LR: 2.27e-05 | 5252.1ms/step | 11.4 samples/s +Step 43680/50462 | Loss: 2.5090 | LR: 2.26e-05 | 5244.4ms/step | 11.4 samples/s +Step 43690/50462 | Loss: 2.5026 | LR: 2.25e-05 | 5250.0ms/step | 11.4 samples/s +Step 43700/50462 | Loss: 2.5204 | LR: 2.25e-05 | 5250.0ms/step | 11.4 samples/s +Step 43710/50462 | Loss: 2.5246 | LR: 2.24e-05 | 5248.4ms/step | 11.4 samples/s +Step 43720/50462 | Loss: 2.4808 | LR: 2.24e-05 | 5246.8ms/step | 11.4 samples/s +Step 43730/50462 | Loss: 2.4704 | LR: 2.23e-05 | 5251.4ms/step | 11.4 samples/s +Step 43740/50462 | Loss: 2.4942 | LR: 2.22e-05 | 5255.3ms/step | 11.4 samples/s +Step 43750/50462 | Loss: 2.4951 | LR: 2.22e-05 | 5257.4ms/step | 11.4 samples/s +Validation at step 43750: Loss = 2.5083, Perplexity = 12.28 +Step 43760/50462 | Loss: 2.5018 | LR: 2.21e-05 | 7858.2ms/step | 7.6 samples/s +Step 43770/50462 | Loss: 2.4864 | LR: 2.20e-05 | 5250.2ms/step | 11.4 samples/s +Step 43780/50462 | Loss: 2.4912 | LR: 2.20e-05 | 5246.1ms/step | 11.4 samples/s +Step 43790/50462 | Loss: 2.5025 | LR: 2.19e-05 | 5261.2ms/step | 11.4 samples/s +Step 43800/50462 | Loss: 2.5137 | LR: 2.18e-05 | 5276.6ms/step | 11.4 samples/s +Step 43810/50462 | Loss: 2.4588 | LR: 2.18e-05 | 5231.3ms/step | 11.5 samples/s +Step 43820/50462 | Loss: 2.4628 | LR: 2.17e-05 | 5291.1ms/step | 11.3 samples/s +Step 43830/50462 | Loss: 2.5073 | LR: 2.16e-05 | 5247.9ms/step | 11.4 samples/s +Step 43840/50462 | Loss: 2.5227 | LR: 2.16e-05 | 5244.9ms/step | 11.4 samples/s +Step 43850/50462 | Loss: 2.5162 | LR: 2.15e-05 | 5257.1ms/step | 11.4 samples/s +Step 43860/50462 | Loss: 2.5003 | LR: 2.14e-05 | 5242.9ms/step | 11.4 samples/s +Step 43870/50462 | Loss: 2.4900 | LR: 2.14e-05 | 5249.2ms/step | 11.4 samples/s +Step 43880/50462 | Loss: 2.5098 | LR: 2.13e-05 | 5248.0ms/step | 11.4 samples/s +Step 43890/50462 | Loss: 2.5024 | LR: 2.13e-05 | 5255.3ms/step | 11.4 samples/s +Step 43900/50462 | Loss: 2.5548 | LR: 2.12e-05 | 5250.4ms/step | 11.4 samples/s +Step 43910/50462 | Loss: 2.5204 | LR: 2.11e-05 | 5254.0ms/step | 11.4 samples/s +Step 43920/50462 | Loss: 2.4948 | LR: 2.11e-05 | 5240.2ms/step | 11.5 samples/s +Step 43930/50462 | Loss: 2.5215 | LR: 2.10e-05 | 5242.9ms/step | 11.4 samples/s +Step 43940/50462 | Loss: 2.4957 | LR: 2.09e-05 | 5246.3ms/step | 11.4 samples/s +Step 43950/50462 | Loss: 2.5084 | LR: 2.09e-05 | 5246.1ms/step | 11.4 samples/s +Step 43960/50462 | Loss: 2.4909 | LR: 2.08e-05 | 5249.6ms/step | 11.4 samples/s +Step 43970/50462 | Loss: 2.4866 | LR: 2.07e-05 | 5244.3ms/step | 11.4 samples/s +Step 43980/50462 | Loss: 2.4778 | LR: 2.07e-05 | 5251.5ms/step | 11.4 samples/s +Step 43990/50462 | Loss: 2.5146 | LR: 2.06e-05 | 5253.2ms/step | 11.4 samples/s +Step 44000/50462 | Loss: 2.5202 | LR: 2.06e-05 | 5245.2ms/step | 11.4 samples/s +Validation at step 44000: Loss = 2.5082, Perplexity = 12.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-44000 +Step 44010/50462 | Loss: 2.5047 | LR: 2.05e-05 | 8805.2ms/step | 6.8 samples/s +Step 44020/50462 | Loss: 2.4979 | LR: 2.04e-05 | 5249.2ms/step | 11.4 samples/s +Step 44030/50462 | Loss: 2.4928 | LR: 2.04e-05 | 5247.2ms/step | 11.4 samples/s +Step 44040/50462 | Loss: 2.4881 | LR: 2.03e-05 | 5292.7ms/step | 11.3 samples/s +Step 44050/50462 | Loss: 2.5201 | LR: 2.02e-05 | 5246.7ms/step | 11.4 samples/s +Step 44060/50462 | Loss: 2.5197 | LR: 2.02e-05 | 5247.9ms/step | 11.4 samples/s +Step 44070/50462 | Loss: 2.5200 | LR: 2.01e-05 | 5248.2ms/step | 11.4 samples/s +Step 44080/50462 | Loss: 2.5016 | LR: 2.01e-05 | 5286.4ms/step | 11.3 samples/s +Step 44090/50462 | Loss: 2.4709 | LR: 2.00e-05 | 5240.7ms/step | 11.4 samples/s +Step 44100/50462 | Loss: 2.4899 | LR: 1.99e-05 | 5290.1ms/step | 11.3 samples/s +Step 44110/50462 | Loss: 2.4920 | LR: 1.99e-05 | 5226.0ms/step | 11.5 samples/s +Step 44120/50462 | Loss: 2.5085 | LR: 1.98e-05 | 5239.8ms/step | 11.5 samples/s +Step 44130/50462 | Loss: 2.5138 | LR: 1.97e-05 | 5246.0ms/step | 11.4 samples/s +Step 44140/50462 | Loss: 2.4829 | LR: 1.97e-05 | 5240.4ms/step | 11.4 samples/s +Step 44150/50462 | Loss: 2.4779 | LR: 1.96e-05 | 5229.8ms/step | 11.5 samples/s +Step 44160/50462 | Loss: 2.5083 | LR: 1.96e-05 | 5244.8ms/step | 11.4 samples/s +Step 44170/50462 | Loss: 2.5123 | LR: 1.95e-05 | 5248.4ms/step | 11.4 samples/s +Step 44180/50462 | Loss: 2.4789 | LR: 1.94e-05 | 5245.6ms/step | 11.4 samples/s +Step 44190/50462 | Loss: 2.4842 | LR: 1.94e-05 | 5238.7ms/step | 11.5 samples/s +Step 44200/50462 | Loss: 2.5065 | LR: 1.93e-05 | 5236.5ms/step | 11.5 samples/s +Step 44210/50462 | Loss: 2.4952 | LR: 1.93e-05 | 5242.4ms/step | 11.4 samples/s +Step 44220/50462 | Loss: 2.5002 | LR: 1.92e-05 | 5251.4ms/step | 11.4 samples/s +Step 44230/50462 | Loss: 2.5118 | LR: 1.91e-05 | 5242.4ms/step | 11.4 samples/s +Step 44240/50462 | Loss: 2.4991 | LR: 1.91e-05 | 5253.6ms/step | 11.4 samples/s +Step 44250/50462 | Loss: 2.5131 | LR: 1.90e-05 | 5249.7ms/step | 11.4 samples/s +Validation at step 44250: Loss = 2.5082, Perplexity = 12.28 +Step 44260/50462 | Loss: 2.5070 | LR: 1.90e-05 | 7822.8ms/step | 7.7 samples/s +Step 44270/50462 | Loss: 2.4775 | LR: 1.89e-05 | 5260.2ms/step | 11.4 samples/s +Step 44280/50462 | Loss: 2.5007 | LR: 1.88e-05 | 5261.6ms/step | 11.4 samples/s +Step 44290/50462 | Loss: 2.4975 | LR: 1.88e-05 | 5254.7ms/step | 11.4 samples/s +Step 44300/50462 | Loss: 2.4473 | LR: 1.87e-05 | 5254.9ms/step | 11.4 samples/s +Step 44310/50462 | Loss: 2.4775 | LR: 1.87e-05 | 5296.7ms/step | 11.3 samples/s +Step 44320/50462 | Loss: 2.5113 | LR: 1.86e-05 | 5237.6ms/step | 11.5 samples/s +Step 44330/50462 | Loss: 2.5013 | LR: 1.85e-05 | 5245.9ms/step | 11.4 samples/s +Step 44340/50462 | Loss: 2.4976 | LR: 1.85e-05 | 5246.6ms/step | 11.4 samples/s +Step 44350/50462 | Loss: 2.5029 | LR: 1.84e-05 | 5243.2ms/step | 11.4 samples/s +Step 44360/50462 | Loss: 2.4756 | LR: 1.84e-05 | 5256.5ms/step | 11.4 samples/s +Step 44370/50462 | Loss: 2.5010 | LR: 1.83e-05 | 5289.9ms/step | 11.3 samples/s +Step 44380/50462 | Loss: 2.4913 | LR: 1.82e-05 | 5255.4ms/step | 11.4 samples/s +Step 44390/50462 | Loss: 2.5129 | LR: 1.82e-05 | 5307.6ms/step | 11.3 samples/s +Step 44400/50462 | Loss: 2.4792 | LR: 1.81e-05 | 5254.3ms/step | 11.4 samples/s +Step 44410/50462 | Loss: 2.4888 | LR: 1.81e-05 | 5264.8ms/step | 11.4 samples/s +Step 44420/50462 | Loss: 2.4986 | LR: 1.80e-05 | 5257.5ms/step | 11.4 samples/s +Step 44430/50462 | Loss: 2.5049 | LR: 1.79e-05 | 5263.0ms/step | 11.4 samples/s +Step 44440/50462 | Loss: 2.4943 | LR: 1.79e-05 | 5266.8ms/step | 11.4 samples/s +Step 44450/50462 | Loss: 2.4828 | LR: 1.78e-05 | 5257.2ms/step | 11.4 samples/s +Step 44460/50462 | Loss: 2.4990 | LR: 1.78e-05 | 5245.0ms/step | 11.4 samples/s +Step 44470/50462 | Loss: 2.5116 | LR: 1.77e-05 | 5257.1ms/step | 11.4 samples/s +Step 44480/50462 | Loss: 2.5263 | LR: 1.77e-05 | 5260.7ms/step | 11.4 samples/s +Step 44490/50462 | Loss: 2.5206 | LR: 1.76e-05 | 5266.0ms/step | 11.4 samples/s +Step 44500/50462 | Loss: 2.4842 | LR: 1.75e-05 | 5262.6ms/step | 11.4 samples/s +Validation at step 44500: Loss = 2.5082, Perplexity = 12.28 +Step 44510/50462 | Loss: 2.4734 | LR: 1.75e-05 | 7823.6ms/step | 7.7 samples/s +Step 44520/50462 | Loss: 2.4719 | LR: 1.74e-05 | 5255.0ms/step | 11.4 samples/s +Step 44530/50462 | Loss: 2.4988 | LR: 1.74e-05 | 5246.9ms/step | 11.4 samples/s +Step 44540/50462 | Loss: 2.4873 | LR: 1.73e-05 | 5253.1ms/step | 11.4 samples/s +Step 44550/50462 | Loss: 2.4983 | LR: 1.72e-05 | 5251.0ms/step | 11.4 samples/s +Step 44560/50462 | Loss: 2.5046 | LR: 1.72e-05 | 5241.3ms/step | 11.4 samples/s +Step 44570/50462 | Loss: 2.4923 | LR: 1.71e-05 | 5256.6ms/step | 11.4 samples/s +Step 44580/50462 | Loss: 2.4695 | LR: 1.71e-05 | 5247.2ms/step | 11.4 samples/s +Step 44590/50462 | Loss: 2.4879 | LR: 1.70e-05 | 5290.5ms/step | 11.3 samples/s +Step 44600/50462 | Loss: 2.5113 | LR: 1.70e-05 | 5245.3ms/step | 11.4 samples/s +Step 44610/50462 | Loss: 2.4952 | LR: 1.69e-05 | 5247.5ms/step | 11.4 samples/s +Step 44620/50462 | Loss: 2.5113 | LR: 1.68e-05 | 5250.8ms/step | 11.4 samples/s +Step 44630/50462 | Loss: 2.4958 | LR: 1.68e-05 | 5231.6ms/step | 11.5 samples/s +Step 44640/50462 | Loss: 2.4870 | LR: 1.67e-05 | 5237.7ms/step | 11.5 samples/s +Step 44650/50462 | Loss: 2.4760 | LR: 1.67e-05 | 5226.3ms/step | 11.5 samples/s +Step 44660/50462 | Loss: 2.5140 | LR: 1.66e-05 | 5288.7ms/step | 11.3 samples/s +Step 44670/50462 | Loss: 2.5236 | LR: 1.66e-05 | 5246.1ms/step | 11.4 samples/s +Step 44680/50462 | Loss: 2.5087 | LR: 1.65e-05 | 5296.5ms/step | 11.3 samples/s +Step 44690/50462 | Loss: 2.5062 | LR: 1.64e-05 | 5254.7ms/step | 11.4 samples/s +Step 44700/50462 | Loss: 2.5114 | LR: 1.64e-05 | 5253.3ms/step | 11.4 samples/s +Step 44710/50462 | Loss: 2.4949 | LR: 1.63e-05 | 5247.2ms/step | 11.4 samples/s +Step 44720/50462 | Loss: 2.4946 | LR: 1.63e-05 | 5246.9ms/step | 11.4 samples/s +Step 44730/50462 | Loss: 2.4843 | LR: 1.62e-05 | 5244.6ms/step | 11.4 samples/s +Step 44740/50462 | Loss: 2.4796 | LR: 1.62e-05 | 5244.9ms/step | 11.4 samples/s +Step 44750/50462 | Loss: 2.5054 | LR: 1.61e-05 | 5247.0ms/step | 11.4 samples/s +Validation at step 44750: Loss = 2.5081, Perplexity = 12.28 +Step 44760/50462 | Loss: 2.5204 | LR: 1.61e-05 | 7821.2ms/step | 7.7 samples/s +Step 44770/50462 | Loss: 2.4700 | LR: 1.60e-05 | 5246.9ms/step | 11.4 samples/s +Step 44780/50462 | Loss: 2.4977 | LR: 1.59e-05 | 5238.1ms/step | 11.5 samples/s +Step 44790/50462 | Loss: 2.4865 | LR: 1.59e-05 | 5250.7ms/step | 11.4 samples/s +Step 44800/50462 | Loss: 2.4877 | LR: 1.58e-05 | 5249.9ms/step | 11.4 samples/s +Step 44810/50462 | Loss: 2.5139 | LR: 1.58e-05 | 5252.9ms/step | 11.4 samples/s +Step 44820/50462 | Loss: 2.5036 | LR: 1.57e-05 | 5256.3ms/step | 11.4 samples/s +Step 44830/50462 | Loss: 2.5253 | LR: 1.57e-05 | 5249.0ms/step | 11.4 samples/s +Step 44840/50462 | Loss: 2.4815 | LR: 1.56e-05 | 5241.2ms/step | 11.4 samples/s +Step 44850/50462 | Loss: 2.4852 | LR: 1.56e-05 | 5246.8ms/step | 11.4 samples/s +Step 44860/50462 | Loss: 2.4845 | LR: 1.55e-05 | 5245.5ms/step | 11.4 samples/s +Step 44870/50462 | Loss: 2.4759 | LR: 1.54e-05 | 5290.3ms/step | 11.3 samples/s +Step 44880/50462 | Loss: 2.4983 | LR: 1.54e-05 | 5254.8ms/step | 11.4 samples/s +Step 44890/50462 | Loss: 2.4572 | LR: 1.53e-05 | 5247.5ms/step | 11.4 samples/s +Step 44900/50462 | Loss: 2.4589 | LR: 1.53e-05 | 5254.3ms/step | 11.4 samples/s +Step 44910/50462 | Loss: 2.4836 | LR: 1.52e-05 | 5257.5ms/step | 11.4 samples/s +Step 44920/50462 | Loss: 2.4857 | LR: 1.52e-05 | 5257.9ms/step | 11.4 samples/s +Step 44930/50462 | Loss: 2.4902 | LR: 1.51e-05 | 5252.9ms/step | 11.4 samples/s +Step 44940/50462 | Loss: 2.4924 | LR: 1.51e-05 | 5299.0ms/step | 11.3 samples/s +Step 44950/50462 | Loss: 2.4503 | LR: 1.50e-05 | 5250.4ms/step | 11.4 samples/s +Step 44960/50462 | Loss: 2.5292 | LR: 1.50e-05 | 5296.8ms/step | 11.3 samples/s +Step 44970/50462 | Loss: 2.4787 | LR: 1.49e-05 | 5255.5ms/step | 11.4 samples/s +Step 44980/50462 | Loss: 2.4903 | LR: 1.49e-05 | 5247.8ms/step | 11.4 samples/s +Step 44990/50462 | Loss: 2.5188 | LR: 1.48e-05 | 5244.1ms/step | 11.4 samples/s +Step 45000/50462 | Loss: 2.5105 | LR: 1.47e-05 | 5250.6ms/step | 11.4 samples/s +Validation at step 45000: Loss = 2.5081, Perplexity = 12.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-45000 +Step 45010/50462 | Loss: 2.5022 | LR: 1.47e-05 | 8768.8ms/step | 6.8 samples/s +Step 45020/50462 | Loss: 2.4862 | LR: 1.46e-05 | 5239.8ms/step | 11.5 samples/s +Step 45030/50462 | Loss: 2.4935 | LR: 1.46e-05 | 5249.9ms/step | 11.4 samples/s +Step 45040/50462 | Loss: 2.5060 | LR: 1.45e-05 | 5253.8ms/step | 11.4 samples/s +Step 45050/50462 | Loss: 2.4811 | LR: 1.45e-05 | 5251.6ms/step | 11.4 samples/s +Step 45060/50462 | Loss: 2.5046 | LR: 1.44e-05 | 5245.8ms/step | 11.4 samples/s +Step 45070/50462 | Loss: 2.4979 | LR: 1.44e-05 | 5245.8ms/step | 11.4 samples/s +Step 45080/50462 | Loss: 2.4891 | LR: 1.43e-05 | 5247.6ms/step | 11.4 samples/s +Step 45090/50462 | Loss: 2.4883 | LR: 1.43e-05 | 5237.3ms/step | 11.5 samples/s +Step 45100/50462 | Loss: 2.5234 | LR: 1.42e-05 | 5255.7ms/step | 11.4 samples/s +Step 45110/50462 | Loss: 2.5227 | LR: 1.42e-05 | 5246.5ms/step | 11.4 samples/s +Step 45120/50462 | Loss: 2.5052 | LR: 1.41e-05 | 5246.3ms/step | 11.4 samples/s +Step 45130/50462 | Loss: 2.5140 | LR: 1.41e-05 | 5231.7ms/step | 11.5 samples/s +Step 45140/50462 | Loss: 2.4823 | LR: 1.40e-05 | 5297.1ms/step | 11.3 samples/s +Step 45150/50462 | Loss: 2.5070 | LR: 1.40e-05 | 5243.4ms/step | 11.4 samples/s +Step 45160/50462 | Loss: 2.5137 | LR: 1.39e-05 | 5251.9ms/step | 11.4 samples/s +Step 45170/50462 | Loss: 2.4743 | LR: 1.39e-05 | 5246.6ms/step | 11.4 samples/s +Step 45180/50462 | Loss: 2.5032 | LR: 1.38e-05 | 5240.8ms/step | 11.4 samples/s +Step 45190/50462 | Loss: 2.4906 | LR: 1.37e-05 | 5251.5ms/step | 11.4 samples/s +Step 45200/50462 | Loss: 2.5146 | LR: 1.37e-05 | 5247.8ms/step | 11.4 samples/s +Step 45210/50462 | Loss: 2.4991 | LR: 1.36e-05 | 5245.8ms/step | 11.4 samples/s +Step 45220/50462 | Loss: 2.4850 | LR: 1.36e-05 | 5240.8ms/step | 11.4 samples/s +Step 45230/50462 | Loss: 2.4788 | LR: 1.35e-05 | 5275.7ms/step | 11.4 samples/s +Step 45240/50462 | Loss: 2.5143 | LR: 1.35e-05 | 5237.4ms/step | 11.5 samples/s +Step 45250/50462 | Loss: 2.4907 | LR: 1.34e-05 | 5284.3ms/step | 11.4 samples/s +Validation at step 45250: Loss = 2.5081, Perplexity = 12.28 +Step 45260/50462 | Loss: 2.5030 | LR: 1.34e-05 | 7816.3ms/step | 7.7 samples/s +Step 45270/50462 | Loss: 2.5130 | LR: 1.33e-05 | 5245.3ms/step | 11.4 samples/s +Step 45280/50462 | Loss: 2.4698 | LR: 1.33e-05 | 5240.4ms/step | 11.4 samples/s +Step 45290/50462 | Loss: 2.4955 | LR: 1.32e-05 | 5244.2ms/step | 11.4 samples/s +Step 45300/50462 | Loss: 2.5139 | LR: 1.32e-05 | 5241.7ms/step | 11.4 samples/s +Step 45310/50462 | Loss: 2.4835 | LR: 1.31e-05 | 5244.6ms/step | 11.4 samples/s +Step 45320/50462 | Loss: 2.4911 | LR: 1.31e-05 | 5248.8ms/step | 11.4 samples/s +Step 45330/50462 | Loss: 2.5064 | LR: 1.30e-05 | 5247.6ms/step | 11.4 samples/s +Step 45340/50462 | Loss: 2.4973 | LR: 1.30e-05 | 5249.9ms/step | 11.4 samples/s +Step 45350/50462 | Loss: 2.4945 | LR: 1.29e-05 | 5245.4ms/step | 11.4 samples/s +Step 45360/50462 | Loss: 2.5155 | LR: 1.29e-05 | 5241.6ms/step | 11.4 samples/s +Step 45370/50462 | Loss: 2.4875 | LR: 1.28e-05 | 5246.3ms/step | 11.4 samples/s +Step 45380/50462 | Loss: 2.4848 | LR: 1.28e-05 | 5239.4ms/step | 11.5 samples/s +Step 45390/50462 | Loss: 2.4913 | LR: 1.27e-05 | 5245.0ms/step | 11.4 samples/s +Step 45400/50462 | Loss: 2.4744 | LR: 1.27e-05 | 5244.9ms/step | 11.4 samples/s +Step 45410/50462 | Loss: 2.5203 | LR: 1.26e-05 | 5254.6ms/step | 11.4 samples/s +Step 45420/50462 | Loss: 2.4903 | LR: 1.26e-05 | 5290.5ms/step | 11.3 samples/s +Step 45430/50462 | Loss: 2.4663 | LR: 1.25e-05 | 5242.4ms/step | 11.4 samples/s +Step 45440/50462 | Loss: 2.5068 | LR: 1.25e-05 | 5252.2ms/step | 11.4 samples/s +Step 45450/50462 | Loss: 2.4974 | LR: 1.24e-05 | 5245.1ms/step | 11.4 samples/s +Step 45460/50462 | Loss: 2.4869 | LR: 1.24e-05 | 5254.6ms/step | 11.4 samples/s +Step 45470/50462 | Loss: 2.4958 | LR: 1.23e-05 | 5248.3ms/step | 11.4 samples/s +Step 45480/50462 | Loss: 2.4861 | LR: 1.23e-05 | 5256.0ms/step | 11.4 samples/s +Step 45490/50462 | Loss: 2.5075 | LR: 1.22e-05 | 5249.0ms/step | 11.4 samples/s +Step 45500/50462 | Loss: 2.5074 | LR: 1.22e-05 | 5249.7ms/step | 11.4 samples/s +Validation at step 45500: Loss = 2.5081, Perplexity = 12.28 +Step 45510/50462 | Loss: 2.4817 | LR: 1.21e-05 | 7819.4ms/step | 7.7 samples/s +Step 45520/50462 | Loss: 2.5274 | LR: 1.21e-05 | 5285.9ms/step | 11.4 samples/s +Step 45530/50462 | Loss: 2.4729 | LR: 1.20e-05 | 5244.7ms/step | 11.4 samples/s +Step 45540/50462 | Loss: 2.5220 | LR: 1.20e-05 | 5286.8ms/step | 11.3 samples/s +Step 45550/50462 | Loss: 2.4823 | LR: 1.19e-05 | 5248.4ms/step | 11.4 samples/s +Step 45560/50462 | Loss: 2.5069 | LR: 1.19e-05 | 5246.5ms/step | 11.4 samples/s +Step 45570/50462 | Loss: 2.5119 | LR: 1.19e-05 | 5235.2ms/step | 11.5 samples/s +Step 45580/50462 | Loss: 2.4891 | LR: 1.18e-05 | 5248.8ms/step | 11.4 samples/s +Step 45590/50462 | Loss: 2.4771 | LR: 1.18e-05 | 5251.9ms/step | 11.4 samples/s +Step 45600/50462 | Loss: 2.4856 | LR: 1.17e-05 | 5253.8ms/step | 11.4 samples/s +Step 45610/50462 | Loss: 2.4639 | LR: 1.17e-05 | 5243.7ms/step | 11.4 samples/s +Step 45620/50462 | Loss: 2.4951 | LR: 1.16e-05 | 5249.5ms/step | 11.4 samples/s +Step 45630/50462 | Loss: 2.5130 | LR: 1.16e-05 | 5251.8ms/step | 11.4 samples/s +Step 45640/50462 | Loss: 2.4900 | LR: 1.15e-05 | 5231.1ms/step | 11.5 samples/s +Step 45650/50462 | Loss: 2.5158 | LR: 1.15e-05 | 5195.2ms/step | 11.5 samples/s +Step 45660/50462 | Loss: 2.5003 | LR: 1.14e-05 | 5228.1ms/step | 11.5 samples/s +Step 45670/50462 | Loss: 2.4674 | LR: 1.14e-05 | 5246.7ms/step | 11.4 samples/s +Step 45680/50462 | Loss: 2.4871 | LR: 1.13e-05 | 5249.5ms/step | 11.4 samples/s +Step 45690/50462 | Loss: 2.5076 | LR: 1.13e-05 | 5289.7ms/step | 11.3 samples/s +Step 45700/50462 | Loss: 2.4995 | LR: 1.12e-05 | 5240.0ms/step | 11.5 samples/s +Step 45710/50462 | Loss: 2.5034 | LR: 1.12e-05 | 5241.5ms/step | 11.4 samples/s +Step 45720/50462 | Loss: 2.4940 | LR: 1.11e-05 | 5254.0ms/step | 11.4 samples/s +Step 45730/50462 | Loss: 2.4748 | LR: 1.11e-05 | 5248.6ms/step | 11.4 samples/s +Step 45740/50462 | Loss: 2.4889 | LR: 1.10e-05 | 5211.2ms/step | 11.5 samples/s +Step 45750/50462 | Loss: 2.5067 | LR: 1.10e-05 | 5191.3ms/step | 11.6 samples/s +Validation at step 45750: Loss = 2.5081, Perplexity = 12.28 +Step 45760/50462 | Loss: 2.4614 | LR: 1.10e-05 | 7728.9ms/step | 7.8 samples/s +Step 45770/50462 | Loss: 2.4935 | LR: 1.09e-05 | 5193.3ms/step | 11.6 samples/s +Step 45780/50462 | Loss: 2.4874 | LR: 1.09e-05 | 5196.4ms/step | 11.5 samples/s +Step 45790/50462 | Loss: 2.5238 | LR: 1.08e-05 | 5177.6ms/step | 11.6 samples/s +Step 45800/50462 | Loss: 2.4373 | LR: 1.08e-05 | 5229.3ms/step | 11.5 samples/s +Step 45810/50462 | Loss: 2.4987 | LR: 1.07e-05 | 5159.4ms/step | 11.6 samples/s +Step 45820/50462 | Loss: 2.4946 | LR: 1.07e-05 | 5203.6ms/step | 11.5 samples/s +Step 45830/50462 | Loss: 2.5014 | LR: 1.06e-05 | 5229.5ms/step | 11.5 samples/s +Step 45840/50462 | Loss: 2.5145 | LR: 1.06e-05 | 5179.7ms/step | 11.6 samples/s +Step 45850/50462 | Loss: 2.4961 | LR: 1.05e-05 | 5183.3ms/step | 11.6 samples/s +Step 45860/50462 | Loss: 2.5206 | LR: 1.05e-05 | 5194.9ms/step | 11.5 samples/s +Step 45870/50462 | Loss: 2.5058 | LR: 1.05e-05 | 5200.8ms/step | 11.5 samples/s +Step 45880/50462 | Loss: 2.5068 | LR: 1.04e-05 | 5189.5ms/step | 11.6 samples/s +Step 45890/50462 | Loss: 2.4702 | LR: 1.04e-05 | 5192.0ms/step | 11.6 samples/s +Step 45900/50462 | Loss: 2.4923 | LR: 1.03e-05 | 5185.3ms/step | 11.6 samples/s +Step 45910/50462 | Loss: 2.5229 | LR: 1.03e-05 | 5194.2ms/step | 11.6 samples/s +Step 45920/50462 | Loss: 2.4760 | LR: 1.02e-05 | 5197.2ms/step | 11.5 samples/s +Step 45930/50462 | Loss: 2.4867 | LR: 1.02e-05 | 5182.7ms/step | 11.6 samples/s +Step 45940/50462 | Loss: 2.5064 | LR: 1.01e-05 | 5169.5ms/step | 11.6 samples/s +Step 45950/50462 | Loss: 2.4934 | LR: 1.01e-05 | 5181.0ms/step | 11.6 samples/s +Step 45960/50462 | Loss: 2.4800 | LR: 1.01e-05 | 5174.8ms/step | 11.6 samples/s +Step 45970/50462 | Loss: 2.4856 | LR: 1.00e-05 | 5214.7ms/step | 11.5 samples/s +Step 45980/50462 | Loss: 2.5301 | LR: 9.96e-06 | 5170.2ms/step | 11.6 samples/s +Step 45990/50462 | Loss: 2.4831 | LR: 9.92e-06 | 5160.8ms/step | 11.6 samples/s +Step 46000/50462 | Loss: 2.4973 | LR: 9.87e-06 | 5162.2ms/step | 11.6 samples/s +Validation at step 46000: Loss = 2.5080, Perplexity = 12.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-46000 +Step 46010/50462 | Loss: 2.5009 | LR: 9.83e-06 | 8711.7ms/step | 6.9 samples/s +Step 46020/50462 | Loss: 2.4869 | LR: 9.79e-06 | 5186.7ms/step | 11.6 samples/s +Step 46030/50462 | Loss: 2.5096 | LR: 9.74e-06 | 5176.5ms/step | 11.6 samples/s +Step 46040/50462 | Loss: 2.5098 | LR: 9.70e-06 | 5179.7ms/step | 11.6 samples/s +Step 46050/50462 | Loss: 2.4920 | LR: 9.65e-06 | 5190.9ms/step | 11.6 samples/s +Step 46060/50462 | Loss: 2.4863 | LR: 9.61e-06 | 5176.6ms/step | 11.6 samples/s +Step 46070/50462 | Loss: 2.5067 | LR: 9.57e-06 | 5194.0ms/step | 11.6 samples/s +Step 46080/50462 | Loss: 2.5020 | LR: 9.52e-06 | 5195.0ms/step | 11.5 samples/s +Step 46090/50462 | Loss: 2.4713 | LR: 9.48e-06 | 5222.0ms/step | 11.5 samples/s +Step 46100/50462 | Loss: 2.4974 | LR: 9.44e-06 | 5194.1ms/step | 11.6 samples/s +Step 46110/50462 | Loss: 2.4927 | LR: 9.40e-06 | 5282.4ms/step | 11.4 samples/s +Step 46120/50462 | Loss: 2.4888 | LR: 9.35e-06 | 5255.4ms/step | 11.4 samples/s +Step 46130/50462 | Loss: 2.4895 | LR: 9.31e-06 | 5245.5ms/step | 11.4 samples/s +Step 46140/50462 | Loss: 2.4897 | LR: 9.27e-06 | 5254.0ms/step | 11.4 samples/s +Step 46150/50462 | Loss: 2.4881 | LR: 9.22e-06 | 5248.3ms/step | 11.4 samples/s +Step 46160/50462 | Loss: 2.5066 | LR: 9.18e-06 | 5250.0ms/step | 11.4 samples/s +Step 46170/50462 | Loss: 2.5060 | LR: 9.14e-06 | 5246.6ms/step | 11.4 samples/s +Step 46180/50462 | Loss: 2.4766 | LR: 9.10e-06 | 5247.5ms/step | 11.4 samples/s +Step 46190/50462 | Loss: 2.4759 | LR: 9.06e-06 | 5246.0ms/step | 11.4 samples/s +Step 46200/50462 | Loss: 2.4832 | LR: 9.01e-06 | 5247.4ms/step | 11.4 samples/s +Step 46210/50462 | Loss: 2.5010 | LR: 8.97e-06 | 5243.0ms/step | 11.4 samples/s +Step 46220/50462 | Loss: 2.5037 | LR: 8.93e-06 | 5250.8ms/step | 11.4 samples/s +Step 46230/50462 | Loss: 2.5104 | LR: 8.89e-06 | 5248.7ms/step | 11.4 samples/s +Step 46240/50462 | Loss: 2.5185 | LR: 8.85e-06 | 5248.2ms/step | 11.4 samples/s +Step 46250/50462 | Loss: 2.5166 | LR: 8.80e-06 | 5292.7ms/step | 11.3 samples/s +Validation at step 46250: Loss = 2.5080, Perplexity = 12.28 +Step 46260/50462 | Loss: 2.5158 | LR: 8.76e-06 | 7809.2ms/step | 7.7 samples/s +Step 46270/50462 | Loss: 2.5032 | LR: 8.72e-06 | 5257.2ms/step | 11.4 samples/s +Step 46280/50462 | Loss: 2.5348 | LR: 8.68e-06 | 5250.4ms/step | 11.4 samples/s +Step 46290/50462 | Loss: 2.5210 | LR: 8.64e-06 | 5250.3ms/step | 11.4 samples/s +Step 46300/50462 | Loss: 2.4909 | LR: 8.60e-06 | 5232.0ms/step | 11.5 samples/s +Step 46310/50462 | Loss: 2.4983 | LR: 8.56e-06 | 5249.8ms/step | 11.4 samples/s +Step 46320/50462 | Loss: 2.5236 | LR: 8.52e-06 | 5244.6ms/step | 11.4 samples/s +Step 46330/50462 | Loss: 2.5038 | LR: 8.47e-06 | 5248.1ms/step | 11.4 samples/s +Step 46340/50462 | Loss: 2.5041 | LR: 8.43e-06 | 5237.4ms/step | 11.5 samples/s +Step 46350/50462 | Loss: 2.5246 | LR: 8.39e-06 | 5251.0ms/step | 11.4 samples/s +Step 46360/50462 | Loss: 2.4882 | LR: 8.35e-06 | 5252.1ms/step | 11.4 samples/s +Step 46370/50462 | Loss: 2.4729 | LR: 8.31e-06 | 5247.1ms/step | 11.4 samples/s +Step 46380/50462 | Loss: 2.4990 | LR: 8.27e-06 | 5280.7ms/step | 11.4 samples/s +Step 46390/50462 | Loss: 2.4876 | LR: 8.23e-06 | 5216.9ms/step | 11.5 samples/s +Step 46400/50462 | Loss: 2.4793 | LR: 8.19e-06 | 5281.8ms/step | 11.4 samples/s +Step 46410/50462 | Loss: 2.4922 | LR: 8.15e-06 | 5225.7ms/step | 11.5 samples/s +Step 46420/50462 | Loss: 2.4899 | LR: 8.11e-06 | 5245.6ms/step | 11.4 samples/s +Step 46430/50462 | Loss: 2.5112 | LR: 8.07e-06 | 5242.8ms/step | 11.4 samples/s +Step 46440/50462 | Loss: 2.4960 | LR: 8.03e-06 | 5231.9ms/step | 11.5 samples/s +Step 46450/50462 | Loss: 2.5206 | LR: 7.99e-06 | 5237.6ms/step | 11.5 samples/s +Step 46460/50462 | Loss: 2.4986 | LR: 7.95e-06 | 5249.7ms/step | 11.4 samples/s +Step 46470/50462 | Loss: 2.4894 | LR: 7.91e-06 | 5236.7ms/step | 11.5 samples/s +Step 46480/50462 | Loss: 2.4853 | LR: 7.87e-06 | 5240.8ms/step | 11.4 samples/s +Step 46490/50462 | Loss: 2.4813 | LR: 7.83e-06 | 5233.3ms/step | 11.5 samples/s +Step 46500/50462 | Loss: 2.4980 | LR: 7.80e-06 | 5254.2ms/step | 11.4 samples/s +Validation at step 46500: Loss = 2.5080, Perplexity = 12.28 +Step 46510/50462 | Loss: 2.4920 | LR: 7.76e-06 | 7802.2ms/step | 7.7 samples/s +Step 46520/50462 | Loss: 2.5163 | LR: 7.72e-06 | 5240.8ms/step | 11.4 samples/s +Step 46530/50462 | Loss: 2.4958 | LR: 7.68e-06 | 5290.4ms/step | 11.3 samples/s +Step 46540/50462 | Loss: 2.5096 | LR: 7.64e-06 | 5235.7ms/step | 11.5 samples/s +Step 46550/50462 | Loss: 2.4693 | LR: 7.60e-06 | 5228.5ms/step | 11.5 samples/s +Step 46560/50462 | Loss: 2.4873 | LR: 7.56e-06 | 5198.1ms/step | 11.5 samples/s +Step 46570/50462 | Loss: 2.4973 | LR: 7.52e-06 | 5176.5ms/step | 11.6 samples/s +Step 46580/50462 | Loss: 2.4908 | LR: 7.49e-06 | 5180.3ms/step | 11.6 samples/s +Step 46590/50462 | Loss: 2.4814 | LR: 7.45e-06 | 5182.4ms/step | 11.6 samples/s +Step 46600/50462 | Loss: 2.4978 | LR: 7.41e-06 | 5173.3ms/step | 11.6 samples/s +Step 46610/50462 | Loss: 2.4956 | LR: 7.37e-06 | 5208.0ms/step | 11.5 samples/s +Step 46620/50462 | Loss: 2.4772 | LR: 7.33e-06 | 5194.9ms/step | 11.5 samples/s +Step 46630/50462 | Loss: 2.4923 | LR: 7.29e-06 | 5184.0ms/step | 11.6 samples/s +Step 46640/50462 | Loss: 2.5045 | LR: 7.26e-06 | 5184.0ms/step | 11.6 samples/s +Step 46650/50462 | Loss: 2.4968 | LR: 7.22e-06 | 5225.5ms/step | 11.5 samples/s +Step 46660/50462 | Loss: 2.4867 | LR: 7.18e-06 | 5197.0ms/step | 11.5 samples/s +Step 46670/50462 | Loss: 2.5014 | LR: 7.14e-06 | 5230.7ms/step | 11.5 samples/s +Step 46680/50462 | Loss: 2.5103 | LR: 7.11e-06 | 5177.7ms/step | 11.6 samples/s +Step 46690/50462 | Loss: 2.5269 | LR: 7.07e-06 | 5218.4ms/step | 11.5 samples/s +Step 46700/50462 | Loss: 2.4880 | LR: 7.03e-06 | 5178.7ms/step | 11.6 samples/s +Step 46710/50462 | Loss: 2.5205 | LR: 6.99e-06 | 5156.8ms/step | 11.6 samples/s +Step 46720/50462 | Loss: 2.5043 | LR: 6.96e-06 | 5192.9ms/step | 11.6 samples/s +Step 46730/50462 | Loss: 2.4833 | LR: 6.92e-06 | 5201.6ms/step | 11.5 samples/s +Step 46740/50462 | Loss: 2.5106 | LR: 6.88e-06 | 5184.3ms/step | 11.6 samples/s +Step 46750/50462 | Loss: 2.4926 | LR: 6.85e-06 | 5199.5ms/step | 11.5 samples/s +Validation at step 46750: Loss = 2.5080, Perplexity = 12.28 +Step 46760/50462 | Loss: 2.5074 | LR: 6.81e-06 | 7737.3ms/step | 7.8 samples/s +Step 46770/50462 | Loss: 2.4977 | LR: 6.77e-06 | 5193.8ms/step | 11.6 samples/s +Step 46780/50462 | Loss: 2.5239 | LR: 6.74e-06 | 5193.1ms/step | 11.6 samples/s +Step 46790/50462 | Loss: 2.4799 | LR: 6.70e-06 | 5218.8ms/step | 11.5 samples/s +Step 46800/50462 | Loss: 2.5171 | LR: 6.66e-06 | 5252.1ms/step | 11.4 samples/s +Step 46810/50462 | Loss: 2.5112 | LR: 6.63e-06 | 5167.4ms/step | 11.6 samples/s +Step 46820/50462 | Loss: 2.4773 | LR: 6.59e-06 | 5193.9ms/step | 11.6 samples/s +Step 46830/50462 | Loss: 2.4761 | LR: 6.56e-06 | 5181.3ms/step | 11.6 samples/s +Step 46840/50462 | Loss: 2.5005 | LR: 6.52e-06 | 5173.1ms/step | 11.6 samples/s +Step 46850/50462 | Loss: 2.4973 | LR: 6.48e-06 | 5217.4ms/step | 11.5 samples/s +Step 46860/50462 | Loss: 2.4968 | LR: 6.45e-06 | 5179.4ms/step | 11.6 samples/s +Step 46870/50462 | Loss: 2.4551 | LR: 6.41e-06 | 5185.3ms/step | 11.6 samples/s +Step 46880/50462 | Loss: 2.4908 | LR: 6.38e-06 | 5181.2ms/step | 11.6 samples/s +Step 46890/50462 | Loss: 2.4965 | LR: 6.34e-06 | 5180.9ms/step | 11.6 samples/s +Step 46900/50462 | Loss: 2.4887 | LR: 6.31e-06 | 5213.7ms/step | 11.5 samples/s +Step 46910/50462 | Loss: 2.4817 | LR: 6.27e-06 | 5206.0ms/step | 11.5 samples/s +Step 46920/50462 | Loss: 2.5206 | LR: 6.24e-06 | 5204.0ms/step | 11.5 samples/s +Step 46930/50462 | Loss: 2.4952 | LR: 6.20e-06 | 5175.8ms/step | 11.6 samples/s +Step 46940/50462 | Loss: 2.4968 | LR: 6.17e-06 | 5175.3ms/step | 11.6 samples/s +Step 46950/50462 | Loss: 2.4812 | LR: 6.13e-06 | 5195.3ms/step | 11.5 samples/s +Step 46960/50462 | Loss: 2.5041 | LR: 6.10e-06 | 5212.5ms/step | 11.5 samples/s +Step 46970/50462 | Loss: 2.5081 | LR: 6.06e-06 | 5166.2ms/step | 11.6 samples/s +Step 46980/50462 | Loss: 2.5057 | LR: 6.03e-06 | 5193.8ms/step | 11.6 samples/s +Step 46990/50462 | Loss: 2.5283 | LR: 5.99e-06 | 5169.3ms/step | 11.6 samples/s +Step 47000/50462 | Loss: 2.5075 | LR: 5.96e-06 | 5161.5ms/step | 11.6 samples/s +Validation at step 47000: Loss = 2.5080, Perplexity = 12.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-47000 +Step 47010/50462 | Loss: 2.5202 | LR: 5.93e-06 | 8733.1ms/step | 6.9 samples/s +Step 47020/50462 | Loss: 2.4965 | LR: 5.89e-06 | 5211.3ms/step | 11.5 samples/s +Step 47030/50462 | Loss: 2.5157 | LR: 5.86e-06 | 5198.5ms/step | 11.5 samples/s +Step 47040/50462 | Loss: 2.4610 | LR: 5.82e-06 | 5214.7ms/step | 11.5 samples/s +Step 47050/50462 | Loss: 2.4988 | LR: 5.79e-06 | 5193.0ms/step | 11.6 samples/s +Step 47060/50462 | Loss: 2.4961 | LR: 5.76e-06 | 5188.5ms/step | 11.6 samples/s +Step 47070/50462 | Loss: 2.4892 | LR: 5.72e-06 | 5191.9ms/step | 11.6 samples/s +Step 47080/50462 | Loss: 2.4968 | LR: 5.69e-06 | 5234.8ms/step | 11.5 samples/s +Step 47090/50462 | Loss: 2.4922 | LR: 5.65e-06 | 5214.2ms/step | 11.5 samples/s +Step 47100/50462 | Loss: 2.4953 | LR: 5.62e-06 | 5224.7ms/step | 11.5 samples/s +Step 47110/50462 | Loss: 2.4988 | LR: 5.59e-06 | 5237.0ms/step | 11.5 samples/s +Step 47120/50462 | Loss: 2.4500 | LR: 5.56e-06 | 5212.9ms/step | 11.5 samples/s +Step 47130/50462 | Loss: 2.4971 | LR: 5.52e-06 | 5198.2ms/step | 11.5 samples/s +Step 47140/50462 | Loss: 2.5098 | LR: 5.49e-06 | 5215.4ms/step | 11.5 samples/s +Step 47150/50462 | Loss: 2.4649 | LR: 5.46e-06 | 5213.5ms/step | 11.5 samples/s +Step 47160/50462 | Loss: 2.5081 | LR: 5.42e-06 | 5204.7ms/step | 11.5 samples/s +Step 47170/50462 | Loss: 2.4903 | LR: 5.39e-06 | 5201.5ms/step | 11.5 samples/s +Step 47180/50462 | Loss: 2.5007 | LR: 5.36e-06 | 5210.5ms/step | 11.5 samples/s +Step 47190/50462 | Loss: 2.4945 | LR: 5.33e-06 | 5200.0ms/step | 11.5 samples/s +Step 47200/50462 | Loss: 2.4919 | LR: 5.29e-06 | 5197.8ms/step | 11.5 samples/s +Step 47210/50462 | Loss: 2.4646 | LR: 5.26e-06 | 5197.1ms/step | 11.5 samples/s +Step 47220/50462 | Loss: 2.5063 | LR: 5.23e-06 | 5209.9ms/step | 11.5 samples/s +Step 47230/50462 | Loss: 2.5033 | LR: 5.20e-06 | 5210.0ms/step | 11.5 samples/s +Step 47240/50462 | Loss: 2.4734 | LR: 5.16e-06 | 5250.8ms/step | 11.4 samples/s +Step 47250/50462 | Loss: 2.5034 | LR: 5.13e-06 | 5197.7ms/step | 11.5 samples/s +Validation at step 47250: Loss = 2.5080, Perplexity = 12.28 +Step 47260/50462 | Loss: 2.4974 | LR: 5.10e-06 | 7741.0ms/step | 7.8 samples/s +Step 47270/50462 | Loss: 2.5107 | LR: 5.07e-06 | 5224.5ms/step | 11.5 samples/s +Step 47280/50462 | Loss: 2.4832 | LR: 5.04e-06 | 5194.6ms/step | 11.6 samples/s +Step 47290/50462 | Loss: 2.5122 | LR: 5.01e-06 | 5189.7ms/step | 11.6 samples/s +Step 47300/50462 | Loss: 2.5107 | LR: 4.97e-06 | 5149.2ms/step | 11.7 samples/s +Step 47310/50462 | Loss: 2.4721 | LR: 4.94e-06 | 5199.0ms/step | 11.5 samples/s +Step 47320/50462 | Loss: 2.4891 | LR: 4.91e-06 | 5208.8ms/step | 11.5 samples/s +Step 47330/50462 | Loss: 2.4923 | LR: 4.88e-06 | 5222.1ms/step | 11.5 samples/s +Step 47340/50462 | Loss: 2.4563 | LR: 4.85e-06 | 5192.2ms/step | 11.6 samples/s +Step 47350/50462 | Loss: 2.5048 | LR: 4.82e-06 | 5197.2ms/step | 11.5 samples/s +Step 47360/50462 | Loss: 2.4741 | LR: 4.79e-06 | 5197.6ms/step | 11.5 samples/s +Step 47370/50462 | Loss: 2.5140 | LR: 4.76e-06 | 5179.7ms/step | 11.6 samples/s +Step 47380/50462 | Loss: 2.5069 | LR: 4.73e-06 | 5214.8ms/step | 11.5 samples/s +Step 47390/50462 | Loss: 2.4887 | LR: 4.70e-06 | 5194.3ms/step | 11.6 samples/s +Step 47400/50462 | Loss: 2.4923 | LR: 4.67e-06 | 5172.3ms/step | 11.6 samples/s +Step 47410/50462 | Loss: 2.5037 | LR: 4.64e-06 | 5170.2ms/step | 11.6 samples/s +Step 47420/50462 | Loss: 2.4542 | LR: 4.61e-06 | 5179.4ms/step | 11.6 samples/s +Step 47430/50462 | Loss: 2.4837 | LR: 4.58e-06 | 5168.2ms/step | 11.6 samples/s +Step 47440/50462 | Loss: 2.4915 | LR: 4.55e-06 | 5204.6ms/step | 11.5 samples/s +Step 47450/50462 | Loss: 2.4665 | LR: 4.52e-06 | 5209.4ms/step | 11.5 samples/s +Step 47460/50462 | Loss: 2.5114 | LR: 4.49e-06 | 5167.9ms/step | 11.6 samples/s +Step 47470/50462 | Loss: 2.5080 | LR: 4.46e-06 | 5173.8ms/step | 11.6 samples/s +Step 47480/50462 | Loss: 2.5068 | LR: 4.43e-06 | 5171.9ms/step | 11.6 samples/s +Step 47490/50462 | Loss: 2.5011 | LR: 4.40e-06 | 5180.4ms/step | 11.6 samples/s +Step 47500/50462 | Loss: 2.4920 | LR: 4.37e-06 | 5195.2ms/step | 11.5 samples/s +Validation at step 47500: Loss = 2.5080, Perplexity = 12.28 +Step 47510/50462 | Loss: 2.4633 | LR: 4.34e-06 | 7785.3ms/step | 7.7 samples/s +Step 47520/50462 | Loss: 2.5104 | LR: 4.31e-06 | 5230.1ms/step | 11.5 samples/s +Step 47530/50462 | Loss: 2.5066 | LR: 4.28e-06 | 5261.2ms/step | 11.4 samples/s +Step 47540/50462 | Loss: 2.4738 | LR: 4.25e-06 | 5223.7ms/step | 11.5 samples/s +Step 47550/50462 | Loss: 2.5033 | LR: 4.22e-06 | 5201.3ms/step | 11.5 samples/s +Step 47560/50462 | Loss: 2.4896 | LR: 4.19e-06 | 5255.3ms/step | 11.4 samples/s +Step 47570/50462 | Loss: 2.4989 | LR: 4.16e-06 | 5210.8ms/step | 11.5 samples/s +Step 47580/50462 | Loss: 2.5353 | LR: 4.13e-06 | 5204.9ms/step | 11.5 samples/s +Step 47590/50462 | Loss: 2.5052 | LR: 4.11e-06 | 5195.4ms/step | 11.5 samples/s +Step 47600/50462 | Loss: 2.4972 | LR: 4.08e-06 | 5204.0ms/step | 11.5 samples/s +Step 47610/50462 | Loss: 2.4754 | LR: 4.05e-06 | 5189.3ms/step | 11.6 samples/s +Step 47620/50462 | Loss: 2.4889 | LR: 4.02e-06 | 5189.9ms/step | 11.6 samples/s +Step 47630/50462 | Loss: 2.5181 | LR: 3.99e-06 | 5204.6ms/step | 11.5 samples/s +Step 47640/50462 | Loss: 2.4717 | LR: 3.97e-06 | 5249.1ms/step | 11.4 samples/s +Step 47650/50462 | Loss: 2.4918 | LR: 3.94e-06 | 5189.0ms/step | 11.6 samples/s +Step 47660/50462 | Loss: 2.4903 | LR: 3.91e-06 | 5187.9ms/step | 11.6 samples/s +Step 47670/50462 | Loss: 2.4829 | LR: 3.88e-06 | 5207.8ms/step | 11.5 samples/s +Step 47680/50462 | Loss: 2.5002 | LR: 3.85e-06 | 5176.2ms/step | 11.6 samples/s +Step 47690/50462 | Loss: 2.4968 | LR: 3.83e-06 | 5190.6ms/step | 11.6 samples/s +Step 47700/50462 | Loss: 2.5043 | LR: 3.80e-06 | 5199.0ms/step | 11.5 samples/s +Step 47710/50462 | Loss: 2.4712 | LR: 3.77e-06 | 5218.9ms/step | 11.5 samples/s +Step 47720/50462 | Loss: 2.4971 | LR: 3.74e-06 | 5236.2ms/step | 11.5 samples/s +Step 47730/50462 | Loss: 2.4986 | LR: 3.72e-06 | 5228.5ms/step | 11.5 samples/s +Step 47740/50462 | Loss: 2.4837 | LR: 3.69e-06 | 5177.9ms/step | 11.6 samples/s +Step 47750/50462 | Loss: 2.5094 | LR: 3.66e-06 | 5203.8ms/step | 11.5 samples/s +Validation at step 47750: Loss = 2.5080, Perplexity = 12.28 +Step 47760/50462 | Loss: 2.4682 | LR: 3.64e-06 | 7778.8ms/step | 7.7 samples/s +Step 47770/50462 | Loss: 2.5306 | LR: 3.61e-06 | 5172.5ms/step | 11.6 samples/s +Step 47780/50462 | Loss: 2.4877 | LR: 3.58e-06 | 5199.9ms/step | 11.5 samples/s +Step 47790/50462 | Loss: 2.4814 | LR: 3.56e-06 | 5212.2ms/step | 11.5 samples/s +Step 47800/50462 | Loss: 2.4739 | LR: 3.53e-06 | 5152.5ms/step | 11.6 samples/s +Step 47810/50462 | Loss: 2.5076 | LR: 3.50e-06 | 5152.0ms/step | 11.6 samples/s +Step 47820/50462 | Loss: 2.5007 | LR: 3.48e-06 | 5255.9ms/step | 11.4 samples/s +Step 47830/50462 | Loss: 2.4928 | LR: 3.45e-06 | 5177.9ms/step | 11.6 samples/s +Step 47840/50462 | Loss: 2.5158 | LR: 3.42e-06 | 5169.7ms/step | 11.6 samples/s +Step 47850/50462 | Loss: 2.5060 | LR: 3.40e-06 | 5242.4ms/step | 11.4 samples/s +Step 47860/50462 | Loss: 2.5083 | LR: 3.37e-06 | 5203.6ms/step | 11.5 samples/s +Step 47870/50462 | Loss: 2.5026 | LR: 3.35e-06 | 5208.0ms/step | 11.5 samples/s +Step 47880/50462 | Loss: 2.4564 | LR: 3.32e-06 | 5214.6ms/step | 11.5 samples/s +Step 47890/50462 | Loss: 2.5147 | LR: 3.30e-06 | 5216.2ms/step | 11.5 samples/s +Step 47900/50462 | Loss: 2.5025 | LR: 3.27e-06 | 5211.8ms/step | 11.5 samples/s +Step 47910/50462 | Loss: 2.5021 | LR: 3.24e-06 | 5226.6ms/step | 11.5 samples/s +Step 47920/50462 | Loss: 2.4761 | LR: 3.22e-06 | 5145.7ms/step | 11.7 samples/s +Step 47930/50462 | Loss: 2.5141 | LR: 3.19e-06 | 5173.0ms/step | 11.6 samples/s +Step 47940/50462 | Loss: 2.4906 | LR: 3.17e-06 | 5191.0ms/step | 11.6 samples/s +Step 47950/50462 | Loss: 2.4971 | LR: 3.14e-06 | 5154.2ms/step | 11.6 samples/s +Step 47960/50462 | Loss: 2.4641 | LR: 3.12e-06 | 5187.1ms/step | 11.6 samples/s +Step 47970/50462 | Loss: 2.5006 | LR: 3.09e-06 | 5192.1ms/step | 11.6 samples/s +Step 47980/50462 | Loss: 2.4975 | LR: 3.07e-06 | 5181.5ms/step | 11.6 samples/s +Step 47990/50462 | Loss: 2.4800 | LR: 3.04e-06 | 5180.0ms/step | 11.6 samples/s +Step 48000/50462 | Loss: 2.4910 | LR: 3.02e-06 | 5183.8ms/step | 11.6 samples/s +Validation at step 48000: Loss = 2.5080, Perplexity = 12.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-48000 +Step 48010/50462 | Loss: 2.5035 | LR: 3.00e-06 | 8691.4ms/step | 6.9 samples/s +Step 48020/50462 | Loss: 2.5315 | LR: 2.97e-06 | 5180.1ms/step | 11.6 samples/s +Step 48030/50462 | Loss: 2.4737 | LR: 2.95e-06 | 5176.9ms/step | 11.6 samples/s +Step 48040/50462 | Loss: 2.4871 | LR: 2.92e-06 | 5171.3ms/step | 11.6 samples/s +Step 48050/50462 | Loss: 2.4791 | LR: 2.90e-06 | 5165.7ms/step | 11.6 samples/s +Step 48060/50462 | Loss: 2.4970 | LR: 2.87e-06 | 5230.3ms/step | 11.5 samples/s +Step 48070/50462 | Loss: 2.4998 | LR: 2.85e-06 | 5172.8ms/step | 11.6 samples/s +Step 48080/50462 | Loss: 2.5235 | LR: 2.83e-06 | 5191.8ms/step | 11.6 samples/s +Step 48090/50462 | Loss: 2.4993 | LR: 2.80e-06 | 5177.0ms/step | 11.6 samples/s +Step 48100/50462 | Loss: 2.5032 | LR: 2.78e-06 | 5184.5ms/step | 11.6 samples/s +Step 48110/50462 | Loss: 2.4920 | LR: 2.76e-06 | 5238.0ms/step | 11.5 samples/s +Step 48120/50462 | Loss: 2.4887 | LR: 2.73e-06 | 5212.3ms/step | 11.5 samples/s +Step 48130/50462 | Loss: 2.4917 | LR: 2.71e-06 | 5256.3ms/step | 11.4 samples/s +Step 48140/50462 | Loss: 2.4733 | LR: 2.69e-06 | 5181.8ms/step | 11.6 samples/s +Step 48150/50462 | Loss: 2.4862 | LR: 2.66e-06 | 5176.9ms/step | 11.6 samples/s +Step 48160/50462 | Loss: 2.5059 | LR: 2.64e-06 | 5192.4ms/step | 11.6 samples/s +Step 48170/50462 | Loss: 2.4861 | LR: 2.62e-06 | 5193.8ms/step | 11.6 samples/s +Step 48180/50462 | Loss: 2.4980 | LR: 2.60e-06 | 5164.5ms/step | 11.6 samples/s +Step 48190/50462 | Loss: 2.4940 | LR: 2.57e-06 | 5239.0ms/step | 11.5 samples/s +Step 48200/50462 | Loss: 2.5045 | LR: 2.55e-06 | 5184.3ms/step | 11.6 samples/s +Step 48210/50462 | Loss: 2.4801 | LR: 2.53e-06 | 5219.1ms/step | 11.5 samples/s +Step 48220/50462 | Loss: 2.4858 | LR: 2.51e-06 | 5183.4ms/step | 11.6 samples/s +Step 48230/50462 | Loss: 2.4930 | LR: 2.48e-06 | 5188.2ms/step | 11.6 samples/s +Step 48240/50462 | Loss: 2.4914 | LR: 2.46e-06 | 5189.7ms/step | 11.6 samples/s +Step 48250/50462 | Loss: 2.5300 | LR: 2.44e-06 | 5187.8ms/step | 11.6 samples/s +Validation at step 48250: Loss = 2.5080, Perplexity = 12.28 +Step 48260/50462 | Loss: 2.4661 | LR: 2.42e-06 | 7735.2ms/step | 7.8 samples/s +Step 48270/50462 | Loss: 2.4931 | LR: 2.39e-06 | 5196.6ms/step | 11.5 samples/s +Step 48280/50462 | Loss: 2.5013 | LR: 2.37e-06 | 5145.3ms/step | 11.7 samples/s +Step 48290/50462 | Loss: 2.4883 | LR: 2.35e-06 | 5135.2ms/step | 11.7 samples/s +Step 48300/50462 | Loss: 2.4817 | LR: 2.33e-06 | 5157.0ms/step | 11.6 samples/s +Step 48310/50462 | Loss: 2.5033 | LR: 2.31e-06 | 5151.2ms/step | 11.6 samples/s +Step 48320/50462 | Loss: 2.5007 | LR: 2.29e-06 | 5151.8ms/step | 11.6 samples/s +Step 48330/50462 | Loss: 2.4844 | LR: 2.27e-06 | 5193.7ms/step | 11.6 samples/s +Step 48340/50462 | Loss: 2.5070 | LR: 2.24e-06 | 5216.9ms/step | 11.5 samples/s +Step 48350/50462 | Loss: 2.5207 | LR: 2.22e-06 | 5182.1ms/step | 11.6 samples/s +Step 48360/50462 | Loss: 2.4767 | LR: 2.20e-06 | 5186.0ms/step | 11.6 samples/s +Step 48370/50462 | Loss: 2.5153 | LR: 2.18e-06 | 5186.6ms/step | 11.6 samples/s +Step 48380/50462 | Loss: 2.4798 | LR: 2.16e-06 | 5169.3ms/step | 11.6 samples/s +Step 48390/50462 | Loss: 2.5295 | LR: 2.14e-06 | 5188.1ms/step | 11.6 samples/s +Step 48400/50462 | Loss: 2.4849 | LR: 2.12e-06 | 5251.5ms/step | 11.4 samples/s +Step 48410/50462 | Loss: 2.5125 | LR: 2.10e-06 | 5213.0ms/step | 11.5 samples/s +Step 48420/50462 | Loss: 2.4878 | LR: 2.08e-06 | 5259.1ms/step | 11.4 samples/s +Step 48430/50462 | Loss: 2.4858 | LR: 2.06e-06 | 5197.2ms/step | 11.5 samples/s +Step 48440/50462 | Loss: 2.5036 | LR: 2.04e-06 | 5197.8ms/step | 11.5 samples/s +Step 48450/50462 | Loss: 2.4684 | LR: 2.02e-06 | 5193.1ms/step | 11.6 samples/s +Step 48460/50462 | Loss: 2.5161 | LR: 2.00e-06 | 5199.7ms/step | 11.5 samples/s +Step 48470/50462 | Loss: 2.5028 | LR: 1.98e-06 | 5239.3ms/step | 11.5 samples/s +Step 48480/50462 | Loss: 2.4865 | LR: 1.96e-06 | 5213.3ms/step | 11.5 samples/s +Step 48490/50462 | Loss: 2.4942 | LR: 1.94e-06 | 5204.4ms/step | 11.5 samples/s +Step 48500/50462 | Loss: 2.4640 | LR: 1.92e-06 | 5218.1ms/step | 11.5 samples/s +Validation at step 48500: Loss = 2.5080, Perplexity = 12.28 +Step 48510/50462 | Loss: 2.4973 | LR: 1.90e-06 | 7744.4ms/step | 7.7 samples/s +Step 48520/50462 | Loss: 2.4969 | LR: 1.88e-06 | 5177.7ms/step | 11.6 samples/s +Step 48530/50462 | Loss: 2.5149 | LR: 1.86e-06 | 5224.5ms/step | 11.5 samples/s +Step 48540/50462 | Loss: 2.4697 | LR: 1.84e-06 | 5194.8ms/step | 11.6 samples/s +Step 48550/50462 | Loss: 2.4797 | LR: 1.82e-06 | 5165.1ms/step | 11.6 samples/s +Step 48560/50462 | Loss: 2.4917 | LR: 1.80e-06 | 5171.6ms/step | 11.6 samples/s +Step 48570/50462 | Loss: 2.4927 | LR: 1.78e-06 | 5141.7ms/step | 11.7 samples/s +Step 48580/50462 | Loss: 2.4852 | LR: 1.77e-06 | 5173.8ms/step | 11.6 samples/s +Step 48590/50462 | Loss: 2.5104 | LR: 1.75e-06 | 5181.1ms/step | 11.6 samples/s +Step 48600/50462 | Loss: 2.5013 | LR: 1.73e-06 | 5201.9ms/step | 11.5 samples/s +Step 48610/50462 | Loss: 2.4857 | LR: 1.71e-06 | 5218.1ms/step | 11.5 samples/s +Step 48620/50462 | Loss: 2.4928 | LR: 1.69e-06 | 5190.8ms/step | 11.6 samples/s +Step 48630/50462 | Loss: 2.5143 | LR: 1.67e-06 | 5207.0ms/step | 11.5 samples/s +Step 48640/50462 | Loss: 2.4963 | LR: 1.66e-06 | 5172.0ms/step | 11.6 samples/s +Step 48650/50462 | Loss: 2.5128 | LR: 1.64e-06 | 5167.7ms/step | 11.6 samples/s +Step 48660/50462 | Loss: 2.5082 | LR: 1.62e-06 | 5146.5ms/step | 11.7 samples/s +Step 48670/50462 | Loss: 2.5106 | LR: 1.60e-06 | 5159.9ms/step | 11.6 samples/s +Step 48680/50462 | Loss: 2.4928 | LR: 1.58e-06 | 5193.4ms/step | 11.6 samples/s +Step 48690/50462 | Loss: 2.5067 | LR: 1.57e-06 | 5182.8ms/step | 11.6 samples/s +Step 48700/50462 | Loss: 2.4667 | LR: 1.55e-06 | 5192.6ms/step | 11.6 samples/s +Step 48710/50462 | Loss: 2.4894 | LR: 1.53e-06 | 5250.0ms/step | 11.4 samples/s +Step 48720/50462 | Loss: 2.5106 | LR: 1.51e-06 | 5172.4ms/step | 11.6 samples/s +Step 48730/50462 | Loss: 2.4956 | LR: 1.50e-06 | 5144.7ms/step | 11.7 samples/s +Step 48740/50462 | Loss: 2.4725 | LR: 1.48e-06 | 5160.9ms/step | 11.6 samples/s +Step 48750/50462 | Loss: 2.5172 | LR: 1.46e-06 | 5245.1ms/step | 11.4 samples/s +Validation at step 48750: Loss = 2.5080, Perplexity = 12.28 +Step 48760/50462 | Loss: 2.5276 | LR: 1.44e-06 | 7733.5ms/step | 7.8 samples/s +Step 48770/50462 | Loss: 2.4676 | LR: 1.43e-06 | 5194.3ms/step | 11.6 samples/s +Step 48780/50462 | Loss: 2.4895 | LR: 1.41e-06 | 5185.3ms/step | 11.6 samples/s +Step 48790/50462 | Loss: 2.4986 | LR: 1.39e-06 | 5180.8ms/step | 11.6 samples/s +Step 48800/50462 | Loss: 2.4778 | LR: 1.38e-06 | 5186.1ms/step | 11.6 samples/s +Step 48810/50462 | Loss: 2.4866 | LR: 1.36e-06 | 5200.1ms/step | 11.5 samples/s +Step 48820/50462 | Loss: 2.4928 | LR: 1.34e-06 | 5192.1ms/step | 11.6 samples/s +Step 48830/50462 | Loss: 2.5085 | LR: 1.33e-06 | 5195.8ms/step | 11.5 samples/s +Step 48840/50462 | Loss: 2.5218 | LR: 1.31e-06 | 5195.6ms/step | 11.5 samples/s +Step 48850/50462 | Loss: 2.4934 | LR: 1.30e-06 | 5184.8ms/step | 11.6 samples/s +Step 48860/50462 | Loss: 2.4587 | LR: 1.28e-06 | 5179.3ms/step | 11.6 samples/s +Step 48870/50462 | Loss: 2.5024 | LR: 1.26e-06 | 5186.2ms/step | 11.6 samples/s +Step 48880/50462 | Loss: 2.4846 | LR: 1.25e-06 | 5196.6ms/step | 11.5 samples/s +Step 48890/50462 | Loss: 2.5153 | LR: 1.23e-06 | 5188.9ms/step | 11.6 samples/s +Step 48900/50462 | Loss: 2.4933 | LR: 1.22e-06 | 5190.6ms/step | 11.6 samples/s +Step 48910/50462 | Loss: 2.5082 | LR: 1.20e-06 | 5189.0ms/step | 11.6 samples/s +Step 48920/50462 | Loss: 2.4938 | LR: 1.19e-06 | 5189.6ms/step | 11.6 samples/s +Step 48930/50462 | Loss: 2.4904 | LR: 1.17e-06 | 5188.0ms/step | 11.6 samples/s +Step 48940/50462 | Loss: 2.4718 | LR: 1.16e-06 | 5210.4ms/step | 11.5 samples/s +Step 48950/50462 | Loss: 2.4709 | LR: 1.14e-06 | 5201.9ms/step | 11.5 samples/s +Step 48960/50462 | Loss: 2.5016 | LR: 1.13e-06 | 5200.4ms/step | 11.5 samples/s +Step 48970/50462 | Loss: 2.4690 | LR: 1.11e-06 | 5241.0ms/step | 11.4 samples/s +Step 48980/50462 | Loss: 2.5380 | LR: 1.10e-06 | 5177.1ms/step | 11.6 samples/s +Step 48990/50462 | Loss: 2.5222 | LR: 1.08e-06 | 5164.7ms/step | 11.6 samples/s +Step 49000/50462 | Loss: 2.5043 | LR: 1.07e-06 | 5215.4ms/step | 11.5 samples/s +Validation at step 49000: Loss = 2.5080, Perplexity = 12.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-49000 +Step 49010/50462 | Loss: 2.4924 | LR: 1.05e-06 | 8764.8ms/step | 6.8 samples/s +Step 49020/50462 | Loss: 2.5011 | LR: 1.04e-06 | 5216.6ms/step | 11.5 samples/s +Step 49030/50462 | Loss: 2.5018 | LR: 1.02e-06 | 5167.6ms/step | 11.6 samples/s +Step 49040/50462 | Loss: 2.4988 | LR: 1.01e-06 | 5167.1ms/step | 11.6 samples/s +Step 49050/50462 | Loss: 2.4982 | LR: 9.95e-07 | 5184.3ms/step | 11.6 samples/s +Step 49060/50462 | Loss: 2.5183 | LR: 9.81e-07 | 5202.9ms/step | 11.5 samples/s +Step 49070/50462 | Loss: 2.5160 | LR: 9.67e-07 | 5202.9ms/step | 11.5 samples/s +Step 49080/50462 | Loss: 2.5177 | LR: 9.53e-07 | 5162.1ms/step | 11.6 samples/s +Step 49090/50462 | Loss: 2.4719 | LR: 9.39e-07 | 5173.1ms/step | 11.6 samples/s +Step 49100/50462 | Loss: 2.5206 | LR: 9.25e-07 | 5183.8ms/step | 11.6 samples/s +Step 49110/50462 | Loss: 2.5062 | LR: 9.12e-07 | 5182.8ms/step | 11.6 samples/s +Step 49120/50462 | Loss: 2.4746 | LR: 8.99e-07 | 5165.5ms/step | 11.6 samples/s +Step 49130/50462 | Loss: 2.5039 | LR: 8.85e-07 | 5167.8ms/step | 11.6 samples/s +Step 49140/50462 | Loss: 2.4756 | LR: 8.72e-07 | 5162.9ms/step | 11.6 samples/s +Step 49150/50462 | Loss: 2.5080 | LR: 8.59e-07 | 5148.0ms/step | 11.7 samples/s +Step 49160/50462 | Loss: 2.4960 | LR: 8.46e-07 | 5182.6ms/step | 11.6 samples/s +Step 49170/50462 | Loss: 2.4946 | LR: 8.33e-07 | 5165.6ms/step | 11.6 samples/s +Step 49180/50462 | Loss: 2.4845 | LR: 8.20e-07 | 5173.8ms/step | 11.6 samples/s +Step 49190/50462 | Loss: 2.4748 | LR: 8.07e-07 | 5177.0ms/step | 11.6 samples/s +Step 49200/50462 | Loss: 2.5281 | LR: 7.95e-07 | 5185.8ms/step | 11.6 samples/s +Step 49210/50462 | Loss: 2.4990 | LR: 7.82e-07 | 5173.0ms/step | 11.6 samples/s +Step 49220/50462 | Loss: 2.5089 | LR: 7.70e-07 | 5168.6ms/step | 11.6 samples/s +Step 49230/50462 | Loss: 2.5097 | LR: 7.57e-07 | 5185.0ms/step | 11.6 samples/s +Step 49240/50462 | Loss: 2.5002 | LR: 7.45e-07 | 5178.3ms/step | 11.6 samples/s +Step 49250/50462 | Loss: 2.4947 | LR: 7.33e-07 | 5170.4ms/step | 11.6 samples/s +Validation at step 49250: Loss = 2.5080, Perplexity = 12.28 +Step 49260/50462 | Loss: 2.4672 | LR: 7.21e-07 | 7789.3ms/step | 7.7 samples/s +Step 49270/50462 | Loss: 2.4867 | LR: 7.09e-07 | 5148.2ms/step | 11.7 samples/s +Step 49280/50462 | Loss: 2.4704 | LR: 6.97e-07 | 5152.5ms/step | 11.6 samples/s +Step 49290/50462 | Loss: 2.4920 | LR: 6.85e-07 | 5194.6ms/step | 11.6 samples/s +Step 49300/50462 | Loss: 2.4889 | LR: 6.74e-07 | 5223.6ms/step | 11.5 samples/s +Step 49310/50462 | Loss: 2.4941 | LR: 6.62e-07 | 5203.6ms/step | 11.5 samples/s +Step 49320/50462 | Loss: 2.4835 | LR: 6.51e-07 | 5183.3ms/step | 11.6 samples/s +Step 49330/50462 | Loss: 2.4920 | LR: 6.39e-07 | 5187.0ms/step | 11.6 samples/s +Step 49340/50462 | Loss: 2.5017 | LR: 6.28e-07 | 5196.2ms/step | 11.5 samples/s +Step 49350/50462 | Loss: 2.5048 | LR: 6.17e-07 | 5193.0ms/step | 11.6 samples/s +Step 49360/50462 | Loss: 2.5156 | LR: 6.06e-07 | 5182.1ms/step | 11.6 samples/s +Step 49370/50462 | Loss: 2.4987 | LR: 5.95e-07 | 5173.5ms/step | 11.6 samples/s +Step 49380/50462 | Loss: 2.5103 | LR: 5.84e-07 | 5189.2ms/step | 11.6 samples/s +Step 49390/50462 | Loss: 2.5097 | LR: 5.73e-07 | 5190.0ms/step | 11.6 samples/s +Step 49400/50462 | Loss: 2.5082 | LR: 5.63e-07 | 5186.8ms/step | 11.6 samples/s +Step 49410/50462 | Loss: 2.4995 | LR: 5.52e-07 | 5189.7ms/step | 11.6 samples/s +Step 49420/50462 | Loss: 2.4972 | LR: 5.42e-07 | 5219.5ms/step | 11.5 samples/s +Step 49430/50462 | Loss: 2.4757 | LR: 5.31e-07 | 5216.4ms/step | 11.5 samples/s +Step 49440/50462 | Loss: 2.4860 | LR: 5.21e-07 | 5189.0ms/step | 11.6 samples/s +Step 49450/50462 | Loss: 2.4718 | LR: 5.11e-07 | 5188.9ms/step | 11.6 samples/s +Step 49460/50462 | Loss: 2.5065 | LR: 5.01e-07 | 5198.7ms/step | 11.5 samples/s +Step 49470/50462 | Loss: 2.5080 | LR: 4.91e-07 | 5214.8ms/step | 11.5 samples/s +Step 49480/50462 | Loss: 2.5016 | LR: 4.81e-07 | 5229.8ms/step | 11.5 samples/s +Step 49490/50462 | Loss: 2.4827 | LR: 4.72e-07 | 5203.9ms/step | 11.5 samples/s +Step 49500/50462 | Loss: 2.5033 | LR: 4.62e-07 | 5221.2ms/step | 11.5 samples/s +Validation at step 49500: Loss = 2.5080, Perplexity = 12.28 +Step 49510/50462 | Loss: 2.5121 | LR: 4.52e-07 | 7757.9ms/step | 7.7 samples/s +Step 49520/50462 | Loss: 2.4931 | LR: 4.43e-07 | 5170.9ms/step | 11.6 samples/s +Step 49530/50462 | Loss: 2.4954 | LR: 4.34e-07 | 5170.1ms/step | 11.6 samples/s +Step 49540/50462 | Loss: 2.4876 | LR: 4.24e-07 | 5189.0ms/step | 11.6 samples/s +Step 49550/50462 | Loss: 2.4944 | LR: 4.15e-07 | 5225.3ms/step | 11.5 samples/s +Step 49560/50462 | Loss: 2.5098 | LR: 4.06e-07 | 5180.7ms/step | 11.6 samples/s +Step 49570/50462 | Loss: 2.5115 | LR: 3.97e-07 | 5236.2ms/step | 11.5 samples/s +Step 49580/50462 | Loss: 2.5109 | LR: 3.88e-07 | 5246.9ms/step | 11.4 samples/s +Step 49590/50462 | Loss: 2.4851 | LR: 3.79e-07 | 5183.8ms/step | 11.6 samples/s +Step 49600/50462 | Loss: 2.5158 | LR: 3.71e-07 | 5198.6ms/step | 11.5 samples/s +Step 49610/50462 | Loss: 2.4967 | LR: 3.62e-07 | 5205.1ms/step | 11.5 samples/s +Step 49620/50462 | Loss: 2.4951 | LR: 3.54e-07 | 5192.0ms/step | 11.6 samples/s +Step 49630/50462 | Loss: 2.4948 | LR: 3.45e-07 | 5190.5ms/step | 11.6 samples/s +Step 49640/50462 | Loss: 2.4970 | LR: 3.37e-07 | 5192.7ms/step | 11.6 samples/s +Step 49650/50462 | Loss: 2.5014 | LR: 3.29e-07 | 5164.3ms/step | 11.6 samples/s +Step 49660/50462 | Loss: 2.4835 | LR: 3.21e-07 | 5161.9ms/step | 11.6 samples/s +Step 49670/50462 | Loss: 2.4707 | LR: 3.13e-07 | 5163.4ms/step | 11.6 samples/s +Step 49680/50462 | Loss: 2.5161 | LR: 3.05e-07 | 5143.0ms/step | 11.7 samples/s +Step 49690/50462 | Loss: 2.5336 | LR: 2.97e-07 | 5209.5ms/step | 11.5 samples/s +Step 49700/50462 | Loss: 2.5058 | LR: 2.90e-07 | 5163.0ms/step | 11.6 samples/s +Step 49710/50462 | Loss: 2.4986 | LR: 2.82e-07 | 5210.6ms/step | 11.5 samples/s +Step 49720/50462 | Loss: 2.4890 | LR: 2.75e-07 | 5209.0ms/step | 11.5 samples/s +Step 49730/50462 | Loss: 2.5052 | LR: 2.67e-07 | 5212.4ms/step | 11.5 samples/s +Step 49740/50462 | Loss: 2.5265 | LR: 2.60e-07 | 5199.3ms/step | 11.5 samples/s +Step 49750/50462 | Loss: 2.4974 | LR: 2.53e-07 | 5197.5ms/step | 11.5 samples/s +Validation at step 49750: Loss = 2.5080, Perplexity = 12.28 +Step 49760/50462 | Loss: 2.5151 | LR: 2.46e-07 | 7708.6ms/step | 7.8 samples/s +Step 49770/50462 | Loss: 2.5040 | LR: 2.39e-07 | 5153.6ms/step | 11.6 samples/s +Step 49780/50462 | Loss: 2.4893 | LR: 2.32e-07 | 5175.9ms/step | 11.6 samples/s +Step 49790/50462 | Loss: 2.4820 | LR: 2.25e-07 | 5199.4ms/step | 11.5 samples/s +Step 49800/50462 | Loss: 2.5250 | LR: 2.19e-07 | 5187.9ms/step | 11.6 samples/s +Step 49810/50462 | Loss: 2.5116 | LR: 2.12e-07 | 5173.6ms/step | 11.6 samples/s +Step 49820/50462 | Loss: 2.5087 | LR: 2.06e-07 | 5164.0ms/step | 11.6 samples/s +Step 49830/50462 | Loss: 2.4972 | LR: 1.99e-07 | 5185.8ms/step | 11.6 samples/s +Step 49840/50462 | Loss: 2.5125 | LR: 1.93e-07 | 5225.1ms/step | 11.5 samples/s +Step 49850/50462 | Loss: 2.4887 | LR: 1.87e-07 | 5176.5ms/step | 11.6 samples/s +Step 49860/50462 | Loss: 2.4893 | LR: 1.81e-07 | 5272.6ms/step | 11.4 samples/s +Step 49870/50462 | Loss: 2.5018 | LR: 1.75e-07 | 5166.5ms/step | 11.6 samples/s +Step 49880/50462 | Loss: 2.4896 | LR: 1.69e-07 | 5181.7ms/step | 11.6 samples/s +Step 49890/50462 | Loss: 2.4987 | LR: 1.63e-07 | 5192.6ms/step | 11.6 samples/s +Step 49900/50462 | Loss: 2.5117 | LR: 1.58e-07 | 5188.9ms/step | 11.6 samples/s +Step 49910/50462 | Loss: 2.4974 | LR: 1.52e-07 | 5186.6ms/step | 11.6 samples/s +Step 49920/50462 | Loss: 2.5116 | LR: 1.47e-07 | 5190.6ms/step | 11.6 samples/s +Step 49930/50462 | Loss: 2.4828 | LR: 1.41e-07 | 5175.2ms/step | 11.6 samples/s +Step 49940/50462 | Loss: 2.4943 | LR: 1.36e-07 | 5192.4ms/step | 11.6 samples/s +Step 49950/50462 | Loss: 2.4552 | LR: 1.31e-07 | 5186.2ms/step | 11.6 samples/s +Step 49960/50462 | Loss: 2.4854 | LR: 1.26e-07 | 5185.1ms/step | 11.6 samples/s +Step 49970/50462 | Loss: 2.4765 | LR: 1.21e-07 | 5187.8ms/step | 11.6 samples/s +Step 49980/50462 | Loss: 2.4920 | LR: 1.16e-07 | 5183.2ms/step | 11.6 samples/s +Step 49990/50462 | Loss: 2.4694 | LR: 1.11e-07 | 5202.4ms/step | 11.5 samples/s +Step 50000/50462 | Loss: 2.4808 | LR: 1.07e-07 | 5207.5ms/step | 11.5 samples/s +Validation at step 50000: Loss = 2.5080, Perplexity = 12.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-50000 +Step 50010/50462 | Loss: 2.4757 | LR: 1.02e-07 | 8669.5ms/step | 6.9 samples/s +Step 50020/50462 | Loss: 2.4837 | LR: 9.75e-08 | 5144.5ms/step | 11.7 samples/s +Step 50030/50462 | Loss: 2.5140 | LR: 9.32e-08 | 5156.2ms/step | 11.6 samples/s +Step 50040/50462 | Loss: 2.5445 | LR: 8.89e-08 | 5185.8ms/step | 11.6 samples/s +Step 50050/50462 | Loss: 2.4984 | LR: 8.47e-08 | 5172.0ms/step | 11.6 samples/s +Step 50060/50462 | Loss: 2.4909 | LR: 8.07e-08 | 5226.6ms/step | 11.5 samples/s +Step 50070/50462 | Loss: 2.5107 | LR: 7.67e-08 | 5236.1ms/step | 11.5 samples/s +Step 50080/50462 | Loss: 2.4822 | LR: 7.28e-08 | 5169.8ms/step | 11.6 samples/s +Step 50090/50462 | Loss: 2.4744 | LR: 6.91e-08 | 5152.0ms/step | 11.6 samples/s +Step 50100/50462 | Loss: 2.4813 | LR: 6.54e-08 | 5200.9ms/step | 11.5 samples/s +Step 50110/50462 | Loss: 2.5206 | LR: 6.19e-08 | 5231.2ms/step | 11.5 samples/s +Step 50120/50462 | Loss: 2.5016 | LR: 5.84e-08 | 5198.0ms/step | 11.5 samples/s +Step 50130/50462 | Loss: 2.5023 | LR: 5.50e-08 | 5273.4ms/step | 11.4 samples/s +Step 50140/50462 | Loss: 2.5205 | LR: 5.18e-08 | 5182.6ms/step | 11.6 samples/s +Step 50150/50462 | Loss: 2.4762 | LR: 4.86e-08 | 5241.2ms/step | 11.4 samples/s +Step 50160/50462 | Loss: 2.4920 | LR: 4.55e-08 | 5145.9ms/step | 11.7 samples/s +Step 50170/50462 | Loss: 2.4594 | LR: 4.26e-08 | 5203.5ms/step | 11.5 samples/s +Step 50180/50462 | Loss: 2.4650 | LR: 3.97e-08 | 5176.8ms/step | 11.6 samples/s +Step 50190/50462 | Loss: 2.5111 | LR: 3.69e-08 | 5190.6ms/step | 11.6 samples/s +Step 50200/50462 | Loss: 2.4811 | LR: 3.43e-08 | 5209.6ms/step | 11.5 samples/s +Step 50210/50462 | Loss: 2.5058 | LR: 3.17e-08 | 5189.5ms/step | 11.6 samples/s +Step 50220/50462 | Loss: 2.5015 | LR: 2.92e-08 | 5224.3ms/step | 11.5 samples/s +Step 50230/50462 | Loss: 2.5197 | LR: 2.69e-08 | 5192.3ms/step | 11.6 samples/s +Step 50240/50462 | Loss: 2.4840 | LR: 2.46e-08 | 5198.7ms/step | 11.5 samples/s +Step 50250/50462 | Loss: 2.4774 | LR: 2.24e-08 | 5146.2ms/step | 11.7 samples/s +Validation at step 50250: Loss = 2.5080, Perplexity = 12.28 +Step 50260/50462 | Loss: 2.5251 | LR: 2.04e-08 | 7742.7ms/step | 7.7 samples/s +Step 50270/50462 | Loss: 2.5358 | LR: 1.84e-08 | 5208.4ms/step | 11.5 samples/s +Step 50280/50462 | Loss: 2.5081 | LR: 1.65e-08 | 5198.1ms/step | 11.5 samples/s +Step 50290/50462 | Loss: 2.5209 | LR: 1.48e-08 | 5206.7ms/step | 11.5 samples/s +Step 50300/50462 | Loss: 2.4763 | LR: 1.31e-08 | 5217.7ms/step | 11.5 samples/s +Step 50310/50462 | Loss: 2.4719 | LR: 1.15e-08 | 5171.9ms/step | 11.6 samples/s +Step 50320/50462 | Loss: 2.4915 | LR: 1.01e-08 | 5176.9ms/step | 11.6 samples/s +Step 50330/50462 | Loss: 2.5186 | LR: 8.70e-09 | 5194.1ms/step | 11.6 samples/s +Step 50340/50462 | Loss: 2.5005 | LR: 7.43e-09 | 5173.7ms/step | 11.6 samples/s +Step 50350/50462 | Loss: 2.4818 | LR: 6.26e-09 | 5190.1ms/step | 11.6 samples/s +Step 50360/50462 | Loss: 2.5042 | LR: 5.19e-09 | 5182.3ms/step | 11.6 samples/s +Step 50370/50462 | Loss: 2.4801 | LR: 4.23e-09 | 5186.0ms/step | 11.6 samples/s +Step 50380/50462 | Loss: 2.4720 | LR: 3.36e-09 | 5202.3ms/step | 11.5 samples/s +Step 50390/50462 | Loss: 2.5123 | LR: 2.59e-09 | 5223.0ms/step | 11.5 samples/s +Step 50400/50462 | Loss: 2.4957 | LR: 1.92e-09 | 5219.1ms/step | 11.5 samples/s +Step 50410/50462 | Loss: 2.4848 | LR: 1.35e-09 | 5241.0ms/step | 11.4 samples/s +Step 50420/50462 | Loss: 2.5131 | LR: 8.81e-10 | 5226.1ms/step | 11.5 samples/s +Step 50430/50462 | Loss: 2.5029 | LR: 5.11e-10 | 5191.7ms/step | 11.6 samples/s +Step 50440/50462 | Loss: 2.4731 | LR: 2.42e-10 | 5246.5ms/step | 11.4 samples/s +Step 50450/50462 | Loss: 2.5147 | LR: 7.19e-11 | 5220.4ms/step | 11.5 samples/s +Step 50460/50462 | Loss: 2.4870 | LR: 2.00e-12 | 5197.1ms/step | 11.5 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260523_215035/checkpoint-50462 +wandb: updating run metadata +wandb: uploading wandb-summary.json; uploading config.yaml; uploading output.log +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/loss █▇▆▆▆▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▇▇▇▆▆▆▅▅▅▅▄▄▃▃▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁█▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec █▁█████████████▁█████▇█████████████▁████ +wandb: speed/steps_per_sec █▂████████████████████▇████████▁████████ +wandb: train/epoch ▁▁▁▁▁▂▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▄▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇█ +wandb: train/global_step ▁▁▁▁▁▁▂▂▂▃▄▄▄▄▄▅▅▅▅▅▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇█████ +wandb: train/learning_rate ███▇▆▆▆▆▅▅▅▅▄▄▄▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁ +wandb: train/loss ▆▄▆▇██▆▆▄▆▃▄▆▅▅▄▅▆▄▂▃▄▅▄▃▄▄▄▆▃▄▄▃▃▃▃▂▁▄▅ +wandb: +wandb: Run summary: +wandb: eval/loss 2.508 +wandb: eval/perplexity 12.28036 +wandb: speed/ms_per_step 5197.08314 +wandb: speed/samples_per_sec 11.54494 +wandb: speed/steps_per_sec 0.19242 +wandb: train/epoch 0.99996 +wandb: train/global_step 50460 +wandb: train/learning_rate 0.0 +wandb: train/loss 2.48699 +wandb: +wandb: 🚀 View run dense 1.2b at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/xj5iv7ks +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260525_182157-xj5iv7ks/logs +Training complete! diff --git a/epfl_final/log_files/pretrain_test.log b/epfl_final/log_files/pretrain_test.log new file mode 100644 index 0000000000000000000000000000000000000000..26082d85e198c61818dbcf8f0fe8c2ddf8b44a93 --- /dev/null +++ b/epfl_final/log_files/pretrain_test.log @@ -0,0 +1,940 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +Resuming: Using existing output directory checkpoints/run_20260614_032718 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "checkpoints/run_20260614_032718", + "resume_from_checkpoint": "checkpoints/run_20260614_032718/checkpoint-38000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "sparse 1mask", + "wandb_tags": [ + "qwen3", + "1.2b", + "sparse_stochastic" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: checkpoints/run_20260614_032718/checkpoint-38000 +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from checkpoints/run_20260614_032718/checkpoint-38000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 38000 (epoch 0, micro_step 380000) +Resuming training from step 38000 +Resuming wandb run aqqleyxv +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.2 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260615_161335-aqqleyxv +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run sparse 1mask +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/aqqleyxv +Starting training... +Validation at step 38000: Loss = 2.6437, Perplexity = 14.07 +wandb: WARNING Tried to log to step 38000 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 38001: sparse_stochastic (configured sparse attention) +Step 38010/50462 | Loss: 2.6372 | LR: 7.35e-05 | 65177.4ms/step | 0.9 samples/s +wandb: WARNING Tried to log to step 38010 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38020/50462 | Loss: 2.6248 | LR: 7.34e-05 | 3082.9ms/step | 19.5 samples/s +wandb: WARNING Tried to log to step 38020 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38030/50462 | Loss: 2.6536 | LR: 7.33e-05 | 3093.2ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38030 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38040/50462 | Loss: 2.6222 | LR: 7.32e-05 | 3180.4ms/step | 18.9 samples/s +wandb: WARNING Tried to log to step 38040 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38050/50462 | Loss: 2.6359 | LR: 7.30e-05 | 3102.9ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38050 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38060/50462 | Loss: 2.6340 | LR: 7.29e-05 | 3102.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38060 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38070/50462 | Loss: 2.6759 | LR: 7.28e-05 | 3182.2ms/step | 18.9 samples/s +wandb: WARNING Tried to log to step 38070 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38080/50462 | Loss: 2.6323 | LR: 7.27e-05 | 3101.2ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38080 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38090/50462 | Loss: 2.6310 | LR: 7.26e-05 | 3185.9ms/step | 18.8 samples/s +wandb: WARNING Tried to log to step 38090 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38100/50462 | Loss: 2.6529 | LR: 7.25e-05 | 3102.7ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38100 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38110/50462 | Loss: 2.5917 | LR: 7.24e-05 | 3100.9ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38110 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38120/50462 | Loss: 2.6380 | LR: 7.23e-05 | 3103.8ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38120 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38130/50462 | Loss: 2.6389 | LR: 7.22e-05 | 3105.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38130 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38140/50462 | Loss: 2.6455 | LR: 7.20e-05 | 3101.9ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38140 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38150/50462 | Loss: 2.6601 | LR: 7.19e-05 | 3103.2ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38150 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38160/50462 | Loss: 2.6155 | LR: 7.18e-05 | 3105.5ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38160 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38170/50462 | Loss: 2.6468 | LR: 7.17e-05 | 3104.2ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38170 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38180/50462 | Loss: 2.6445 | LR: 7.16e-05 | 3103.3ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38180 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38190/50462 | Loss: 2.6304 | LR: 7.15e-05 | 3106.4ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38190 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38200/50462 | Loss: 2.6668 | LR: 7.14e-05 | 3098.0ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38200 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38210/50462 | Loss: 2.6550 | LR: 7.13e-05 | 3103.9ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38210 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38220/50462 | Loss: 2.6251 | LR: 7.12e-05 | 3101.2ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38220 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38230/50462 | Loss: 2.6414 | LR: 7.10e-05 | 3102.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38230 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38240/50462 | Loss: 2.6386 | LR: 7.09e-05 | 3098.8ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38240 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38250/50462 | Loss: 2.6448 | LR: 7.08e-05 | 3102.7ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38250 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 38250: Loss = 2.6435, Perplexity = 14.06 +wandb: WARNING Tried to log to step 38250 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38260/50462 | Loss: 2.6255 | LR: 7.07e-05 | 4745.7ms/step | 12.6 samples/s +wandb: WARNING Tried to log to step 38260 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38270/50462 | Loss: 2.6637 | LR: 7.06e-05 | 3100.1ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38270 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38280/50462 | Loss: 2.6036 | LR: 7.05e-05 | 3100.2ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38280 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38290/50462 | Loss: 2.6254 | LR: 7.04e-05 | 3095.9ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38290 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38300/50462 | Loss: 2.6355 | LR: 7.03e-05 | 3099.1ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38300 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38310/50462 | Loss: 2.6625 | LR: 7.02e-05 | 3094.0ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38310 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38320/50462 | Loss: 2.6659 | LR: 7.01e-05 | 3098.7ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38320 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38330/50462 | Loss: 2.6248 | LR: 6.99e-05 | 3102.7ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38330 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38340/50462 | Loss: 2.6298 | LR: 6.98e-05 | 3106.0ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38340 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38350/50462 | Loss: 2.6472 | LR: 6.97e-05 | 3104.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38350 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38360/50462 | Loss: 2.6444 | LR: 6.96e-05 | 3165.3ms/step | 19.0 samples/s +wandb: WARNING Tried to log to step 38360 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38370/50462 | Loss: 2.6431 | LR: 6.95e-05 | 3101.6ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38370 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38380/50462 | Loss: 2.6235 | LR: 6.94e-05 | 3102.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38380 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38390/50462 | Loss: 2.6393 | LR: 6.93e-05 | 3102.5ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38390 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38400/50462 | Loss: 2.6323 | LR: 6.92e-05 | 3165.0ms/step | 19.0 samples/s +wandb: WARNING Tried to log to step 38400 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38410/50462 | Loss: 2.6342 | LR: 6.91e-05 | 3108.5ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38410 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38420/50462 | Loss: 2.6500 | LR: 6.90e-05 | 3163.0ms/step | 19.0 samples/s +wandb: WARNING Tried to log to step 38420 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38430/50462 | Loss: 2.6352 | LR: 6.89e-05 | 3102.7ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38430 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38440/50462 | Loss: 2.6230 | LR: 6.87e-05 | 3104.4ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38440 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38450/50462 | Loss: 2.6454 | LR: 6.86e-05 | 3102.4ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38450 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38460/50462 | Loss: 2.6300 | LR: 6.85e-05 | 3101.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38460 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38470/50462 | Loss: 2.6327 | LR: 6.84e-05 | 3104.9ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38470 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38480/50462 | Loss: 2.6224 | LR: 6.83e-05 | 3102.8ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38480 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38490/50462 | Loss: 2.6616 | LR: 6.82e-05 | 3103.8ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38490 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38500/50462 | Loss: 2.6361 | LR: 6.81e-05 | 3103.8ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38500 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 38500: Loss = 2.6431, Perplexity = 14.06 +wandb: WARNING Tried to log to step 38500 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38510/50462 | Loss: 2.6342 | LR: 6.80e-05 | 4750.1ms/step | 12.6 samples/s +wandb: WARNING Tried to log to step 38510 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38520/50462 | Loss: 2.6211 | LR: 6.79e-05 | 3104.7ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38520 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38530/50462 | Loss: 2.6270 | LR: 6.78e-05 | 3101.4ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38530 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38540/50462 | Loss: 2.6055 | LR: 6.77e-05 | 3104.0ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38540 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38550/50462 | Loss: 2.6426 | LR: 6.76e-05 | 3102.4ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38550 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38560/50462 | Loss: 2.6174 | LR: 6.74e-05 | 3106.2ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38560 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38570/50462 | Loss: 2.6192 | LR: 6.73e-05 | 3103.9ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38570 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38580/50462 | Loss: 2.6287 | LR: 6.72e-05 | 3105.5ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38580 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38590/50462 | Loss: 2.6305 | LR: 6.71e-05 | 3106.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38590 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38600/50462 | Loss: 2.6362 | LR: 6.70e-05 | 3105.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38600 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38610/50462 | Loss: 2.6111 | LR: 6.69e-05 | 3101.4ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38610 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38620/50462 | Loss: 2.6210 | LR: 6.68e-05 | 3105.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38620 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38630/50462 | Loss: 2.6446 | LR: 6.67e-05 | 3102.9ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38630 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38640/50462 | Loss: 2.6216 | LR: 6.66e-05 | 3105.8ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38640 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38650/50462 | Loss: 2.6492 | LR: 6.65e-05 | 3103.1ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38650 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38660/50462 | Loss: 2.6194 | LR: 6.64e-05 | 3102.0ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38660 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38670/50462 | Loss: 2.6255 | LR: 6.63e-05 | 3093.1ms/step | 19.4 samples/s +wandb: WARNING Tried to log to step 38670 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38680/50462 | Loss: 2.6144 | LR: 6.62e-05 | 3102.9ms/step | 19.3 samples/s +wandb: WARNING Tried to log to step 38680 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 38690/50462 | Loss: 2.5945 | LR: 6.60e-05 | 3160.8ms/step | 19.0 samples/s +Step 38700/50462 | Loss: 2.6453 | LR: 6.59e-05 | 3107.2ms/step | 19.3 samples/s +Step 38710/50462 | Loss: 2.6212 | LR: 6.58e-05 | 3108.5ms/step | 19.3 samples/s +Step 38720/50462 | Loss: 2.6206 | LR: 6.57e-05 | 3093.7ms/step | 19.4 samples/s +Step 38730/50462 | Loss: 2.6654 | LR: 6.56e-05 | 3161.6ms/step | 19.0 samples/s +Step 38740/50462 | Loss: 2.6295 | LR: 6.55e-05 | 3104.8ms/step | 19.3 samples/s +Step 38750/50462 | Loss: 2.6357 | LR: 6.54e-05 | 3153.5ms/step | 19.0 samples/s +Validation at step 38750: Loss = 2.6427, Perplexity = 14.05 +Step 38760/50462 | Loss: 2.6332 | LR: 6.53e-05 | 4737.8ms/step | 12.7 samples/s +Step 38770/50462 | Loss: 2.6514 | LR: 6.52e-05 | 3106.1ms/step | 19.3 samples/s +Step 38780/50462 | Loss: 2.6232 | LR: 6.51e-05 | 3107.4ms/step | 19.3 samples/s +Step 38790/50462 | Loss: 2.6761 | LR: 6.50e-05 | 3105.6ms/step | 19.3 samples/s +Step 38800/50462 | Loss: 2.6396 | LR: 6.49e-05 | 3104.2ms/step | 19.3 samples/s +Step 38810/50462 | Loss: 2.6241 | LR: 6.48e-05 | 3106.3ms/step | 19.3 samples/s +Step 38820/50462 | Loss: 2.6285 | LR: 6.47e-05 | 3100.0ms/step | 19.4 samples/s +Step 38830/50462 | Loss: 2.6349 | LR: 6.46e-05 | 3092.5ms/step | 19.4 samples/s +Step 38840/50462 | Loss: 2.6476 | LR: 6.45e-05 | 3101.2ms/step | 19.3 samples/s +Step 38850/50462 | Loss: 2.6546 | LR: 6.43e-05 | 3099.4ms/step | 19.4 samples/s +Step 38860/50462 | Loss: 2.6219 | LR: 6.42e-05 | 3103.6ms/step | 19.3 samples/s +Step 38870/50462 | Loss: 2.6352 | LR: 6.41e-05 | 3103.7ms/step | 19.3 samples/s +Step 38880/50462 | Loss: 2.6302 | LR: 6.40e-05 | 3088.0ms/step | 19.4 samples/s +Step 38890/50462 | Loss: 2.6396 | LR: 6.39e-05 | 3096.0ms/step | 19.4 samples/s +Step 38900/50462 | Loss: 2.6379 | LR: 6.38e-05 | 3102.8ms/step | 19.3 samples/s +Step 38910/50462 | Loss: 2.6392 | LR: 6.37e-05 | 3098.2ms/step | 19.4 samples/s +Step 38920/50462 | Loss: 2.6302 | LR: 6.36e-05 | 3103.9ms/step | 19.3 samples/s +Step 38930/50462 | Loss: 2.6451 | LR: 6.35e-05 | 3106.4ms/step | 19.3 samples/s +Step 38940/50462 | Loss: 2.6385 | LR: 6.34e-05 | 3105.8ms/step | 19.3 samples/s +Step 38950/50462 | Loss: 2.6476 | LR: 6.33e-05 | 3104.1ms/step | 19.3 samples/s +Step 38960/50462 | Loss: 2.6291 | LR: 6.32e-05 | 3112.1ms/step | 19.3 samples/s +Step 38970/50462 | Loss: 2.6216 | LR: 6.31e-05 | 3108.4ms/step | 19.3 samples/s +Step 38980/50462 | Loss: 2.6327 | LR: 6.30e-05 | 3101.7ms/step | 19.3 samples/s +Step 38990/50462 | Loss: 2.6079 | LR: 6.29e-05 | 3104.3ms/step | 19.3 samples/s +Step 39000/50462 | Loss: 2.6368 | LR: 6.28e-05 | 3096.0ms/step | 19.4 samples/s +Validation at step 39000: Loss = 2.6424, Perplexity = 14.05 +Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-39000 +Step 39010/50462 | Loss: 2.6528 | LR: 6.27e-05 | 5767.6ms/step | 10.4 samples/s +Step 39020/50462 | Loss: 2.6120 | LR: 6.26e-05 | 3098.6ms/step | 19.4 samples/s +Step 39030/50462 | Loss: 2.6402 | LR: 6.25e-05 | 3105.3ms/step | 19.3 samples/s +Step 39040/50462 | Loss: 2.6268 | LR: 6.23e-05 | 3112.1ms/step | 19.3 samples/s +Step 39050/50462 | Loss: 2.6069 | LR: 6.22e-05 | 3176.2ms/step | 18.9 samples/s +Step 39060/50462 | Loss: 2.6179 | LR: 6.21e-05 | 3109.0ms/step | 19.3 samples/s +Step 39070/50462 | Loss: 2.6384 | LR: 6.20e-05 | 3166.1ms/step | 19.0 samples/s +Step 39080/50462 | Loss: 2.6572 | LR: 6.19e-05 | 3103.1ms/step | 19.3 samples/s +Step 39090/50462 | Loss: 2.6121 | LR: 6.18e-05 | 3107.5ms/step | 19.3 samples/s +Step 39100/50462 | Loss: 2.6351 | LR: 6.17e-05 | 3105.5ms/step | 19.3 samples/s +Step 39110/50462 | Loss: 2.6112 | LR: 6.16e-05 | 3105.5ms/step | 19.3 samples/s +Step 39120/50462 | Loss: 2.6188 | LR: 6.15e-05 | 3108.3ms/step | 19.3 samples/s +Step 39130/50462 | Loss: 2.6537 | LR: 6.14e-05 | 3106.1ms/step | 19.3 samples/s +Step 39140/50462 | Loss: 2.6276 | LR: 6.13e-05 | 3109.5ms/step | 19.3 samples/s +Step 39150/50462 | Loss: 2.6292 | LR: 6.12e-05 | 3108.1ms/step | 19.3 samples/s +Step 39160/50462 | Loss: 2.6176 | LR: 6.11e-05 | 3110.6ms/step | 19.3 samples/s +Step 39170/50462 | Loss: 2.6455 | LR: 6.10e-05 | 3109.2ms/step | 19.3 samples/s +Step 39180/50462 | Loss: 2.6548 | LR: 6.09e-05 | 3111.3ms/step | 19.3 samples/s +Step 39190/50462 | Loss: 2.6373 | LR: 6.08e-05 | 3093.5ms/step | 19.4 samples/s +Step 39200/50462 | Loss: 2.6099 | LR: 6.07e-05 | 3103.4ms/step | 19.3 samples/s +Step 39210/50462 | Loss: 2.6291 | LR: 6.06e-05 | 3103.2ms/step | 19.3 samples/s +Step 39220/50462 | Loss: 2.6030 | LR: 6.05e-05 | 3100.2ms/step | 19.4 samples/s +Step 39230/50462 | Loss: 2.6506 | LR: 6.04e-05 | 3105.8ms/step | 19.3 samples/s +Step 39240/50462 | Loss: 2.6270 | LR: 6.03e-05 | 3110.1ms/step | 19.3 samples/s +Step 39250/50462 | Loss: 2.6464 | LR: 6.02e-05 | 3105.9ms/step | 19.3 samples/s +Validation at step 39250: Loss = 2.6422, Perplexity = 14.04 +Step 39260/50462 | Loss: 2.6306 | LR: 6.01e-05 | 4748.9ms/step | 12.6 samples/s +Step 39270/50462 | Loss: 2.6341 | LR: 6.00e-05 | 3105.2ms/step | 19.3 samples/s +Step 39280/50462 | Loss: 2.6461 | LR: 5.99e-05 | 3109.1ms/step | 19.3 samples/s +Step 39290/50462 | Loss: 2.6252 | LR: 5.98e-05 | 3104.6ms/step | 19.3 samples/s +Step 39300/50462 | Loss: 2.6466 | LR: 5.97e-05 | 3108.2ms/step | 19.3 samples/s +Step 39310/50462 | Loss: 2.6364 | LR: 5.96e-05 | 3108.1ms/step | 19.3 samples/s +Step 39320/50462 | Loss: 2.5891 | LR: 5.95e-05 | 3102.4ms/step | 19.3 samples/s +Step 39330/50462 | Loss: 2.6347 | LR: 5.94e-05 | 3169.6ms/step | 18.9 samples/s +Step 39340/50462 | Loss: 2.6379 | LR: 5.93e-05 | 3100.1ms/step | 19.4 samples/s +Step 39350/50462 | Loss: 2.6316 | LR: 5.91e-05 | 3099.8ms/step | 19.4 samples/s +Step 39360/50462 | Loss: 2.6088 | LR: 5.90e-05 | 3098.5ms/step | 19.4 samples/s +Step 39370/50462 | Loss: 2.6354 | LR: 5.89e-05 | 3106.0ms/step | 19.3 samples/s +Step 39380/50462 | Loss: 2.6223 | LR: 5.88e-05 | 3159.4ms/step | 19.0 samples/s +Step 39390/50462 | Loss: 2.6365 | LR: 5.87e-05 | 3102.1ms/step | 19.3 samples/s +Step 39400/50462 | Loss: 2.6451 | LR: 5.86e-05 | 3166.4ms/step | 18.9 samples/s +Step 39410/50462 | Loss: 2.5963 | LR: 5.85e-05 | 3107.7ms/step | 19.3 samples/s +Step 39420/50462 | Loss: 2.6139 | LR: 5.84e-05 | 3107.6ms/step | 19.3 samples/s +Step 39430/50462 | Loss: 2.6114 | LR: 5.83e-05 | 3108.3ms/step | 19.3 samples/s +Step 39440/50462 | Loss: 2.6301 | LR: 5.82e-05 | 3107.6ms/step | 19.3 samples/s +Step 39450/50462 | Loss: 2.6201 | LR: 5.81e-05 | 3107.5ms/step | 19.3 samples/s +Step 39460/50462 | Loss: 2.6346 | LR: 5.80e-05 | 3096.2ms/step | 19.4 samples/s +Step 39470/50462 | Loss: 2.6693 | LR: 5.79e-05 | 3107.1ms/step | 19.3 samples/s +Step 39480/50462 | Loss: 2.6531 | LR: 5.78e-05 | 3105.1ms/step | 19.3 samples/s +Step 39490/50462 | Loss: 2.6243 | LR: 5.77e-05 | 3107.5ms/step | 19.3 samples/s +Step 39500/50462 | Loss: 2.6363 | LR: 5.76e-05 | 3107.6ms/step | 19.3 samples/s +Validation at step 39500: Loss = 2.6419, Perplexity = 14.04 +Step 39510/50462 | Loss: 2.6362 | LR: 5.75e-05 | 4745.3ms/step | 12.6 samples/s +Step 39520/50462 | Loss: 2.6455 | LR: 5.74e-05 | 3092.5ms/step | 19.4 samples/s +Step 39530/50462 | Loss: 2.6347 | LR: 5.73e-05 | 3104.4ms/step | 19.3 samples/s +Step 39540/50462 | Loss: 2.6335 | LR: 5.72e-05 | 3109.8ms/step | 19.3 samples/s +Step 39550/50462 | Loss: 2.6375 | LR: 5.71e-05 | 3106.2ms/step | 19.3 samples/s +Step 39560/50462 | Loss: 2.6295 | LR: 5.70e-05 | 3110.5ms/step | 19.3 samples/s +Step 39570/50462 | Loss: 2.6386 | LR: 5.69e-05 | 3108.8ms/step | 19.3 samples/s +Step 39580/50462 | Loss: 2.6195 | LR: 5.68e-05 | 3111.5ms/step | 19.3 samples/s +Step 39590/50462 | Loss: 2.6433 | LR: 5.67e-05 | 3111.1ms/step | 19.3 samples/s +Step 39600/50462 | Loss: 2.6273 | LR: 5.66e-05 | 3113.4ms/step | 19.3 samples/s +Step 39610/50462 | Loss: 2.6192 | LR: 5.65e-05 | 3096.5ms/step | 19.4 samples/s +Step 39620/50462 | Loss: 2.6145 | LR: 5.64e-05 | 3105.3ms/step | 19.3 samples/s +Step 39630/50462 | Loss: 2.6042 | LR: 5.63e-05 | 3105.0ms/step | 19.3 samples/s +Step 39640/50462 | Loss: 2.6600 | LR: 5.62e-05 | 3099.7ms/step | 19.4 samples/s +Step 39650/50462 | Loss: 2.6344 | LR: 5.61e-05 | 3100.7ms/step | 19.4 samples/s +Step 39660/50462 | Loss: 2.6342 | LR: 5.60e-05 | 3146.4ms/step | 19.1 samples/s +Step 39670/50462 | Loss: 2.6355 | LR: 5.59e-05 | 3090.3ms/step | 19.4 samples/s +Step 39680/50462 | Loss: 2.6455 | LR: 5.58e-05 | 3098.5ms/step | 19.4 samples/s +Step 39690/50462 | Loss: 2.6268 | LR: 5.57e-05 | 3103.1ms/step | 19.3 samples/s +Step 39700/50462 | Loss: 2.6238 | LR: 5.56e-05 | 3171.7ms/step | 18.9 samples/s +Step 39710/50462 | Loss: 2.6153 | LR: 5.55e-05 | 3098.8ms/step | 19.4 samples/s +Step 39720/50462 | Loss: 2.6079 | LR: 5.54e-05 | 3161.7ms/step | 19.0 samples/s +Step 39730/50462 | Loss: 2.6075 | LR: 5.53e-05 | 3104.6ms/step | 19.3 samples/s +Step 39740/50462 | Loss: 2.6299 | LR: 5.52e-05 | 3107.4ms/step | 19.3 samples/s +Step 39750/50462 | Loss: 2.6408 | LR: 5.51e-05 | 3109.0ms/step | 19.3 samples/s +Validation at step 39750: Loss = 2.6417, Perplexity = 14.04 +Step 39760/50462 | Loss: 2.6271 | LR: 5.50e-05 | 4737.5ms/step | 12.7 samples/s +Step 39770/50462 | Loss: 2.6507 | LR: 5.49e-05 | 3099.7ms/step | 19.4 samples/s +Step 39780/50462 | Loss: 2.6530 | LR: 5.48e-05 | 3105.6ms/step | 19.3 samples/s +Step 39790/50462 | Loss: 2.6272 | LR: 5.47e-05 | 3105.5ms/step | 19.3 samples/s +Step 39800/50462 | Loss: 2.6212 | LR: 5.46e-05 | 3107.5ms/step | 19.3 samples/s +Step 39810/50462 | Loss: 2.6340 | LR: 5.45e-05 | 3086.7ms/step | 19.4 samples/s +Step 39820/50462 | Loss: 2.6545 | LR: 5.44e-05 | 3096.2ms/step | 19.4 samples/s +Step 39830/50462 | Loss: 2.6266 | LR: 5.43e-05 | 3098.4ms/step | 19.4 samples/s +Step 39840/50462 | Loss: 2.6373 | LR: 5.42e-05 | 3098.4ms/step | 19.4 samples/s +Step 39850/50462 | Loss: 2.6453 | LR: 5.41e-05 | 3100.5ms/step | 19.4 samples/s +Step 39860/50462 | Loss: 2.6087 | LR: 5.40e-05 | 3107.2ms/step | 19.3 samples/s +Step 39870/50462 | Loss: 2.6347 | LR: 5.39e-05 | 3100.1ms/step | 19.4 samples/s +Step 39880/50462 | Loss: 2.6325 | LR: 5.38e-05 | 3095.0ms/step | 19.4 samples/s +Step 39890/50462 | Loss: 2.6401 | LR: 5.38e-05 | 3095.8ms/step | 19.4 samples/s +Step 39900/50462 | Loss: 2.6237 | LR: 5.37e-05 | 3096.0ms/step | 19.4 samples/s +Step 39910/50462 | Loss: 2.6303 | LR: 5.36e-05 | 3098.2ms/step | 19.4 samples/s +Step 39920/50462 | Loss: 2.6290 | LR: 5.35e-05 | 3104.0ms/step | 19.3 samples/s +Step 39930/50462 | Loss: 2.6326 | LR: 5.34e-05 | 3110.8ms/step | 19.3 samples/s +Step 39940/50462 | Loss: 2.6269 | LR: 5.33e-05 | 3106.9ms/step | 19.3 samples/s +Step 39950/50462 | Loss: 2.6144 | LR: 5.32e-05 | 3096.8ms/step | 19.4 samples/s +Step 39960/50462 | Loss: 2.6312 | LR: 5.31e-05 | 3103.1ms/step | 19.3 samples/s +Step 39970/50462 | Loss: 2.6558 | LR: 5.30e-05 | 3098.7ms/step | 19.4 samples/s +Step 39980/50462 | Loss: 2.6311 | LR: 5.29e-05 | 3102.5ms/step | 19.3 samples/s +Step 39990/50462 | Loss: 2.6671 | LR: 5.28e-05 | 3156.3ms/step | 19.0 samples/s +Step 40000/50462 | Loss: 2.6052 | LR: 5.27e-05 | 3097.6ms/step | 19.4 samples/s +Validation at step 40000: Loss = 2.6415, Perplexity = 14.03 +Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-40000 +Step 40010/50462 | Loss: 2.6490 | LR: 5.26e-05 | 5679.6ms/step | 10.6 samples/s +Step 40020/50462 | Loss: 2.6703 | LR: 5.25e-05 | 3090.2ms/step | 19.4 samples/s +Step 40030/50462 | Loss: 2.6162 | LR: 5.24e-05 | 3162.4ms/step | 19.0 samples/s +Step 40040/50462 | Loss: 2.6445 | LR: 5.23e-05 | 3099.4ms/step | 19.4 samples/s +Step 40050/50462 | Loss: 2.6335 | LR: 5.22e-05 | 3159.7ms/step | 19.0 samples/s +Step 40060/50462 | Loss: 2.6271 | LR: 5.21e-05 | 3103.0ms/step | 19.3 samples/s +Step 40070/50462 | Loss: 2.6395 | LR: 5.20e-05 | 3110.1ms/step | 19.3 samples/s +Step 40080/50462 | Loss: 2.6135 | LR: 5.19e-05 | 3113.3ms/step | 19.3 samples/s +Step 40090/50462 | Loss: 2.6480 | LR: 5.18e-05 | 3111.3ms/step | 19.3 samples/s +Step 40100/50462 | Loss: 2.6454 | LR: 5.17e-05 | 3108.5ms/step | 19.3 samples/s +Step 40110/50462 | Loss: 2.6325 | LR: 5.16e-05 | 3110.6ms/step | 19.3 samples/s +Step 40120/50462 | Loss: 2.6450 | LR: 5.15e-05 | 3113.1ms/step | 19.3 samples/s +Step 40130/50462 | Loss: 2.6260 | LR: 5.14e-05 | 3112.1ms/step | 19.3 samples/s +Step 40140/50462 | Loss: 2.6237 | LR: 5.13e-05 | 3110.3ms/step | 19.3 samples/s +Step 40150/50462 | Loss: 2.6010 | LR: 5.12e-05 | 3108.7ms/step | 19.3 samples/s +Step 40160/50462 | Loss: 2.5960 | LR: 5.11e-05 | 3114.7ms/step | 19.3 samples/s +Step 40170/50462 | Loss: 2.6377 | LR: 5.10e-05 | 3111.2ms/step | 19.3 samples/s +Step 40180/50462 | Loss: 2.6411 | LR: 5.09e-05 | 3103.1ms/step | 19.3 samples/s +Step 40190/50462 | Loss: 2.6388 | LR: 5.09e-05 | 3104.4ms/step | 19.3 samples/s +Step 40200/50462 | Loss: 2.6457 | LR: 5.08e-05 | 3100.5ms/step | 19.4 samples/s +Step 40210/50462 | Loss: 2.6515 | LR: 5.07e-05 | 3111.0ms/step | 19.3 samples/s +Step 40220/50462 | Loss: 2.6501 | LR: 5.06e-05 | 3105.4ms/step | 19.3 samples/s +Step 40230/50462 | Loss: 2.6434 | LR: 5.05e-05 | 3111.1ms/step | 19.3 samples/s +Step 40240/50462 | Loss: 2.6716 | LR: 5.04e-05 | 3112.6ms/step | 19.3 samples/s +Step 40250/50462 | Loss: 2.6310 | LR: 5.03e-05 | 3110.7ms/step | 19.3 samples/s +Validation at step 40250: Loss = 2.6413, Perplexity = 14.03 +Step 40260/50462 | Loss: 2.6382 | LR: 5.02e-05 | 4753.9ms/step | 12.6 samples/s +Step 40270/50462 | Loss: 2.6129 | LR: 5.01e-05 | 3110.8ms/step | 19.3 samples/s +Step 40280/50462 | Loss: 2.6371 | LR: 5.00e-05 | 3097.7ms/step | 19.4 samples/s +Step 40290/50462 | Loss: 2.6331 | LR: 4.99e-05 | 3102.6ms/step | 19.3 samples/s +Step 40300/50462 | Loss: 2.6288 | LR: 4.98e-05 | 3109.0ms/step | 19.3 samples/s +Step 40310/50462 | Loss: 2.6168 | LR: 4.97e-05 | 3169.3ms/step | 18.9 samples/s +Step 40320/50462 | Loss: 2.6343 | LR: 4.96e-05 | 3110.7ms/step | 19.3 samples/s +Step 40330/50462 | Loss: 2.6565 | LR: 4.95e-05 | 3112.3ms/step | 19.3 samples/s +Step 40340/50462 | Loss: 2.6189 | LR: 4.94e-05 | 3110.0ms/step | 19.3 samples/s +Step 40350/50462 | Loss: 2.6102 | LR: 4.93e-05 | 3107.0ms/step | 19.3 samples/s +Step 40360/50462 | Loss: 2.6350 | LR: 4.92e-05 | 3169.5ms/step | 18.9 samples/s +Step 40370/50462 | Loss: 2.6500 | LR: 4.91e-05 | 3101.0ms/step | 19.3 samples/s +Step 40380/50462 | Loss: 2.6362 | LR: 4.90e-05 | 3161.5ms/step | 19.0 samples/s +Step 40390/50462 | Loss: 2.6318 | LR: 4.90e-05 | 3106.9ms/step | 19.3 samples/s +Step 40400/50462 | Loss: 2.6466 | LR: 4.89e-05 | 3104.9ms/step | 19.3 samples/s +Step 40410/50462 | Loss: 2.6596 | LR: 4.88e-05 | 3107.5ms/step | 19.3 samples/s +Step 40420/50462 | Loss: 2.6568 | LR: 4.87e-05 | 3104.3ms/step | 19.3 samples/s +Step 40430/50462 | Loss: 2.6259 | LR: 4.86e-05 | 3105.6ms/step | 19.3 samples/s +Step 40440/50462 | Loss: 2.6427 | LR: 4.85e-05 | 3096.3ms/step | 19.4 samples/s +Step 40450/50462 | Loss: 2.6338 | LR: 4.84e-05 | 3103.3ms/step | 19.3 samples/s +Step 40460/50462 | Loss: 2.6674 | LR: 4.83e-05 | 3105.9ms/step | 19.3 samples/s +Step 40470/50462 | Loss: 2.6192 | LR: 4.82e-05 | 3100.9ms/step | 19.3 samples/s +Step 40480/50462 | Loss: 2.6301 | LR: 4.81e-05 | 3103.8ms/step | 19.3 samples/s +Step 40490/50462 | Loss: 2.6437 | LR: 4.80e-05 | 3104.6ms/step | 19.3 samples/s +Step 40500/50462 | Loss: 2.6379 | LR: 4.79e-05 | 3113.6ms/step | 19.3 samples/s +Validation at step 40500: Loss = 2.6412, Perplexity = 14.03 +Step 40510/50462 | Loss: 2.6354 | LR: 4.78e-05 | 4758.1ms/step | 12.6 samples/s +Step 40520/50462 | Loss: 2.6201 | LR: 4.77e-05 | 3109.1ms/step | 19.3 samples/s +Step 40530/50462 | Loss: 2.6251 | LR: 4.76e-05 | 3102.2ms/step | 19.3 samples/s +Step 40540/50462 | Loss: 2.5998 | LR: 4.76e-05 | 3108.0ms/step | 19.3 samples/s +Step 40550/50462 | Loss: 2.6508 | LR: 4.75e-05 | 3109.8ms/step | 19.3 samples/s +Step 40560/50462 | Loss: 2.6438 | LR: 4.74e-05 | 3109.7ms/step | 19.3 samples/s +Step 40570/50462 | Loss: 2.6204 | LR: 4.73e-05 | 3107.7ms/step | 19.3 samples/s +Step 40580/50462 | Loss: 2.6464 | LR: 4.72e-05 | 3107.6ms/step | 19.3 samples/s +Step 40590/50462 | Loss: 2.6308 | LR: 4.71e-05 | 3099.8ms/step | 19.4 samples/s +Step 40600/50462 | Loss: 2.6320 | LR: 4.70e-05 | 3103.7ms/step | 19.3 samples/s +Step 40610/50462 | Loss: 2.6518 | LR: 4.69e-05 | 3100.9ms/step | 19.3 samples/s +Step 40620/50462 | Loss: 2.6081 | LR: 4.68e-05 | 3104.7ms/step | 19.3 samples/s +Step 40630/50462 | Loss: 2.6257 | LR: 4.67e-05 | 3106.7ms/step | 19.3 samples/s +Step 40640/50462 | Loss: 2.6367 | LR: 4.66e-05 | 3171.5ms/step | 18.9 samples/s +Step 40650/50462 | Loss: 2.6395 | LR: 4.65e-05 | 3112.4ms/step | 19.3 samples/s +Step 40660/50462 | Loss: 2.6493 | LR: 4.64e-05 | 3106.2ms/step | 19.3 samples/s +Step 40670/50462 | Loss: 2.6275 | LR: 4.64e-05 | 3105.5ms/step | 19.3 samples/s +Step 40680/50462 | Loss: 2.6334 | LR: 4.63e-05 | 3173.7ms/step | 18.9 samples/s +Step 40690/50462 | Loss: 2.6264 | LR: 4.62e-05 | 3098.4ms/step | 19.4 samples/s +Step 40700/50462 | Loss: 2.6013 | LR: 4.61e-05 | 3165.6ms/step | 19.0 samples/s +Step 40710/50462 | Loss: 2.6432 | LR: 4.60e-05 | 3103.9ms/step | 19.3 samples/s +Step 40720/50462 | Loss: 2.6059 | LR: 4.59e-05 | 3102.3ms/step | 19.3 samples/s +Step 40730/50462 | Loss: 2.6162 | LR: 4.58e-05 | 3105.9ms/step | 19.3 samples/s +Step 40740/50462 | Loss: 2.6157 | LR: 4.57e-05 | 3113.2ms/step | 19.3 samples/s +Step 40750/50462 | Loss: 2.6254 | LR: 4.56e-05 | 3110.3ms/step | 19.3 samples/s +Validation at step 40750: Loss = 2.6409, Perplexity = 14.03 +Step 40760/50462 | Loss: 2.6512 | LR: 4.55e-05 | 4752.8ms/step | 12.6 samples/s +Step 40770/50462 | Loss: 2.6311 | LR: 4.54e-05 | 3107.3ms/step | 19.3 samples/s +Step 40780/50462 | Loss: 2.6454 | LR: 4.54e-05 | 3099.4ms/step | 19.4 samples/s +Step 40790/50462 | Loss: 2.6283 | LR: 4.53e-05 | 3100.6ms/step | 19.4 samples/s +Step 40800/50462 | Loss: 2.6387 | LR: 4.52e-05 | 3100.8ms/step | 19.3 samples/s +Step 40810/50462 | Loss: 2.5951 | LR: 4.51e-05 | 3104.0ms/step | 19.3 samples/s +Step 40820/50462 | Loss: 2.6696 | LR: 4.50e-05 | 3094.9ms/step | 19.4 samples/s +Step 40830/50462 | Loss: 2.5875 | LR: 4.49e-05 | 3101.2ms/step | 19.3 samples/s +Step 40840/50462 | Loss: 2.6108 | LR: 4.48e-05 | 3097.4ms/step | 19.4 samples/s +Step 40850/50462 | Loss: 2.6535 | LR: 4.47e-05 | 3106.8ms/step | 19.3 samples/s +Step 40860/50462 | Loss: 2.5977 | LR: 4.46e-05 | 3106.4ms/step | 19.3 samples/s +Step 40870/50462 | Loss: 2.6558 | LR: 4.45e-05 | 3113.3ms/step | 19.3 samples/s +Step 40880/50462 | Loss: 2.6613 | LR: 4.45e-05 | 3110.2ms/step | 19.3 samples/s +Step 40890/50462 | Loss: 2.5926 | LR: 4.44e-05 | 3114.1ms/step | 19.3 samples/s +Step 40900/50462 | Loss: 2.6177 | LR: 4.43e-05 | 3094.3ms/step | 19.4 samples/s +Step 40910/50462 | Loss: 2.6397 | LR: 4.42e-05 | 3094.7ms/step | 19.4 samples/s +Step 40920/50462 | Loss: 2.6467 | LR: 4.41e-05 | 3097.5ms/step | 19.4 samples/s +Step 40930/50462 | Loss: 2.6377 | LR: 4.40e-05 | 3105.9ms/step | 19.3 samples/s +Step 40940/50462 | Loss: 2.6555 | LR: 4.39e-05 | 3107.3ms/step | 19.3 samples/s +Step 40950/50462 | Loss: 2.6094 | LR: 4.38e-05 | 3100.3ms/step | 19.4 samples/s +Step 40960/50462 | Loss: 2.6384 | LR: 4.37e-05 | 3159.1ms/step | 19.0 samples/s +Step 40970/50462 | Loss: 2.6288 | LR: 4.36e-05 | 3096.6ms/step | 19.4 samples/s +Step 40980/50462 | Loss: 2.6306 | LR: 4.36e-05 | 3101.0ms/step | 19.3 samples/s +Step 40990/50462 | Loss: 2.5982 | LR: 4.35e-05 | 3102.4ms/step | 19.3 samples/s +Step 41000/50462 | Loss: 2.6444 | LR: 4.34e-05 | 3105.8ms/step | 19.3 samples/s +Validation at step 41000: Loss = 2.6407, Perplexity = 14.02 +Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-41000 +Step 41010/50462 | Loss: 2.6469 | LR: 4.33e-05 | 5733.0ms/step | 10.5 samples/s +Step 41020/50462 | Loss: 2.6459 | LR: 4.32e-05 | 3092.3ms/step | 19.4 samples/s +Step 41030/50462 | Loss: 2.6474 | LR: 4.31e-05 | 3160.7ms/step | 19.0 samples/s +Step 41040/50462 | Loss: 2.6125 | LR: 4.30e-05 | 3102.9ms/step | 19.3 samples/s +Step 41050/50462 | Loss: 2.6447 | LR: 4.29e-05 | 3100.8ms/step | 19.3 samples/s +Step 41060/50462 | Loss: 2.6494 | LR: 4.28e-05 | 3102.2ms/step | 19.3 samples/s +Step 41070/50462 | Loss: 2.6253 | LR: 4.28e-05 | 3108.5ms/step | 19.3 samples/s +Step 41080/50462 | Loss: 2.6180 | LR: 4.27e-05 | 3105.8ms/step | 19.3 samples/s +Step 41090/50462 | Loss: 2.5981 | LR: 4.26e-05 | 3109.2ms/step | 19.3 samples/s +Step 41100/50462 | Loss: 2.6309 | LR: 4.25e-05 | 3107.1ms/step | 19.3 samples/s +Step 41110/50462 | Loss: 2.6336 | LR: 4.24e-05 | 3101.8ms/step | 19.3 samples/s +Step 41120/50462 | Loss: 2.6261 | LR: 4.23e-05 | 3103.2ms/step | 19.3 samples/s +Step 41130/50462 | Loss: 2.6496 | LR: 4.22e-05 | 3108.4ms/step | 19.3 samples/s +Step 41140/50462 | Loss: 2.6220 | LR: 4.21e-05 | 3103.7ms/step | 19.3 samples/s +Step 41150/50462 | Loss: 2.6335 | LR: 4.21e-05 | 3105.2ms/step | 19.3 samples/s +Step 41160/50462 | Loss: 2.6344 | LR: 4.20e-05 | 3106.3ms/step | 19.3 samples/s +Step 41170/50462 | Loss: 2.6412 | LR: 4.19e-05 | 3110.5ms/step | 19.3 samples/s +Step 41180/50462 | Loss: 2.6224 | LR: 4.18e-05 | 3110.1ms/step | 19.3 samples/s +Step 41190/50462 | Loss: 2.5813 | LR: 4.17e-05 | 3110.3ms/step | 19.3 samples/s +Step 41200/50462 | Loss: 2.6209 | LR: 4.16e-05 | 3109.8ms/step | 19.3 samples/s +Step 41210/50462 | Loss: 2.5964 | LR: 4.15e-05 | 3098.0ms/step | 19.4 samples/s +Step 41220/50462 | Loss: 2.6115 | LR: 4.14e-05 | 3091.7ms/step | 19.4 samples/s +Step 41230/50462 | Loss: 2.6349 | LR: 4.14e-05 | 3105.1ms/step | 19.3 samples/s +Step 41240/50462 | Loss: 2.6276 | LR: 4.13e-05 | 3098.6ms/step | 19.4 samples/s +Step 41250/50462 | Loss: 2.6386 | LR: 4.12e-05 | 3102.5ms/step | 19.3 samples/s +Validation at step 41250: Loss = 2.6406, Perplexity = 14.02 +Step 41260/50462 | Loss: 2.6347 | LR: 4.11e-05 | 4743.7ms/step | 12.6 samples/s +Step 41270/50462 | Loss: 2.6102 | LR: 4.10e-05 | 3104.9ms/step | 19.3 samples/s +Step 41280/50462 | Loss: 2.5885 | LR: 4.09e-05 | 3163.8ms/step | 19.0 samples/s +Step 41290/50462 | Loss: 2.6491 | LR: 4.08e-05 | 3104.2ms/step | 19.3 samples/s +Step 41300/50462 | Loss: 2.6392 | LR: 4.07e-05 | 3099.5ms/step | 19.4 samples/s +Step 41310/50462 | Loss: 2.6026 | LR: 4.07e-05 | 3101.9ms/step | 19.3 samples/s +Step 41320/50462 | Loss: 2.6830 | LR: 4.06e-05 | 3103.3ms/step | 19.3 samples/s +Step 41330/50462 | Loss: 2.6195 | LR: 4.05e-05 | 3166.1ms/step | 19.0 samples/s +Step 41340/50462 | Loss: 2.6133 | LR: 4.04e-05 | 3096.8ms/step | 19.4 samples/s +Step 41350/50462 | Loss: 2.6335 | LR: 4.03e-05 | 3157.3ms/step | 19.0 samples/s +Step 41360/50462 | Loss: 2.6618 | LR: 4.02e-05 | 3095.9ms/step | 19.4 samples/s +Step 41370/50462 | Loss: 2.6304 | LR: 4.01e-05 | 3090.3ms/step | 19.4 samples/s +Step 41380/50462 | Loss: 2.6278 | LR: 4.01e-05 | 3099.1ms/step | 19.4 samples/s +Step 41390/50462 | Loss: 2.6008 | LR: 4.00e-05 | 3097.5ms/step | 19.4 samples/s +Step 41400/50462 | Loss: 2.6418 | LR: 3.99e-05 | 3098.6ms/step | 19.4 samples/s +Step 41410/50462 | Loss: 2.6462 | LR: 3.98e-05 | 3100.0ms/step | 19.4 samples/s +Step 41420/50462 | Loss: 2.6076 | LR: 3.97e-05 | 3100.7ms/step | 19.4 samples/s +Step 41430/50462 | Loss: 2.6408 | LR: 3.96e-05 | 3100.3ms/step | 19.4 samples/s +Step 41440/50462 | Loss: 2.6108 | LR: 3.95e-05 | 3100.1ms/step | 19.4 samples/s +Step 41450/50462 | Loss: 2.6291 | LR: 3.95e-05 | 3100.5ms/step | 19.4 samples/s +Step 41460/50462 | Loss: 2.6136 | LR: 3.94e-05 | 3090.0ms/step | 19.4 samples/s +Step 41470/50462 | Loss: 2.5906 | LR: 3.93e-05 | 3091.5ms/step | 19.4 samples/s +Step 41480/50462 | Loss: 2.6483 | LR: 3.92e-05 | 3097.9ms/step | 19.4 samples/s +Step 41490/50462 | Loss: 2.6016 | LR: 3.91e-05 | 3098.7ms/step | 19.4 samples/s +Step 41500/50462 | Loss: 2.6471 | LR: 3.90e-05 | 3103.7ms/step | 19.3 samples/s +Validation at step 41500: Loss = 2.6405, Perplexity = 14.02 +Step 41510/50462 | Loss: 2.6157 | LR: 3.90e-05 | 4749.6ms/step | 12.6 samples/s +Step 41520/50462 | Loss: 2.6303 | LR: 3.89e-05 | 3107.4ms/step | 19.3 samples/s +Step 41530/50462 | Loss: 2.6413 | LR: 3.88e-05 | 3110.8ms/step | 19.3 samples/s +Step 41540/50462 | Loss: 2.6120 | LR: 3.87e-05 | 3106.3ms/step | 19.3 samples/s +Step 41550/50462 | Loss: 2.6386 | LR: 3.86e-05 | 3100.7ms/step | 19.4 samples/s +Step 41560/50462 | Loss: 2.6489 | LR: 3.85e-05 | 3102.8ms/step | 19.3 samples/s +Step 41570/50462 | Loss: 2.6375 | LR: 3.84e-05 | 3100.9ms/step | 19.3 samples/s +Step 41580/50462 | Loss: 2.6412 | LR: 3.84e-05 | 3110.3ms/step | 19.3 samples/s +Step 41590/50462 | Loss: 2.6235 | LR: 3.83e-05 | 3108.7ms/step | 19.3 samples/s +Step 41600/50462 | Loss: 2.6384 | LR: 3.82e-05 | 3098.2ms/step | 19.4 samples/s +Step 41610/50462 | Loss: 2.6220 | LR: 3.81e-05 | 3159.7ms/step | 19.0 samples/s +Step 41620/50462 | Loss: 2.6250 | LR: 3.80e-05 | 3096.2ms/step | 19.4 samples/s +Step 41630/50462 | Loss: 2.6084 | LR: 3.79e-05 | 3104.0ms/step | 19.3 samples/s +Step 41640/50462 | Loss: 2.6228 | LR: 3.79e-05 | 3103.0ms/step | 19.3 samples/s +Step 41650/50462 | Loss: 2.6108 | LR: 3.78e-05 | 3109.8ms/step | 19.3 samples/s +Step 41660/50462 | Loss: 2.6389 | LR: 3.77e-05 | 3155.4ms/step | 19.0 samples/s +Step 41670/50462 | Loss: 2.6321 | LR: 3.76e-05 | 3102.1ms/step | 19.3 samples/s +Step 41680/50462 | Loss: 2.6163 | LR: 3.75e-05 | 3152.2ms/step | 19.0 samples/s +Step 41690/50462 | Loss: 2.6126 | LR: 3.74e-05 | 3098.7ms/step | 19.4 samples/s +Step 41700/50462 | Loss: 2.6276 | LR: 3.74e-05 | 3107.8ms/step | 19.3 samples/s +Step 41710/50462 | Loss: 2.6148 | LR: 3.73e-05 | 3103.8ms/step | 19.3 samples/s +Step 41720/50462 | Loss: 2.6130 | LR: 3.72e-05 | 3098.3ms/step | 19.4 samples/s +Step 41730/50462 | Loss: 2.6186 | LR: 3.71e-05 | 3102.7ms/step | 19.3 samples/s +Step 41740/50462 | Loss: 2.6116 | LR: 3.70e-05 | 3104.1ms/step | 19.3 samples/s +Step 41750/50462 | Loss: 2.6222 | LR: 3.69e-05 | 3103.9ms/step | 19.3 samples/s +Validation at step 41750: Loss = 2.6403, Perplexity = 14.02 +Step 41760/50462 | Loss: 2.6299 | LR: 3.69e-05 | 4749.4ms/step | 12.6 samples/s +Step 41770/50462 | Loss: 2.6370 | LR: 3.68e-05 | 3101.9ms/step | 19.3 samples/s +Step 41780/50462 | Loss: 2.6242 | LR: 3.67e-05 | 3101.0ms/step | 19.3 samples/s +Step 41790/50462 | Loss: 2.6101 | LR: 3.66e-05 | 3105.7ms/step | 19.3 samples/s +Step 41800/50462 | Loss: 2.6123 | LR: 3.65e-05 | 3096.9ms/step | 19.4 samples/s +Step 41810/50462 | Loss: 2.6467 | LR: 3.64e-05 | 3106.4ms/step | 19.3 samples/s +Step 41820/50462 | Loss: 2.5728 | LR: 3.64e-05 | 3110.6ms/step | 19.3 samples/s +Step 41830/50462 | Loss: 2.6227 | LR: 3.63e-05 | 3106.2ms/step | 19.3 samples/s +Step 41840/50462 | Loss: 2.5983 | LR: 3.62e-05 | 3104.4ms/step | 19.3 samples/s +Step 41850/50462 | Loss: 2.6394 | LR: 3.61e-05 | 3107.3ms/step | 19.3 samples/s +Step 41860/50462 | Loss: 2.6247 | LR: 3.60e-05 | 3101.2ms/step | 19.3 samples/s +Step 41870/50462 | Loss: 2.6215 | LR: 3.60e-05 | 3102.7ms/step | 19.3 samples/s +Step 41880/50462 | Loss: 2.6355 | LR: 3.59e-05 | 3109.6ms/step | 19.3 samples/s +Step 41890/50462 | Loss: 2.6242 | LR: 3.58e-05 | 3107.2ms/step | 19.3 samples/s +Step 41900/50462 | Loss: 2.6497 | LR: 3.57e-05 | 3104.4ms/step | 19.3 samples/s +Step 41910/50462 | Loss: 2.6393 | LR: 3.56e-05 | 3099.5ms/step | 19.4 samples/s +Step 41920/50462 | Loss: 2.6343 | LR: 3.55e-05 | 3106.3ms/step | 19.3 samples/s +Step 41930/50462 | Loss: 2.6136 | LR: 3.55e-05 | 3109.8ms/step | 19.3 samples/s +Step 41940/50462 | Loss: 2.6422 | LR: 3.54e-05 | 3167.9ms/step | 18.9 samples/s +Step 41950/50462 | Loss: 2.6103 | LR: 3.53e-05 | 3104.7ms/step | 19.3 samples/s +Step 41960/50462 | Loss: 2.6339 | LR: 3.52e-05 | 3108.1ms/step | 19.3 samples/s +Step 41970/50462 | Loss: 2.6263 | LR: 3.51e-05 | 3109.8ms/step | 19.3 samples/s +Step 41980/50462 | Loss: 2.6231 | LR: 3.51e-05 | 3104.2ms/step | 19.3 samples/s +Step 41990/50462 | Loss: 2.6560 | LR: 3.50e-05 | 3161.1ms/step | 19.0 samples/s +Step 42000/50462 | Loss: 2.6325 | LR: 3.49e-05 | 3093.5ms/step | 19.4 samples/s +Validation at step 42000: Loss = 2.6402, Perplexity = 14.02 +Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-42000 +Step 42010/50462 | Loss: 2.6028 | LR: 3.48e-05 | 5728.2ms/step | 10.5 samples/s +Step 42020/50462 | Loss: 2.6459 | LR: 3.47e-05 | 3097.1ms/step | 19.4 samples/s +Step 42030/50462 | Loss: 2.6658 | LR: 3.47e-05 | 3110.7ms/step | 19.3 samples/s +Step 42040/50462 | Loss: 2.6303 | LR: 3.46e-05 | 3104.5ms/step | 19.3 samples/s +Step 42050/50462 | Loss: 2.6295 | LR: 3.45e-05 | 3108.1ms/step | 19.3 samples/s +Step 42060/50462 | Loss: 2.6660 | LR: 3.44e-05 | 3105.0ms/step | 19.3 samples/s +Step 42070/50462 | Loss: 2.6462 | LR: 3.43e-05 | 3108.8ms/step | 19.3 samples/s +Step 42080/50462 | Loss: 2.6169 | LR: 3.43e-05 | 3104.5ms/step | 19.3 samples/s +Step 42090/50462 | Loss: 2.6463 | LR: 3.42e-05 | 3104.8ms/step | 19.3 samples/s +Step 42100/50462 | Loss: 2.6313 | LR: 3.41e-05 | 3107.9ms/step | 19.3 samples/s +Step 42110/50462 | Loss: 2.5984 | LR: 3.40e-05 | 3112.0ms/step | 19.3 samples/s +Step 42120/50462 | Loss: 2.6334 | LR: 3.39e-05 | 3107.5ms/step | 19.3 samples/s +Step 42130/50462 | Loss: 2.6430 | LR: 3.39e-05 | 3104.6ms/step | 19.3 samples/s +Step 42140/50462 | Loss: 2.6399 | LR: 3.38e-05 | 3100.6ms/step | 19.4 samples/s +Step 42150/50462 | Loss: 2.6337 | LR: 3.37e-05 | 3100.0ms/step | 19.4 samples/s +Step 42160/50462 | Loss: 2.5948 | LR: 3.36e-05 | 3100.3ms/step | 19.4 samples/s +Step 42170/50462 | Loss: 2.6227 | LR: 3.35e-05 | 3100.9ms/step | 19.3 samples/s +Step 42180/50462 | Loss: 2.6304 | LR: 3.35e-05 | 3104.6ms/step | 19.3 samples/s +Step 42190/50462 | Loss: 2.6587 | LR: 3.34e-05 | 3099.7ms/step | 19.4 samples/s +Step 42200/50462 | Loss: 2.6169 | LR: 3.33e-05 | 3094.2ms/step | 19.4 samples/s +Step 42210/50462 | Loss: 2.6198 | LR: 3.32e-05 | 3104.7ms/step | 19.3 samples/s +Step 42220/50462 | Loss: 2.6485 | LR: 3.32e-05 | 3098.1ms/step | 19.4 samples/s +Step 42230/50462 | Loss: 2.6349 | LR: 3.31e-05 | 3103.9ms/step | 19.3 samples/s +Step 42240/50462 | Loss: 2.5899 | LR: 3.30e-05 | 3105.4ms/step | 19.3 samples/s +Step 42250/50462 | Loss: 2.6400 | LR: 3.29e-05 | 3098.6ms/step | 19.4 samples/s +Validation at step 42250: Loss = 2.6401, Perplexity = 14.01 +Step 42260/50462 | Loss: 2.6462 | LR: 3.28e-05 | 4808.6ms/step | 12.5 samples/s +Step 42270/50462 | Loss: 2.6570 | LR: 3.28e-05 | 3108.3ms/step | 19.3 samples/s +Step 42280/50462 | Loss: 2.6274 | LR: 3.27e-05 | 3105.7ms/step | 19.3 samples/s +Step 42290/50462 | Loss: 2.6421 | LR: 3.26e-05 | 3109.0ms/step | 19.3 samples/s +Step 42300/50462 | Loss: 2.6278 | LR: 3.25e-05 | 3108.8ms/step | 19.3 samples/s +Step 42310/50462 | Loss: 2.6202 | LR: 3.24e-05 | 3099.3ms/step | 19.4 samples/s +Step 42320/50462 | Loss: 2.6038 | LR: 3.24e-05 | 3167.9ms/step | 18.9 samples/s +Step 42330/50462 | Loss: 2.6483 | LR: 3.23e-05 | 3162.9ms/step | 19.0 samples/s +Step 42340/50462 | Loss: 2.6577 | LR: 3.22e-05 | 3093.3ms/step | 19.4 samples/s +Step 42350/50462 | Loss: 2.6050 | LR: 3.21e-05 | 3097.9ms/step | 19.4 samples/s +Step 42360/50462 | Loss: 2.6165 | LR: 3.21e-05 | 3094.1ms/step | 19.4 samples/s +Step 42370/50462 | Loss: 2.6269 | LR: 3.20e-05 | 3097.6ms/step | 19.4 samples/s +Step 42380/50462 | Loss: 2.6129 | LR: 3.19e-05 | 3104.4ms/step | 19.3 samples/s +Step 42390/50462 | Loss: 2.6413 | LR: 3.18e-05 | 3101.9ms/step | 19.3 samples/s +Step 42400/50462 | Loss: 2.6494 | LR: 3.18e-05 | 3108.6ms/step | 19.3 samples/s +Step 42410/50462 | Loss: 2.6410 | LR: 3.17e-05 | 3100.4ms/step | 19.4 samples/s +Step 42420/50462 | Loss: 2.6341 | LR: 3.16e-05 | 3097.8ms/step | 19.4 samples/s +Step 42430/50462 | Loss: 2.6215 | LR: 3.15e-05 | 3101.7ms/step | 19.3 samples/s +Step 42440/50462 | Loss: 2.6232 | LR: 3.14e-05 | 3105.0ms/step | 19.3 samples/s +Step 42450/50462 | Loss: 2.6404 | LR: 3.14e-05 | 3108.8ms/step | 19.3 samples/s +Step 42460/50462 | Loss: 2.6193 | LR: 3.13e-05 | 3105.4ms/step | 19.3 samples/s +Step 42470/50462 | Loss: 2.6314 | LR: 3.12e-05 | 3105.2ms/step | 19.3 samples/s +Step 42480/50462 | Loss: 2.5899 | LR: 3.11e-05 | 3109.1ms/step | 19.3 samples/s +Step 42490/50462 | Loss: 2.6345 | LR: 3.11e-05 | 3104.4ms/step | 19.3 samples/s +Step 42500/50462 | Loss: 2.6459 | LR: 3.10e-05 | 3110.7ms/step | 19.3 samples/s +Validation at step 42500: Loss = 2.6400, Perplexity = 14.01 +Step 42510/50462 | Loss: 2.6430 | LR: 3.09e-05 | 4752.2ms/step | 12.6 samples/s +Step 42520/50462 | Loss: 2.6256 | LR: 3.08e-05 | 3101.2ms/step | 19.3 samples/s +Step 42530/50462 | Loss: 2.6180 | LR: 3.08e-05 | 3093.9ms/step | 19.4 samples/s +Step 42540/50462 | Loss: 2.6226 | LR: 3.07e-05 | 3105.8ms/step | 19.3 samples/s +Step 42550/50462 | Loss: 2.6512 | LR: 3.06e-05 | 3097.9ms/step | 19.4 samples/s +Step 42560/50462 | Loss: 2.7099 | LR: 3.05e-05 | 3105.7ms/step | 19.3 samples/s +Step 42570/50462 | Loss: 2.6386 | LR: 3.05e-05 | 3103.5ms/step | 19.3 samples/s +Step 42580/50462 | Loss: 2.6321 | LR: 3.04e-05 | 3091.0ms/step | 19.4 samples/s +Step 42590/50462 | Loss: 2.6229 | LR: 3.03e-05 | 3157.2ms/step | 19.0 samples/s +Step 42600/50462 | Loss: 2.6743 | LR: 3.02e-05 | 3086.3ms/step | 19.4 samples/s +Step 42610/50462 | Loss: 2.6238 | LR: 3.02e-05 | 3094.3ms/step | 19.4 samples/s +Step 42620/50462 | Loss: 2.6174 | LR: 3.01e-05 | 3102.9ms/step | 19.3 samples/s +Step 42630/50462 | Loss: 2.6473 | LR: 3.00e-05 | 3108.1ms/step | 19.3 samples/s +Step 42640/50462 | Loss: 2.6753 | LR: 2.99e-05 | 3171.6ms/step | 18.9 samples/s +Step 42650/50462 | Loss: 2.6417 | LR: 2.99e-05 | 3101.9ms/step | 19.3 samples/s +Step 42660/50462 | Loss: 2.6465 | LR: 2.98e-05 | 3158.6ms/step | 19.0 samples/s +Step 42670/50462 | Loss: 2.6182 | LR: 2.97e-05 | 3109.0ms/step | 19.3 samples/s +Step 42680/50462 | Loss: 2.6167 | LR: 2.96e-05 | 3104.0ms/step | 19.3 samples/s +Step 42690/50462 | Loss: 2.6355 | LR: 2.96e-05 | 3103.9ms/step | 19.3 samples/s +Step 42700/50462 | Loss: 2.6435 | LR: 2.95e-05 | 3103.1ms/step | 19.3 samples/s +Step 42710/50462 | Loss: 2.6237 | LR: 2.94e-05 | 3105.6ms/step | 19.3 samples/s +Step 42720/50462 | Loss: 2.6235 | LR: 2.93e-05 | 3108.4ms/step | 19.3 samples/s +Step 42730/50462 | Loss: 2.6237 | LR: 2.93e-05 | 3102.3ms/step | 19.3 samples/s +Step 42740/50462 | Loss: 2.6299 | LR: 2.92e-05 | 3096.2ms/step | 19.4 samples/s +Step 42750/50462 | Loss: 2.6242 | LR: 2.91e-05 | 3105.1ms/step | 19.3 samples/s +Validation at step 42750: Loss = 2.6399, Perplexity = 14.01 +Step 42760/50462 | Loss: 2.6416 | LR: 2.90e-05 | 4733.0ms/step | 12.7 samples/s +Step 42770/50462 | Loss: 2.6302 | LR: 2.90e-05 | 3103.4ms/step | 19.3 samples/s +Step 42780/50462 | Loss: 2.6302 | LR: 2.89e-05 | 3090.9ms/step | 19.4 samples/s +Step 42790/50462 | Loss: 2.6477 | LR: 2.88e-05 | 3103.9ms/step | 19.3 samples/s +Step 42800/50462 | Loss: 2.6483 | LR: 2.87e-05 | 3100.4ms/step | 19.4 samples/s +Step 42810/50462 | Loss: 2.6538 | LR: 2.87e-05 | 3102.2ms/step | 19.3 samples/s +Step 42820/50462 | Loss: 2.6212 | LR: 2.86e-05 | 3106.1ms/step | 19.3 samples/s +Step 42830/50462 | Loss: 2.6162 | LR: 2.85e-05 | 3108.7ms/step | 19.3 samples/s +Step 42840/50462 | Loss: 2.6422 | LR: 2.84e-05 | 3109.3ms/step | 19.3 samples/s +Step 42850/50462 | Loss: 2.6320 | LR: 2.84e-05 | 3102.0ms/step | 19.3 samples/s +Step 42860/50462 | Loss: 2.6614 | LR: 2.83e-05 | 3104.0ms/step | 19.3 samples/s +Step 42870/50462 | Loss: 2.6432 | LR: 2.82e-05 | 3105.5ms/step | 19.3 samples/s +Step 42880/50462 | Loss: 2.6396 | LR: 2.82e-05 | 3097.4ms/step | 19.4 samples/s +Step 42890/50462 | Loss: 2.6219 | LR: 2.81e-05 | 3093.7ms/step | 19.4 samples/s +Step 42900/50462 | Loss: 2.6319 | LR: 2.80e-05 | 3098.6ms/step | 19.4 samples/s +Step 42910/50462 | Loss: 2.6611 | LR: 2.79e-05 | 3161.2ms/step | 19.0 samples/s +Step 42920/50462 | Loss: 2.6202 | LR: 2.79e-05 | 3101.4ms/step | 19.3 samples/s +Step 42930/50462 | Loss: 2.6184 | LR: 2.78e-05 | 3101.6ms/step | 19.3 samples/s +Step 42940/50462 | Loss: 2.6085 | LR: 2.77e-05 | 3101.2ms/step | 19.3 samples/s +Step 42950/50462 | Loss: 2.6597 | LR: 2.76e-05 | 3104.6ms/step | 19.3 samples/s +Step 42960/50462 | Loss: 2.6426 | LR: 2.76e-05 | 3172.5ms/step | 18.9 samples/s +Step 42970/50462 | Loss: 2.6142 | LR: 2.75e-05 | 3099.4ms/step | 19.4 samples/s +Step 42980/50462 | Loss: 2.6274 | LR: 2.74e-05 | 3166.4ms/step | 18.9 samples/s +Step 42990/50462 | Loss: 2.6351 | LR: 2.74e-05 | 3104.1ms/step | 19.3 samples/s +Step 43000/50462 | Loss: 2.6137 | LR: 2.73e-05 | 3102.2ms/step | 19.3 samples/s +Validation at step 43000: Loss = 2.6398, Perplexity = 14.01 +Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-43000 +Step 43010/50462 | Loss: 2.6536 | LR: 2.72e-05 | 5666.9ms/step | 10.6 samples/s +Step 43020/50462 | Loss: 2.6259 | LR: 2.71e-05 | 3090.4ms/step | 19.4 samples/s +Step 43030/50462 | Loss: 2.6330 | LR: 2.71e-05 | 3099.6ms/step | 19.4 samples/s +Step 43040/50462 | Loss: 2.6176 | LR: 2.70e-05 | 3097.4ms/step | 19.4 samples/s +Step 43050/50462 | Loss: 2.6389 | LR: 2.69e-05 | 3093.2ms/step | 19.4 samples/s +Step 43060/50462 | Loss: 2.6145 | LR: 2.69e-05 | 3100.3ms/step | 19.4 samples/s +Step 43070/50462 | Loss: 2.6337 | LR: 2.68e-05 | 3103.9ms/step | 19.3 samples/s +Step 43080/50462 | Loss: 2.6110 | LR: 2.67e-05 | 3102.9ms/step | 19.3 samples/s +Step 43090/50462 | Loss: 2.6125 | LR: 2.66e-05 | 3099.3ms/step | 19.4 samples/s +Step 43100/50462 | Loss: 2.6059 | LR: 2.66e-05 | 3104.9ms/step | 19.3 samples/s +Step 43110/50462 | Loss: 2.6154 | LR: 2.65e-05 | 3094.1ms/step | 19.4 samples/s +Step 43120/50462 | Loss: 2.6289 | LR: 2.64e-05 | 3098.3ms/step | 19.4 samples/s +Step 43130/50462 | Loss: 2.6407 | LR: 2.64e-05 | 3107.2ms/step | 19.3 samples/s +Step 43140/50462 | Loss: 2.6105 | LR: 2.63e-05 | 3104.0ms/step | 19.3 samples/s +Step 43150/50462 | Loss: 2.6128 | LR: 2.62e-05 | 3107.8ms/step | 19.3 samples/s +Step 43160/50462 | Loss: 2.6500 | LR: 2.61e-05 | 3113.4ms/step | 19.3 samples/s +Step 43170/50462 | Loss: 2.6424 | LR: 2.61e-05 | 3108.1ms/step | 19.3 samples/s +Step 43180/50462 | Loss: 2.5981 | LR: 2.60e-05 | 3100.4ms/step | 19.4 samples/s +Step 43190/50462 | Loss: 2.6505 | LR: 2.59e-05 | 3101.8ms/step | 19.3 samples/s +Step 43200/50462 | Loss: 2.6305 | LR: 2.59e-05 | 3104.1ms/step | 19.3 samples/s +Step 43210/50462 | Loss: 2.6396 | LR: 2.58e-05 | 3105.1ms/step | 19.3 samples/s +Step 43220/50462 | Loss: 2.6427 | LR: 2.57e-05 | 3107.3ms/step | 19.3 samples/s +Step 43230/50462 | Loss: 2.6279 | LR: 2.57e-05 | 3105.3ms/step | 19.3 samples/s +Step 43240/50462 | Loss: 2.6531 | LR: 2.56e-05 | 3160.9ms/step | 19.0 samples/s +Step 43250/50462 | Loss: 2.6026 | LR: 2.55e-05 | 3103.4ms/step | 19.3 samples/s +Validation at step 43250: Loss = 2.6398, Perplexity = 14.01 +Step 43260/50462 | Loss: 2.6044 | LR: 2.54e-05 | 4752.2ms/step | 12.6 samples/s +Step 43270/50462 | Loss: 2.6133 | LR: 2.54e-05 | 3104.1ms/step | 19.3 samples/s +Step 43280/50462 | Loss: 2.6349 | LR: 2.53e-05 | 3105.5ms/step | 19.3 samples/s +Step 43290/50462 | Loss: 2.6141 | LR: 2.52e-05 | 3166.2ms/step | 19.0 samples/s +Step 43300/50462 | Loss: 2.6585 | LR: 2.52e-05 | 3164.4ms/step | 19.0 samples/s +Step 43310/50462 | Loss: 2.6236 | LR: 2.51e-05 | 3107.1ms/step | 19.3 samples/s +Step 43320/50462 | Loss: 2.6280 | LR: 2.50e-05 | 3108.0ms/step | 19.3 samples/s +Step 43330/50462 | Loss: 2.6150 | LR: 2.50e-05 | 3105.9ms/step | 19.3 samples/s +Step 43340/50462 | Loss: 2.6238 | LR: 2.49e-05 | 3104.6ms/step | 19.3 samples/s +Step 43350/50462 | Loss: 2.6116 | LR: 2.48e-05 | 3106.1ms/step | 19.3 samples/s +Step 43360/50462 | Loss: 2.6435 | LR: 2.48e-05 | 3105.9ms/step | 19.3 samples/s +Step 43370/50462 | Loss: 2.6340 | LR: 2.47e-05 | 3106.5ms/step | 19.3 samples/s +Step 43380/50462 | Loss: 2.6152 | LR: 2.46e-05 | 3106.7ms/step | 19.3 samples/s +Step 43390/50462 | Loss: 2.6442 | LR: 2.46e-05 | 3105.3ms/step | 19.3 samples/s +Step 43400/50462 | Loss: 2.6231 | LR: 2.45e-05 | 3104.6ms/step | 19.3 samples/s +Step 43410/50462 | Loss: 2.6438 | LR: 2.44e-05 | 3103.5ms/step | 19.3 samples/s +Step 43420/50462 | Loss: 2.6254 | LR: 2.44e-05 | 3104.8ms/step | 19.3 samples/s +Step 43430/50462 | Loss: 2.6290 | LR: 2.43e-05 | 3103.8ms/step | 19.3 samples/s +Step 43440/50462 | Loss: 2.6264 | LR: 2.42e-05 | 3107.0ms/step | 19.3 samples/s +Step 43450/50462 | Loss: 2.6498 | LR: 2.41e-05 | 3104.3ms/step | 19.3 samples/s +Step 43460/50462 | Loss: 2.6380 | LR: 2.41e-05 | 3105.1ms/step | 19.3 samples/s +Step 43470/50462 | Loss: 2.6508 | LR: 2.40e-05 | 3107.1ms/step | 19.3 samples/s +Step 43480/50462 | Loss: 2.6522 | LR: 2.39e-05 | 3107.6ms/step | 19.3 samples/s +Step 43490/50462 | Loss: 2.6212 | LR: 2.39e-05 | 3105.9ms/step | 19.3 samples/s +Step 43500/50462 | Loss: 2.6380 | LR: 2.38e-05 | 3105.7ms/step | 19.3 samples/s +Validation at step 43500: Loss = 2.6397, Perplexity = 14.01 +Step 43510/50462 | Loss: 2.6416 | LR: 2.37e-05 | 4753.2ms/step | 12.6 samples/s +Step 43520/50462 | Loss: 2.6063 | LR: 2.37e-05 | 3105.4ms/step | 19.3 samples/s +Step 43530/50462 | Loss: 2.6292 | LR: 2.36e-05 | 3103.0ms/step | 19.3 samples/s +Step 43540/50462 | Loss: 2.6216 | LR: 2.35e-05 | 3103.8ms/step | 19.3 samples/s +Step 43550/50462 | Loss: 2.6310 | LR: 2.35e-05 | 3103.0ms/step | 19.3 samples/s +Step 43560/50462 | Loss: 2.6106 | LR: 2.34e-05 | 3161.7ms/step | 19.0 samples/s +Step 43570/50462 | Loss: 2.6299 | LR: 2.33e-05 | 3093.0ms/step | 19.4 samples/s +Step 43580/50462 | Loss: 2.6580 | LR: 2.33e-05 | 3101.5ms/step | 19.3 samples/s +Step 43590/50462 | Loss: 2.6199 | LR: 2.32e-05 | 3103.5ms/step | 19.3 samples/s +Step 43600/50462 | Loss: 2.6518 | LR: 2.31e-05 | 3104.4ms/step | 19.3 samples/s +Step 43610/50462 | Loss: 2.6343 | LR: 2.31e-05 | 3098.1ms/step | 19.4 samples/s +Step 43620/50462 | Loss: 2.6155 | LR: 2.30e-05 | 3164.6ms/step | 19.0 samples/s +Step 43630/50462 | Loss: 2.6549 | LR: 2.29e-05 | 3166.9ms/step | 18.9 samples/s +Step 43640/50462 | Loss: 2.6178 | LR: 2.29e-05 | 3104.2ms/step | 19.3 samples/s +Step 43650/50462 | Loss: 2.6365 | LR: 2.28e-05 | 3104.9ms/step | 19.3 samples/s +Step 43660/50462 | Loss: 2.6485 | LR: 2.27e-05 | 3104.8ms/step | 19.3 samples/s +Step 43670/50462 | Loss: 2.6379 | LR: 2.27e-05 | 3103.0ms/step | 19.3 samples/s +Step 43680/50462 | Loss: 2.6448 | LR: 2.26e-05 | 3104.6ms/step | 19.3 samples/s +Step 43690/50462 | Loss: 2.6368 | LR: 2.25e-05 | 3104.5ms/step | 19.3 samples/s +Step 43700/50462 | Loss: 2.6508 | LR: 2.25e-05 | 3102.7ms/step | 19.3 samples/s +Step 43710/50462 | Loss: 2.6564 | LR: 2.24e-05 | 3097.6ms/step | 19.4 samples/s +Step 43720/50462 | Loss: 2.6131 | LR: 2.24e-05 | 3098.2ms/step | 19.4 samples/s +Step 43730/50462 | Loss: 2.6036 | LR: 2.23e-05 | 3100.5ms/step | 19.4 samples/s +Step 43740/50462 | Loss: 2.6266 | LR: 2.22e-05 | 3098.2ms/step | 19.4 samples/s +Step 43750/50462 | Loss: 2.6281 | LR: 2.22e-05 | 3102.2ms/step | 19.3 samples/s +Validation at step 43750: Loss = 2.6396, Perplexity = 14.01 +Step 43760/50462 | Loss: 2.6321 | LR: 2.21e-05 | 4753.0ms/step | 12.6 samples/s +Step 43770/50462 | Loss: 2.6191 | LR: 2.20e-05 | 3103.1ms/step | 19.3 samples/s +Step 43780/50462 | Loss: 2.6260 | LR: 2.20e-05 | 3105.2ms/step | 19.3 samples/s +Step 43790/50462 | Loss: 2.6309 | LR: 2.19e-05 | 3103.4ms/step | 19.3 samples/s +Step 43800/50462 | Loss: 2.6495 | LR: 2.18e-05 | 3104.2ms/step | 19.3 samples/s +Step 43810/50462 | Loss: 2.5939 | LR: 2.18e-05 | 3105.0ms/step | 19.3 samples/s +Step 43820/50462 | Loss: 2.5984 | LR: 2.17e-05 | 3105.8ms/step | 19.3 samples/s +Step 43830/50462 | Loss: 2.6382 | LR: 2.16e-05 | 3106.4ms/step | 19.3 samples/s +Step 43840/50462 | Loss: 2.6584 | LR: 2.16e-05 | 3105.7ms/step | 19.3 samples/s +Step 43850/50462 | Loss: 2.6500 | LR: 2.15e-05 | 3106.6ms/step | 19.3 samples/s +Step 43860/50462 | Loss: 2.6336 | LR: 2.14e-05 | 3104.8ms/step | 19.3 samples/s +Step 43870/50462 | Loss: 2.6184 | LR: 2.14e-05 | 3099.8ms/step | 19.4 samples/s +Step 43880/50462 | Loss: 2.6424 | LR: 2.13e-05 | 3095.6ms/step | 19.4 samples/s +Step 43890/50462 | Loss: 2.6326 | LR: 2.13e-05 | 3161.3ms/step | 19.0 samples/s +Step 43900/50462 | Loss: 2.6873 | LR: 2.12e-05 | 3082.7ms/step | 19.5 samples/s +Step 43910/50462 | Loss: 2.6524 | LR: 2.11e-05 | 3098.4ms/step | 19.4 samples/s +Step 43920/50462 | Loss: 2.6287 | LR: 2.11e-05 | 3105.9ms/step | 19.3 samples/s +Step 43930/50462 | Loss: 2.6554 | LR: 2.10e-05 | 3100.9ms/step | 19.3 samples/s +Step 43940/50462 | Loss: 2.6294 | LR: 2.09e-05 | 3162.8ms/step | 19.0 samples/s +Step 43950/50462 | Loss: 2.6434 | LR: 2.09e-05 | 3102.9ms/step | 19.3 samples/s +Step 43960/50462 | Loss: 2.6222 | LR: 2.08e-05 | 3160.7ms/step | 19.0 samples/s +Step 43970/50462 | Loss: 2.6198 | LR: 2.07e-05 | 3092.3ms/step | 19.4 samples/s +Step 43980/50462 | Loss: 2.6110 | LR: 2.07e-05 | 3094.1ms/step | 19.4 samples/s +Step 43990/50462 | Loss: 2.6470 | LR: 2.06e-05 | 3096.3ms/step | 19.4 samples/s +Step 44000/50462 | Loss: 2.6527 | LR: 2.06e-05 | 3098.4ms/step | 19.4 samples/s +Validation at step 44000: Loss = 2.6396, Perplexity = 14.01 +Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-44000 +Step 44010/50462 | Loss: 2.6386 | LR: 2.05e-05 | 5652.4ms/step | 10.6 samples/s +Step 44020/50462 | Loss: 2.6288 | LR: 2.04e-05 | 3094.8ms/step | 19.4 samples/s +Step 44030/50462 | Loss: 2.6275 | LR: 2.04e-05 | 3101.2ms/step | 19.3 samples/s +Step 44040/50462 | Loss: 2.6191 | LR: 2.03e-05 | 3103.1ms/step | 19.3 samples/s +Step 44050/50462 | Loss: 2.6543 | LR: 2.02e-05 | 3101.4ms/step | 19.3 samples/s +Step 44060/50462 | Loss: 2.6522 | LR: 2.02e-05 | 3103.9ms/step | 19.3 samples/s +Step 44070/50462 | Loss: 2.6536 | LR: 2.01e-05 | 3103.0ms/step | 19.3 samples/s +Step 44080/50462 | Loss: 2.6341 | LR: 2.01e-05 | 3105.4ms/step | 19.3 samples/s +Step 44090/50462 | Loss: 2.6013 | LR: 2.00e-05 | 3105.4ms/step | 19.3 samples/s +Step 44100/50462 | Loss: 2.6217 | LR: 1.99e-05 | 3106.4ms/step | 19.3 samples/s +Step 44110/50462 | Loss: 2.6240 | LR: 1.99e-05 | 3104.4ms/step | 19.3 samples/s +Step 44120/50462 | Loss: 2.6411 | LR: 1.98e-05 | 3097.7ms/step | 19.4 samples/s +Step 44130/50462 | Loss: 2.6472 | LR: 1.97e-05 | 3096.4ms/step | 19.4 samples/s +Step 44140/50462 | Loss: 2.6183 | LR: 1.97e-05 | 3100.6ms/step | 19.4 samples/s +Step 44150/50462 | Loss: 2.6123 | LR: 1.96e-05 | 3099.9ms/step | 19.4 samples/s +Step 44160/50462 | Loss: 2.6386 | LR: 1.96e-05 | 3100.9ms/step | 19.3 samples/s +Step 44170/50462 | Loss: 2.6467 | LR: 1.95e-05 | 3098.0ms/step | 19.4 samples/s +Step 44180/50462 | Loss: 2.6136 | LR: 1.94e-05 | 3100.6ms/step | 19.4 samples/s +Step 44190/50462 | Loss: 2.6188 | LR: 1.94e-05 | 3100.9ms/step | 19.3 samples/s +Step 44200/50462 | Loss: 2.6393 | LR: 1.93e-05 | 3101.5ms/step | 19.3 samples/s +Step 44210/50462 | Loss: 2.6293 | LR: 1.93e-05 | 3161.4ms/step | 19.0 samples/s +Step 44220/50462 | Loss: 2.6317 | LR: 1.92e-05 | 3096.4ms/step | 19.4 samples/s +Step 44230/50462 | Loss: 2.6451 | LR: 1.91e-05 | 3100.4ms/step | 19.4 samples/s +Step 44240/50462 | Loss: 2.6336 | LR: 1.91e-05 | 3100.1ms/step | 19.4 samples/s +Step 44250/50462 | Loss: 2.6451 | LR: 1.90e-05 | 3101.7ms/step | 19.3 samples/s +Validation at step 44250: Loss = 2.6396, Perplexity = 14.01 +Step 44260/50462 | Loss: 2.6393 | LR: 1.90e-05 | 4759.9ms/step | 12.6 samples/s +Step 44270/50462 | Loss: 2.6144 | LR: 1.89e-05 | 3162.5ms/step | 19.0 samples/s +Step 44280/50462 | Loss: 2.6330 | LR: 1.88e-05 | 3161.3ms/step | 19.0 samples/s +Step 44290/50462 | Loss: 2.6317 | LR: 1.88e-05 | 3096.5ms/step | 19.4 samples/s +Step 44300/50462 | Loss: 2.5841 | LR: 1.87e-05 | 3101.5ms/step | 19.3 samples/s +Step 44310/50462 | Loss: 2.6100 | LR: 1.87e-05 | 3103.8ms/step | 19.3 samples/s +Step 44320/50462 | Loss: 2.6443 | LR: 1.86e-05 | 3103.4ms/step | 19.3 samples/s +Step 44330/50462 | Loss: 2.6365 | LR: 1.85e-05 | 3103.7ms/step | 19.3 samples/s +Step 44340/50462 | Loss: 2.6298 | LR: 1.85e-05 | 3107.3ms/step | 19.3 samples/s +Step 44350/50462 | Loss: 2.6370 | LR: 1.84e-05 | 3105.9ms/step | 19.3 samples/s +Step 44360/50462 | Loss: 2.6084 | LR: 1.84e-05 | 3106.6ms/step | 19.3 samples/s +Step 44370/50462 | Loss: 2.6344 | LR: 1.83e-05 | 3108.1ms/step | 19.3 samples/s +Step 44380/50462 | Loss: 2.6256 | LR: 1.82e-05 | 3107.0ms/step | 19.3 samples/s +Step 44390/50462 | Loss: 2.6452 | LR: 1.82e-05 | 3107.1ms/step | 19.3 samples/s +Step 44400/50462 | Loss: 2.6140 | LR: 1.81e-05 | 3105.4ms/step | 19.3 samples/s +Step 44410/50462 | Loss: 2.6197 | LR: 1.81e-05 | 3102.4ms/step | 19.3 samples/s +Step 44420/50462 | Loss: 2.6305 | LR: 1.80e-05 | 3104.1ms/step | 19.3 samples/s +Step 44430/50462 | Loss: 2.6395 | LR: 1.79e-05 | 3105.4ms/step | 19.3 samples/s +Step 44440/50462 | Loss: 2.6276 | LR: 1.79e-05 | 3113.4ms/step | 19.3 samples/s +Step 44450/50462 | Loss: 2.6148 | LR: 1.78e-05 | 3110.0ms/step | 19.3 samples/s +Step 44460/50462 | Loss: 2.6332 | LR: 1.78e-05 | 3113.9ms/step | 19.3 samples/s +Step 44470/50462 | Loss: 2.6444 | LR: 1.77e-05 | 3115.9ms/step | 19.3 samples/s diff --git a/epfl_final/log_files/sparse_30%.log b/epfl_final/log_files/sparse_30%.log new file mode 100644 index 0000000000000000000000000000000000000000..935d9b02819f989e71d1399b7f965c259af2fdaa --- /dev/null +++ b/epfl_final/log_files/sparse_30%.log @@ -0,0 +1,367 @@ +Auto-resuming from latest checkpoint: checkpoints/run_20260517_023855/checkpoint-18000 +Resuming: Using existing output directory checkpoints/run_20260517_023855 +Resuming wandb run hi2z3jh6 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260518_025234-hi2z3jh6 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run sparse 30% +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/hi2z3jh6 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "checkpoints/run_20260517_023855", + "resume_from_checkpoint": "checkpoints/run_20260517_023855/checkpoint-18000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 20005, + "per_device_batch_size": 3, + "gradient_accumulation_steps": 8, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 250, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 200, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "sparse 30%", + "wandb_tags": [ + "qwen3", + "0.6b" + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 18, + "sparse_mask_pool_size": 2000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_19", + "dense_mode": false +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: checkpoints/run_20260517_023855/checkpoint-18000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1024 + num_hidden_layers: 28 + num_attention_heads: 16 + num_key_value_heads: 8 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 18 + sparse_mask_pool_size: 2000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 596,049,920 + Trainable parameters: 596,049,920 + Non-trainable parameters: 0 + Model size (BF16): 1.19 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [3, 8192] -- -- True +├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True +│ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True +├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True +================================================================================================================================================================ +Total params: 751,632,384 +Trainable params: 751,632,384 +Non-trainable params: 0 +Total mult-adds (G): 2.25 +================================================================================================================================================================ +Input size (MB): 0.20 +Forward/backward pass size (MB): 63052.97 +Params size (MB): 1503.26 +Estimated Total Size (MB): 64556.43 +================================================================================================================================================================ +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003) +Dataset split: 1017602 train, 254401 validation +Training examples (packed, train split): 1017602 +Validation examples (packed, val split): 254401 +Per-device batch size: 3 +World size (GPUs): 1 +Gradient accumulation steps: 8 +Effective batch size (seqs/step): 24 +Sequence length: 8192 +Tokens per optimizer step: 196,608 +Micro-batches per epoch (per rank): 339200 +Optimizer steps per epoch: 42400 +Total optimizer steps: 20005 (capped by max_steps; 0.47 epochs) +Loading checkpoint from checkpoints/run_20260517_023855/checkpoint-18000... +Resumed from checkpoint at step 17999 (epoch 0, micro_step 144016) +Resuming training from step 17999 +Starting training... +Step 18000/20005 | Loss: 0.2871 | LR: 1.26e-05 | 347868.6ms/step | 0.1 samples/s +wandb: WARNING Tried to log to step 18000 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 18000: Loss = 2.9449, Perplexity = 19.01 +Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-18000 +wandb: WARNING Tried to log to step 18000 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18010/20005 | Loss: 2.9262 | LR: 1.24e-05 | 20258.4ms/step | 1.2 samples/s +wandb: WARNING Tried to log to step 18010 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18020/20005 | Loss: 2.9380 | LR: 1.23e-05 | 2325.3ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18020 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18030/20005 | Loss: 2.9472 | LR: 1.22e-05 | 2322.8ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18030 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18040/20005 | Loss: 2.9313 | LR: 1.21e-05 | 2315.8ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18040 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18050/20005 | Loss: 2.9340 | LR: 1.19e-05 | 2315.6ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18050 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18060/20005 | Loss: 2.9068 | LR: 1.18e-05 | 2316.8ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18060 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18070/20005 | Loss: 2.8988 | LR: 1.17e-05 | 2315.9ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18070 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18080/20005 | Loss: 2.9077 | LR: 1.16e-05 | 2314.4ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18080 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18090/20005 | Loss: 2.9596 | LR: 1.15e-05 | 2314.3ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18090 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18100/20005 | Loss: 2.9119 | LR: 1.13e-05 | 2316.3ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18100 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18110/20005 | Loss: 2.9399 | LR: 1.12e-05 | 2315.4ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18110 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18120/20005 | Loss: 2.9164 | LR: 1.11e-05 | 2315.4ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18120 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18130/20005 | Loss: 2.9187 | LR: 1.10e-05 | 2321.0ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18130 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18140/20005 | Loss: 2.9256 | LR: 1.09e-05 | 2315.5ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18140 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18150/20005 | Loss: 2.9161 | LR: 1.08e-05 | 2318.5ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18150 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18160/20005 | Loss: 2.9449 | LR: 1.06e-05 | 2323.3ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18160 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18170/20005 | Loss: 2.9103 | LR: 1.05e-05 | 2322.9ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18170 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18180/20005 | Loss: 2.9492 | LR: 1.04e-05 | 2322.2ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18180 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18190/20005 | Loss: 2.9149 | LR: 1.03e-05 | 2318.4ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18190 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18200/20005 | Loss: 2.9357 | LR: 1.02e-05 | 2313.1ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18200 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18210/20005 | Loss: 2.9313 | LR: 1.01e-05 | 2313.5ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18210 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18220/20005 | Loss: 2.9328 | LR: 9.97e-06 | 2310.1ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18220 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18230/20005 | Loss: 2.9211 | LR: 9.86e-06 | 2314.0ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18230 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18240/20005 | Loss: 2.9613 | LR: 9.75e-06 | 2312.3ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18240 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18250/20005 | Loss: 2.9196 | LR: 9.64e-06 | 2310.7ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18250 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 18250: Loss = 2.9449, Perplexity = 19.01 +wandb: WARNING Tried to log to step 18250 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18260/20005 | Loss: 2.9467 | LR: 9.53e-06 | 17425.7ms/step | 1.4 samples/s +wandb: WARNING Tried to log to step 18260 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18270/20005 | Loss: 2.9209 | LR: 9.42e-06 | 2315.3ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18270 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18280/20005 | Loss: 2.9144 | LR: 9.32e-06 | 2314.6ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18280 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18290/20005 | Loss: 2.9284 | LR: 9.21e-06 | 2312.5ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18290 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18300/20005 | Loss: 2.9517 | LR: 9.10e-06 | 2314.0ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18300 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18310/20005 | Loss: 2.9304 | LR: 9.00e-06 | 2315.0ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18310 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18320/20005 | Loss: 2.9358 | LR: 8.89e-06 | 2315.1ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18320 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18330/20005 | Loss: 2.9497 | LR: 8.79e-06 | 2314.7ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18330 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18340/20005 | Loss: 2.9259 | LR: 8.68e-06 | 2313.7ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18340 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18350/20005 | Loss: 2.9214 | LR: 8.58e-06 | 2314.9ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18350 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18360/20005 | Loss: 2.9279 | LR: 8.47e-06 | 2315.6ms/step | 10.4 samples/s +wandb: WARNING Tried to log to step 18360 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18370/20005 | Loss: 2.9385 | LR: 8.37e-06 | 2482.2ms/step | 9.7 samples/s +wandb: WARNING Tried to log to step 18370 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18380/20005 | Loss: 2.9436 | LR: 8.27e-06 | 2321.8ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18380 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18390/20005 | Loss: 2.9214 | LR: 8.17e-06 | 2324.1ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18390 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18400/20005 | Loss: 2.9564 | LR: 8.07e-06 | 2323.5ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18400 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18410/20005 | Loss: 2.9666 | LR: 7.97e-06 | 2326.4ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18410 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18420/20005 | Loss: 2.9280 | LR: 7.87e-06 | 2322.6ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18420 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18430/20005 | Loss: 2.9215 | LR: 7.77e-06 | 2324.1ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18430 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18440/20005 | Loss: 2.9387 | LR: 7.67e-06 | 2321.5ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18440 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18450/20005 | Loss: 2.9111 | LR: 7.58e-06 | 2322.5ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18450 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18460/20005 | Loss: 2.9352 | LR: 7.48e-06 | 2323.8ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18460 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18470/20005 | Loss: 2.9359 | LR: 7.38e-06 | 2322.9ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18470 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18480/20005 | Loss: 2.9074 | LR: 7.29e-06 | 2323.3ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18480 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18490/20005 | Loss: 2.9367 | LR: 7.19e-06 | 2324.8ms/step | 10.3 samples/s +wandb: WARNING Tried to log to step 18490 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18500/20005 | Loss: 2.9431 | LR: 7.10e-06 | 2324.3ms/step | 10.3 samples/s +Validation at step 18500: Loss = 2.9448, Perplexity = 19.01 +Step 18510/20005 | Loss: 2.9130 | LR: 7.00e-06 | 17470.7ms/step | 1.4 samples/s +Step 18520/20005 | Loss: 2.9038 | LR: 6.91e-06 | 2322.2ms/step | 10.3 samples/s +Step 18530/20005 | Loss: 2.9199 | LR: 6.82e-06 | 2322.1ms/step | 10.3 samples/s +Step 18540/20005 | Loss: 2.9125 | LR: 6.73e-06 | 2323.1ms/step | 10.3 samples/s +Step 18550/20005 | Loss: 2.9177 | LR: 6.64e-06 | 2323.3ms/step | 10.3 samples/s +Step 18560/20005 | Loss: 2.9444 | LR: 6.54e-06 | 2324.7ms/step | 10.3 samples/s +Step 18570/20005 | Loss: 2.9868 | LR: 6.45e-06 | 2322.0ms/step | 10.3 samples/s +Step 18580/20005 | Loss: 2.9507 | LR: 6.37e-06 | 2325.7ms/step | 10.3 samples/s +Step 18590/20005 | Loss: 2.9149 | LR: 6.28e-06 | 2322.9ms/step | 10.3 samples/s +Step 18600/20005 | Loss: 2.9023 | LR: 6.19e-06 | 2322.2ms/step | 10.3 samples/s +Step 18610/20005 | Loss: 2.9032 | LR: 6.10e-06 | 2323.1ms/step | 10.3 samples/s +Step 18620/20005 | Loss: 2.9311 | LR: 6.01e-06 | 2324.1ms/step | 10.3 samples/s +Step 18630/20005 | Loss: 2.9542 | LR: 5.93e-06 | 2321.4ms/step | 10.3 samples/s +Step 18640/20005 | Loss: 2.9366 | LR: 5.84e-06 | 2323.6ms/step | 10.3 samples/s +Step 18650/20005 | Loss: 2.9361 | LR: 5.76e-06 | 2321.7ms/step | 10.3 samples/s +Step 18660/20005 | Loss: 2.9005 | LR: 5.67e-06 | 2323.5ms/step | 10.3 samples/s +Step 18670/20005 | Loss: 2.9555 | LR: 5.59e-06 | 2323.3ms/step | 10.3 samples/s +Step 18680/20005 | Loss: 2.9134 | LR: 5.50e-06 | 2321.8ms/step | 10.3 samples/s +Step 18690/20005 | Loss: 2.9358 | LR: 5.42e-06 | 2322.4ms/step | 10.3 samples/s +Step 18700/20005 | Loss: 2.9642 | LR: 5.34e-06 | 2324.8ms/step | 10.3 samples/s +Step 18710/20005 | Loss: 2.9552 | LR: 5.26e-06 | 2324.6ms/step | 10.3 samples/s +Step 18720/20005 | Loss: 2.9366 | LR: 5.18e-06 | 2321.9ms/step | 10.3 samples/s +Step 18730/20005 | Loss: 2.9471 | LR: 5.10e-06 | 2324.7ms/step | 10.3 samples/s +Step 18740/20005 | Loss: 2.9533 | LR: 5.02e-06 | 2322.0ms/step | 10.3 samples/s +Step 18750/20005 | Loss: 2.9640 | LR: 4.94e-06 | 2324.2ms/step | 10.3 samples/s +Validation at step 18750: Loss = 2.9448, Perplexity = 19.01 +Step 18760/20005 | Loss: 2.9110 | LR: 4.86e-06 | 17455.1ms/step | 1.4 samples/s +Step 18770/20005 | Loss: 2.9430 | LR: 4.78e-06 | 2324.1ms/step | 10.3 samples/s +Step 18780/20005 | Loss: 2.9269 | LR: 4.71e-06 | 2324.1ms/step | 10.3 samples/s +Step 18790/20005 | Loss: 2.9330 | LR: 4.63e-06 | 2322.9ms/step | 10.3 samples/s +Step 18800/20005 | Loss: 2.9265 | LR: 4.55e-06 | 2324.7ms/step | 10.3 samples/s +Step 18810/20005 | Loss: 2.9540 | LR: 4.48e-06 | 2322.3ms/step | 10.3 samples/s +Step 18820/20005 | Loss: 2.9309 | LR: 4.40e-06 | 2323.8ms/step | 10.3 samples/s +Step 18830/20005 | Loss: 2.9221 | LR: 4.33e-06 | 2322.5ms/step | 10.3 samples/s +Step 18840/20005 | Loss: 2.9307 | LR: 4.26e-06 | 2324.1ms/step | 10.3 samples/s +Step 18850/20005 | Loss: 2.9446 | LR: 4.18e-06 | 2324.1ms/step | 10.3 samples/s +Step 18860/20005 | Loss: 2.9341 | LR: 4.11e-06 | 2325.6ms/step | 10.3 samples/s +Step 18870/20005 | Loss: 2.9326 | LR: 4.04e-06 | 2320.8ms/step | 10.3 samples/s +Step 18880/20005 | Loss: 2.9186 | LR: 3.97e-06 | 2324.9ms/step | 10.3 samples/s +Step 18890/20005 | Loss: 2.8829 | LR: 3.90e-06 | 2321.1ms/step | 10.3 samples/s +Step 18900/20005 | Loss: 2.9331 | LR: 3.83e-06 | 2323.1ms/step | 10.3 samples/s +Step 18910/20005 | Loss: 2.9285 | LR: 3.76e-06 | 2322.2ms/step | 10.3 samples/s +Step 18920/20005 | Loss: 2.9191 | LR: 3.69e-06 | 2324.2ms/step | 10.3 samples/s +Step 18930/20005 | Loss: 2.9445 | LR: 3.62e-06 | 2322.4ms/step | 10.3 samples/s +Step 18940/20005 | Loss: 2.9198 | LR: 3.56e-06 | 2322.4ms/step | 10.3 samples/s +Step 18950/20005 | Loss: 2.9121 | LR: 3.49e-06 | 2322.2ms/step | 10.3 samples/s +Step 18960/20005 | Loss: 2.9489 | LR: 3.42e-06 | 2322.4ms/step | 10.3 samples/s +Step 18970/20005 | Loss: 2.9495 | LR: 3.36e-06 | 2323.5ms/step | 10.3 samples/s +Step 18980/20005 | Loss: 2.9574 | LR: 3.29e-06 | 2324.0ms/step | 10.3 samples/s +Step 18990/20005 | Loss: 2.9411 | LR: 3.23e-06 | 2323.4ms/step | 10.3 samples/s +Step 19000/20005 | Loss: 2.9165 | LR: 3.17e-06 | 2323.8ms/step | 10.3 samples/s +Validation at step 19000: Loss = 2.9448, Perplexity = 19.01 +Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-19000 +Step 19010/20005 | Loss: 2.9485 | LR: 3.10e-06 | 17957.6ms/step | 1.3 samples/s +Step 19020/20005 | Loss: 2.9212 | LR: 3.04e-06 | 2320.5ms/step | 10.3 samples/s +Step 19030/20005 | Loss: 2.9169 | LR: 2.98e-06 | 2321.8ms/step | 10.3 samples/s +Step 19040/20005 | Loss: 2.9269 | LR: 2.92e-06 | 2319.3ms/step | 10.3 samples/s +Step 19050/20005 | Loss: 2.9127 | LR: 2.86e-06 | 2323.1ms/step | 10.3 samples/s +Step 19060/20005 | Loss: 2.9314 | LR: 2.80e-06 | 2320.0ms/step | 10.3 samples/s +Step 19070/20005 | Loss: 2.9063 | LR: 2.74e-06 | 2321.0ms/step | 10.3 samples/s +Step 19080/20005 | Loss: 2.9503 | LR: 2.68e-06 | 2322.3ms/step | 10.3 samples/s +Step 19090/20005 | Loss: 2.9136 | LR: 2.62e-06 | 2322.7ms/step | 10.3 samples/s +Step 19100/20005 | Loss: 2.9468 | LR: 2.57e-06 | 2320.4ms/step | 10.3 samples/s +Step 19110/20005 | Loss: 2.9347 | LR: 2.51e-06 | 2321.7ms/step | 10.3 samples/s +Step 19120/20005 | Loss: 2.9539 | LR: 2.46e-06 | 2323.6ms/step | 10.3 samples/s +Step 19130/20005 | Loss: 2.9157 | LR: 2.40e-06 | 2323.4ms/step | 10.3 samples/s +Step 19140/20005 | Loss: 2.9245 | LR: 2.35e-06 | 2322.1ms/step | 10.3 samples/s +Step 19150/20005 | Loss: 2.9305 | LR: 2.29e-06 | 2322.3ms/step | 10.3 samples/s +Step 19160/20005 | Loss: 2.9189 | LR: 2.24e-06 | 2322.3ms/step | 10.3 samples/s +Step 19170/20005 | Loss: 2.9229 | LR: 2.19e-06 | 2320.1ms/step | 10.3 samples/s +Step 19180/20005 | Loss: 2.9198 | LR: 2.13e-06 | 2321.5ms/step | 10.3 samples/s +Step 19190/20005 | Loss: 2.9404 | LR: 2.08e-06 | 2321.7ms/step | 10.3 samples/s +Step 19200/20005 | Loss: 2.9275 | LR: 2.03e-06 | 2320.3ms/step | 10.3 samples/s +Step 19210/20005 | Loss: 2.8980 | LR: 1.98e-06 | 2321.6ms/step | 10.3 samples/s +Step 19220/20005 | Loss: 2.9544 | LR: 1.93e-06 | 2321.7ms/step | 10.3 samples/s +Step 19230/20005 | Loss: 2.9006 | LR: 1.88e-06 | 2321.7ms/step | 10.3 samples/s +Step 19240/20005 | Loss: 2.9618 | LR: 1.83e-06 | 2322.7ms/step | 10.3 samples/s +Step 19250/20005 | Loss: 2.9312 | LR: 1.79e-06 | 2322.5ms/step | 10.3 samples/s +Validation at step 19250: Loss = 2.9449, Perplexity = 19.01 +Step 19260/20005 | Loss: 2.9488 | LR: 1.74e-06 | 17457.2ms/step | 1.4 samples/s +Step 19270/20005 | Loss: 2.9053 | LR: 1.69e-06 | 2323.7ms/step | 10.3 samples/s +Step 19280/20005 | Loss: 2.9239 | LR: 1.65e-06 | 2322.0ms/step | 10.3 samples/s +Step 19290/20005 | Loss: 2.9398 | LR: 1.60e-06 | 2324.0ms/step | 10.3 samples/s +Step 19300/20005 | Loss: 2.9380 | LR: 1.56e-06 | 2320.1ms/step | 10.3 samples/s +Step 19310/20005 | Loss: 2.9300 | LR: 1.51e-06 | 2324.3ms/step | 10.3 samples/s +Step 19320/20005 | Loss: 2.9248 | LR: 1.47e-06 | 2323.7ms/step | 10.3 samples/s +Step 19330/20005 | Loss: 2.9279 | LR: 1.43e-06 | 2323.4ms/step | 10.3 samples/s +Step 19340/20005 | Loss: 2.9459 | LR: 1.38e-06 | 2325.4ms/step | 10.3 samples/s +Step 19350/20005 | Loss: 2.9041 | LR: 1.34e-06 | 2322.1ms/step | 10.3 samples/s +Step 19360/20005 | Loss: 2.9376 | LR: 1.30e-06 | 2322.6ms/step | 10.3 samples/s +Step 19370/20005 | Loss: 2.9294 | LR: 1.26e-06 | 2322.7ms/step | 10.3 samples/s diff --git a/epfl_final/log_files/sparse_30%_continued.log b/epfl_final/log_files/sparse_30%_continued.log new file mode 100644 index 0000000000000000000000000000000000000000..488bfbf96d1ace2b0af9d01055af641c66f71a14 --- /dev/null +++ b/epfl_final/log_files/sparse_30%_continued.log @@ -0,0 +1,226 @@ +New run: Checkpoints will be saved to ./checkpoints/run_20260519_212829 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.0 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260519_212833-oeak7lgl +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dense continued 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/oeak7lgl +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260519_212829", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": 20005, + "per_device_batch_size": 3, + "gradient_accumulation_steps": 8, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 250, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 200, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "dense continued 8k", + "wandb_tags": [ + "qwen3", + "0.6b", + "dense-resume" + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": true +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1024 + num_hidden_layers: 28 + num_attention_heads: 16 + num_key_value_heads: 8 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 596,049,920 + Trainable parameters: 596,049,920 + Non-trainable parameters: 0 + Model size (BF16): 1.19 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [3, 8192] -- -- True +├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True +│ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True +│ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True +├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True +================================================================================================================================================================ +Total params: 751,632,384 +Trainable params: 751,632,384 +Non-trainable params: 0 +Total mult-adds (G): 2.25 +================================================================================================================================================================ +Input size (MB): 0.20 +Forward/backward pass size (MB): 63052.97 +Params size (MB): 1503.26 +Estimated Total Size (MB): 64556.43 +================================================================================================================================================================ +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized... +Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003) +Dataset split: 1017602 train, 254401 validation +Training examples (packed, train split): 1017602 +Validation examples (packed, val split): 254401 +Per-device batch size: 3 +World size (GPUs): 1 +Gradient accumulation steps: 8 +Effective batch size (seqs/step): 24 +Sequence length: 8192 +Tokens per optimizer step: 196,608 +Micro-batches per epoch (per rank): 339200 +Optimizer steps per epoch: 42400 +Total optimizer steps: 20005 (capped by max_steps; 0.47 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000... +Resumed from checkpoint at step 17999 (epoch 0, micro_step 144024) +Resuming training from step 17999 +Starting training... +Step 18000/20005 | Loss: 0.2932 | LR: 1.26e-05 | 638407.9ms/step | 0.0 samples/s +Validation at step 18000: Loss = 2.9421, Perplexity = 18.96 +Saved checkpoint to checkpoints/run_20260519_212829/checkpoint-18000 +Step 18010/20005 | Loss: 2.9429 | LR: 1.24e-05 | 44368.4ms/step | 0.5 samples/s +Step 18020/20005 | Loss: 2.8996 | LR: 1.23e-05 | 5892.3ms/step | 4.1 samples/s +Step 18030/20005 | Loss: 2.8949 | LR: 1.22e-05 | 5974.8ms/step | 4.0 samples/s +Step 18040/20005 | Loss: 2.8822 | LR: 1.21e-05 | 5889.9ms/step | 4.1 samples/s +Step 18050/20005 | Loss: 2.9279 | LR: 1.19e-05 | 5891.4ms/step | 4.1 samples/s +Step 18060/20005 | Loss: 2.9004 | LR: 1.18e-05 | 5883.8ms/step | 4.1 samples/s +Step 18070/20005 | Loss: 2.8831 | LR: 1.17e-05 | 5895.3ms/step | 4.1 samples/s +Step 18080/20005 | Loss: 2.8618 | LR: 1.16e-05 | 5892.2ms/step | 4.1 samples/s +Step 18090/20005 | Loss: 2.8818 | LR: 1.15e-05 | 5892.1ms/step | 4.1 samples/s +Step 18100/20005 | Loss: 2.8697 | LR: 1.13e-05 | 5898.6ms/step | 4.1 samples/s +Step 18110/20005 | Loss: 2.8987 | LR: 1.12e-05 | 5892.7ms/step | 4.1 samples/s +Step 18120/20005 | Loss: 2.9018 | LR: 1.11e-05 | 5889.0ms/step | 4.1 samples/s +Step 18130/20005 | Loss: 2.9144 | LR: 1.10e-05 | 5893.2ms/step | 4.1 samples/s +Step 18140/20005 | Loss: 2.9089 | LR: 1.09e-05 | 5888.2ms/step | 4.1 samples/s +Step 18150/20005 | Loss: 2.8999 | LR: 1.08e-05 | 5889.4ms/step | 4.1 samples/s +Step 18160/20005 | Loss: 2.8862 | LR: 1.06e-05 | 5887.0ms/step | 4.1 samples/s +Step 18170/20005 | Loss: 2.9011 | LR: 1.05e-05 | 5886.8ms/step | 4.1 samples/s +Step 18180/20005 | Loss: 2.9096 | LR: 1.04e-05 | 5897.3ms/step | 4.1 samples/s +Step 18190/20005 | Loss: 2.8956 | LR: 1.03e-05 | 5897.9ms/step | 4.1 samples/s +Step 18200/20005 | Loss: 2.8945 | LR: 1.02e-05 | 5894.8ms/step | 4.1 samples/s +Step 18210/20005 | Loss: 2.8873 | LR: 1.01e-05 | 5887.9ms/step | 4.1 samples/s +Step 18220/20005 | Loss: 2.8870 | LR: 9.96e-06 | 5889.2ms/step | 4.1 samples/s +Step 18230/20005 | Loss: 2.9008 | LR: 9.85e-06 | 5886.2ms/step | 4.1 samples/s +Step 18240/20005 | Loss: 2.8994 | LR: 9.74e-06 | 5890.5ms/step | 4.1 samples/s +Step 18250/20005 | Loss: 2.9039 | LR: 9.63e-06 | 5887.4ms/step | 4.1 samples/s +Validation at step 18250: Loss = 2.9114, Perplexity = 18.38 +Step 18260/20005 | Loss: 2.8679 | LR: 9.52e-06 | 40989.8ms/step | 0.6 samples/s +Step 18270/20005 | Loss: 2.8799 | LR: 9.41e-06 | 5890.8ms/step | 4.1 samples/s +Step 18280/20005 | Loss: 2.8720 | LR: 9.30e-06 | 5890.4ms/step | 4.1 samples/s +Step 18290/20005 | Loss: 2.8980 | LR: 9.20e-06 | 5890.3ms/step | 4.1 samples/s +Step 18300/20005 | Loss: 2.9070 | LR: 9.09e-06 | 5890.3ms/step | 4.1 samples/s +Step 18310/20005 | Loss: 2.8755 | LR: 8.99e-06 | 5888.9ms/step | 4.1 samples/s +Step 18320/20005 | Loss: 2.9267 | LR: 8.88e-06 | 5891.0ms/step | 4.1 samples/s +Step 18330/20005 | Loss: 2.9028 | LR: 8.78e-06 | 5887.3ms/step | 4.1 samples/s +Step 18340/20005 | Loss: 2.8972 | LR: 8.67e-06 | 5883.4ms/step | 4.1 samples/s +Step 18350/20005 | Loss: 2.9147 | LR: 8.57e-06 | 5882.8ms/step | 4.1 samples/s +Step 18360/20005 | Loss: 2.8991 | LR: 8.46e-06 | 5897.9ms/step | 4.1 samples/s +Step 18370/20005 | Loss: 2.9054 | LR: 8.36e-06 | 5898.7ms/step | 4.1 samples/s +Step 18380/20005 | Loss: 2.8886 | LR: 8.26e-06 | 5897.2ms/step | 4.1 samples/s +Step 18390/20005 | Loss: 2.8998 | LR: 8.16e-06 | 5885.8ms/step | 4.1 samples/s +Step 18400/20005 | Loss: 2.9087 | LR: 8.06e-06 | 5890.7ms/step | 4.1 samples/s +Step 18410/20005 | Loss: 2.8897 | LR: 7.96e-06 | 5889.8ms/step | 4.1 samples/s +Step 18420/20005 | Loss: 2.8887 | LR: 7.86e-06 | 5887.1ms/step | 4.1 samples/s +Step 18430/20005 | Loss: 2.8875 | LR: 7.76e-06 | 5885.5ms/step | 4.1 samples/s +Step 18440/20005 | Loss: 2.8767 | LR: 7.66e-06 | 5888.8ms/step | 4.1 samples/s +Step 18450/20005 | Loss: 2.9146 | LR: 7.57e-06 | 5886.7ms/step | 4.1 samples/s +Step 18460/20005 | Loss: 2.8945 | LR: 7.47e-06 | 5884.0ms/step | 4.1 samples/s +Step 18470/20005 | Loss: 2.9128 | LR: 7.37e-06 | 5895.7ms/step | 4.1 samples/s +Step 18480/20005 | Loss: 2.9192 | LR: 7.28e-06 | 5897.7ms/step | 4.1 samples/s +Step 18490/20005 | Loss: 2.8914 | LR: 7.18e-06 | 5895.0ms/step | 4.1 samples/s +Step 18500/20005 | Loss: 2.8621 | LR: 7.09e-06 | 5954.1ms/step | 4.0 samples/s +Validation at step 18500: Loss = 2.9103, Perplexity = 18.36 +Step 18510/20005 | Loss: 2.9280 | LR: 6.99e-06 | 41109.7ms/step | 0.6 samples/s +Step 18520/20005 | Loss: 2.8590 | LR: 6.90e-06 | 5906.6ms/step | 4.1 samples/s +Step 18530/20005 | Loss: 2.8556 | LR: 6.81e-06 | 5913.0ms/step | 4.1 samples/s +Step 18540/20005 | Loss: 2.8744 | LR: 6.72e-06 | 5908.0ms/step | 4.1 samples/s +Step 18550/20005 | Loss: 2.8819 | LR: 6.63e-06 | 5907.4ms/step | 4.1 samples/s +Step 18560/20005 | Loss: 2.8989 | LR: 6.54e-06 | 5904.5ms/step | 4.1 samples/s +Step 18570/20005 | Loss: 2.8603 | LR: 6.45e-06 | 5903.6ms/step | 4.1 samples/s +Step 18580/20005 | Loss: 2.9193 | LR: 6.36e-06 | 5907.4ms/step | 4.1 samples/s +Step 18590/20005 | Loss: 2.8858 | LR: 6.27e-06 | 5903.6ms/step | 4.1 samples/s +Step 18600/20005 | Loss: 2.9107 | LR: 6.18e-06 | 5915.9ms/step | 4.1 samples/s +Step 18610/20005 | Loss: 2.9068 | LR: 6.09e-06 | 5902.7ms/step | 4.1 samples/s diff --git a/epfl_final/log_files/sparse_eval_dense.log b/epfl_final/log_files/sparse_eval_dense.log new file mode 100644 index 0000000000000000000000000000000000000000..c8df61b33f6f82a5920320f9eeb2578db55efa19 --- /dev/null +++ b/epfl_final/log_files/sparse_eval_dense.log @@ -0,0 +1,1323 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260616_011729 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260616_011729", + "resume_from_checkpoint": "checkpoints/run_20260523_215035/checkpoint-40000", + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "sparse 1mask", + "wandb_tags": [ + "qwen3", + "1.2b", + "sparse_stochastic" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: checkpoints/run_20260523_215035/checkpoint-40000 +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... + +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from checkpoints/run_20260523_215035/checkpoint-40000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 40000 (epoch 0, micro_step 400000) +Resuming training from step 40000 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.2 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260616_012002-apvt4901 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run sparse 1mask +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/apvt4901 +Starting training... +Validation at step 40000: Loss = 2.7035, Perplexity = 14.93 +Switching attention mode for step 40001: sparse_stochastic (configured sparse attention) +Step 40010/50462 | Loss: 2.7082 | LR: 5.26e-05 | 58207.0ms/step | 1.0 samples/s +Step 40020/50462 | Loss: 2.7067 | LR: 5.25e-05 | 3114.1ms/step | 19.3 samples/s +Step 40030/50462 | Loss: 2.6470 | LR: 5.24e-05 | 3121.8ms/step | 19.2 samples/s +Step 40040/50462 | Loss: 2.6722 | LR: 5.23e-05 | 3122.8ms/step | 19.2 samples/s +Step 40050/50462 | Loss: 2.6606 | LR: 5.22e-05 | 3124.2ms/step | 19.2 samples/s +Step 40060/50462 | Loss: 2.6534 | LR: 5.21e-05 | 3125.4ms/step | 19.2 samples/s +Step 40070/50462 | Loss: 2.6650 | LR: 5.20e-05 | 3113.3ms/step | 19.3 samples/s +Step 40080/50462 | Loss: 2.6391 | LR: 5.19e-05 | 3115.6ms/step | 19.3 samples/s +Step 40090/50462 | Loss: 2.6721 | LR: 5.18e-05 | 3119.6ms/step | 19.2 samples/s +Step 40100/50462 | Loss: 2.6684 | LR: 5.17e-05 | 3127.1ms/step | 19.2 samples/s +Step 40110/50462 | Loss: 2.6566 | LR: 5.16e-05 | 3124.9ms/step | 19.2 samples/s +Step 40120/50462 | Loss: 2.6681 | LR: 5.15e-05 | 3114.0ms/step | 19.3 samples/s +Step 40130/50462 | Loss: 2.6465 | LR: 5.14e-05 | 3117.6ms/step | 19.2 samples/s +Step 40140/50462 | Loss: 2.6470 | LR: 5.13e-05 | 3114.3ms/step | 19.3 samples/s +Step 40150/50462 | Loss: 2.6224 | LR: 5.12e-05 | 3119.1ms/step | 19.2 samples/s +Step 40160/50462 | Loss: 2.6181 | LR: 5.11e-05 | 3193.4ms/step | 18.8 samples/s +Step 40170/50462 | Loss: 2.6592 | LR: 5.10e-05 | 3130.4ms/step | 19.2 samples/s +Step 40180/50462 | Loss: 2.6618 | LR: 5.09e-05 | 3120.6ms/step | 19.2 samples/s +Step 40190/50462 | Loss: 2.6587 | LR: 5.09e-05 | 3126.3ms/step | 19.2 samples/s +Step 40200/50462 | Loss: 2.6662 | LR: 5.08e-05 | 3124.4ms/step | 19.2 samples/s +Step 40210/50462 | Loss: 2.6719 | LR: 5.07e-05 | 3184.7ms/step | 18.8 samples/s +Step 40220/50462 | Loss: 2.6685 | LR: 5.06e-05 | 3122.8ms/step | 19.2 samples/s +Step 40230/50462 | Loss: 2.6634 | LR: 5.05e-05 | 3125.6ms/step | 19.2 samples/s +Step 40240/50462 | Loss: 2.6897 | LR: 5.04e-05 | 3127.3ms/step | 19.2 samples/s +Step 40250/50462 | Loss: 2.6498 | LR: 5.03e-05 | 3119.3ms/step | 19.2 samples/s +Validation at step 40250: Loss = 2.6608, Perplexity = 14.31 +Step 40260/50462 | Loss: 2.6581 | LR: 5.02e-05 | 4781.8ms/step | 12.5 samples/s +Step 40270/50462 | Loss: 2.6324 | LR: 5.01e-05 | 3123.0ms/step | 19.2 samples/s +Step 40280/50462 | Loss: 2.6559 | LR: 5.00e-05 | 3122.6ms/step | 19.2 samples/s +Step 40290/50462 | Loss: 2.6511 | LR: 4.99e-05 | 3126.6ms/step | 19.2 samples/s +Step 40300/50462 | Loss: 2.6480 | LR: 4.98e-05 | 3123.7ms/step | 19.2 samples/s +Step 40310/50462 | Loss: 2.6356 | LR: 4.97e-05 | 3129.0ms/step | 19.2 samples/s +Step 40320/50462 | Loss: 2.6541 | LR: 4.96e-05 | 3121.3ms/step | 19.2 samples/s +Step 40330/50462 | Loss: 2.6754 | LR: 4.95e-05 | 3123.3ms/step | 19.2 samples/s +Step 40340/50462 | Loss: 2.6381 | LR: 4.94e-05 | 3122.1ms/step | 19.2 samples/s +Step 40350/50462 | Loss: 2.6280 | LR: 4.93e-05 | 3120.3ms/step | 19.2 samples/s +Step 40360/50462 | Loss: 2.6519 | LR: 4.92e-05 | 3125.3ms/step | 19.2 samples/s +Step 40370/50462 | Loss: 2.6669 | LR: 4.91e-05 | 3127.9ms/step | 19.2 samples/s +Step 40380/50462 | Loss: 2.6537 | LR: 4.90e-05 | 3120.6ms/step | 19.2 samples/s +Step 40390/50462 | Loss: 2.6498 | LR: 4.90e-05 | 3125.7ms/step | 19.2 samples/s +Step 40400/50462 | Loss: 2.6641 | LR: 4.89e-05 | 3123.8ms/step | 19.2 samples/s +Step 40410/50462 | Loss: 2.6764 | LR: 4.88e-05 | 3124.2ms/step | 19.2 samples/s +Step 40420/50462 | Loss: 2.6737 | LR: 4.87e-05 | 3125.2ms/step | 19.2 samples/s +Step 40430/50462 | Loss: 2.6439 | LR: 4.86e-05 | 3129.5ms/step | 19.2 samples/s +Step 40440/50462 | Loss: 2.6604 | LR: 4.85e-05 | 3128.8ms/step | 19.2 samples/s +Step 40450/50462 | Loss: 2.6494 | LR: 4.84e-05 | 3123.3ms/step | 19.2 samples/s +Step 40460/50462 | Loss: 2.6850 | LR: 4.83e-05 | 3122.2ms/step | 19.2 samples/s +Step 40470/50462 | Loss: 2.6370 | LR: 4.82e-05 | 3124.7ms/step | 19.2 samples/s +Step 40480/50462 | Loss: 2.6473 | LR: 4.81e-05 | 3187.6ms/step | 18.8 samples/s +Step 40490/50462 | Loss: 2.6615 | LR: 4.80e-05 | 3121.7ms/step | 19.2 samples/s +Step 40500/50462 | Loss: 2.6541 | LR: 4.79e-05 | 3120.5ms/step | 19.2 samples/s +Validation at step 40500: Loss = 2.6585, Perplexity = 14.27 +Step 40510/50462 | Loss: 2.6518 | LR: 4.78e-05 | 4784.4ms/step | 12.5 samples/s +Step 40520/50462 | Loss: 2.6362 | LR: 4.77e-05 | 3128.3ms/step | 19.2 samples/s +Step 40530/50462 | Loss: 2.6407 | LR: 4.76e-05 | 3180.9ms/step | 18.9 samples/s +Step 40540/50462 | Loss: 2.6172 | LR: 4.76e-05 | 3126.6ms/step | 19.2 samples/s +Step 40550/50462 | Loss: 2.6673 | LR: 4.75e-05 | 3118.5ms/step | 19.2 samples/s +Step 40560/50462 | Loss: 2.6612 | LR: 4.74e-05 | 3129.7ms/step | 19.2 samples/s +Step 40570/50462 | Loss: 2.6362 | LR: 4.73e-05 | 3125.2ms/step | 19.2 samples/s +Step 40580/50462 | Loss: 2.6611 | LR: 4.72e-05 | 3123.4ms/step | 19.2 samples/s +Step 40590/50462 | Loss: 2.6473 | LR: 4.71e-05 | 3121.1ms/step | 19.2 samples/s +Step 40600/50462 | Loss: 2.6488 | LR: 4.70e-05 | 3122.1ms/step | 19.2 samples/s +Step 40610/50462 | Loss: 2.6682 | LR: 4.69e-05 | 3124.5ms/step | 19.2 samples/s +Step 40620/50462 | Loss: 2.6237 | LR: 4.68e-05 | 3126.5ms/step | 19.2 samples/s +Step 40630/50462 | Loss: 2.6426 | LR: 4.67e-05 | 3121.7ms/step | 19.2 samples/s +Step 40640/50462 | Loss: 2.6532 | LR: 4.66e-05 | 3125.7ms/step | 19.2 samples/s +Step 40650/50462 | Loss: 2.6554 | LR: 4.65e-05 | 3123.1ms/step | 19.2 samples/s +Step 40660/50462 | Loss: 2.6634 | LR: 4.64e-05 | 3127.2ms/step | 19.2 samples/s +Step 40670/50462 | Loss: 2.6435 | LR: 4.64e-05 | 3125.6ms/step | 19.2 samples/s +Step 40680/50462 | Loss: 2.6486 | LR: 4.63e-05 | 3120.1ms/step | 19.2 samples/s +Step 40690/50462 | Loss: 2.6423 | LR: 4.62e-05 | 3122.6ms/step | 19.2 samples/s +Step 40700/50462 | Loss: 2.6177 | LR: 4.61e-05 | 3121.9ms/step | 19.2 samples/s +Step 40710/50462 | Loss: 2.6586 | LR: 4.60e-05 | 3124.7ms/step | 19.2 samples/s +Step 40720/50462 | Loss: 2.6209 | LR: 4.59e-05 | 3127.2ms/step | 19.2 samples/s +Step 40730/50462 | Loss: 2.6326 | LR: 4.58e-05 | 3119.0ms/step | 19.2 samples/s +Step 40740/50462 | Loss: 2.6319 | LR: 4.57e-05 | 3121.8ms/step | 19.2 samples/s +Step 40750/50462 | Loss: 2.6411 | LR: 4.56e-05 | 3125.7ms/step | 19.2 samples/s +Validation at step 40750: Loss = 2.6570, Perplexity = 14.25 +Step 40760/50462 | Loss: 2.6677 | LR: 4.55e-05 | 4788.3ms/step | 12.5 samples/s +Step 40770/50462 | Loss: 2.6458 | LR: 4.54e-05 | 3128.4ms/step | 19.2 samples/s +Step 40780/50462 | Loss: 2.6611 | LR: 4.54e-05 | 3127.1ms/step | 19.2 samples/s +Step 40790/50462 | Loss: 2.6447 | LR: 4.53e-05 | 3124.8ms/step | 19.2 samples/s +Step 40800/50462 | Loss: 2.6538 | LR: 4.52e-05 | 3185.1ms/step | 18.8 samples/s +Step 40810/50462 | Loss: 2.6106 | LR: 4.51e-05 | 3118.0ms/step | 19.2 samples/s +Step 40820/50462 | Loss: 2.6861 | LR: 4.50e-05 | 3114.1ms/step | 19.3 samples/s +Step 40830/50462 | Loss: 2.6020 | LR: 4.49e-05 | 3119.0ms/step | 19.2 samples/s +Step 40840/50462 | Loss: 2.6264 | LR: 4.48e-05 | 3120.5ms/step | 19.2 samples/s +Step 40850/50462 | Loss: 2.6688 | LR: 4.47e-05 | 3123.3ms/step | 19.2 samples/s +Step 40860/50462 | Loss: 2.6123 | LR: 4.46e-05 | 3180.5ms/step | 18.9 samples/s +Step 40870/50462 | Loss: 2.6717 | LR: 4.45e-05 | 3117.5ms/step | 19.2 samples/s +Step 40880/50462 | Loss: 2.6772 | LR: 4.45e-05 | 3116.1ms/step | 19.3 samples/s +Step 40890/50462 | Loss: 2.6077 | LR: 4.44e-05 | 3120.8ms/step | 19.2 samples/s +Step 40900/50462 | Loss: 2.6344 | LR: 4.43e-05 | 3118.5ms/step | 19.2 samples/s +Step 40910/50462 | Loss: 2.6552 | LR: 4.42e-05 | 3122.3ms/step | 19.2 samples/s +Step 40920/50462 | Loss: 2.6611 | LR: 4.41e-05 | 3119.1ms/step | 19.2 samples/s +Step 40930/50462 | Loss: 2.6526 | LR: 4.40e-05 | 3121.3ms/step | 19.2 samples/s +Step 40940/50462 | Loss: 2.6698 | LR: 4.39e-05 | 3119.1ms/step | 19.2 samples/s +Step 40950/50462 | Loss: 2.6247 | LR: 4.38e-05 | 3120.9ms/step | 19.2 samples/s +Step 40960/50462 | Loss: 2.6547 | LR: 4.37e-05 | 3120.2ms/step | 19.2 samples/s +Step 40970/50462 | Loss: 2.6436 | LR: 4.36e-05 | 3114.7ms/step | 19.3 samples/s +Step 40980/50462 | Loss: 2.6450 | LR: 4.36e-05 | 3120.9ms/step | 19.2 samples/s +Step 40990/50462 | Loss: 2.6142 | LR: 4.35e-05 | 3126.2ms/step | 19.2 samples/s +Step 41000/50462 | Loss: 2.6599 | LR: 4.34e-05 | 3127.9ms/step | 19.2 samples/s +Validation at step 41000: Loss = 2.6562, Perplexity = 14.24 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-41000 +Step 41010/50462 | Loss: 2.6629 | LR: 4.33e-05 | 5643.9ms/step | 10.6 samples/s +Step 41020/50462 | Loss: 2.6598 | LR: 4.32e-05 | 3118.6ms/step | 19.2 samples/s +Step 41030/50462 | Loss: 2.6610 | LR: 4.31e-05 | 3123.6ms/step | 19.2 samples/s +Step 41040/50462 | Loss: 2.6267 | LR: 4.30e-05 | 3112.6ms/step | 19.3 samples/s +Step 41050/50462 | Loss: 2.6599 | LR: 4.29e-05 | 3117.9ms/step | 19.2 samples/s +Step 41060/50462 | Loss: 2.6648 | LR: 4.28e-05 | 3117.9ms/step | 19.2 samples/s +Step 41070/50462 | Loss: 2.6405 | LR: 4.28e-05 | 3120.3ms/step | 19.2 samples/s +Step 41080/50462 | Loss: 2.6324 | LR: 4.27e-05 | 3120.3ms/step | 19.2 samples/s +Step 41090/50462 | Loss: 2.6138 | LR: 4.26e-05 | 3115.3ms/step | 19.3 samples/s +Step 41100/50462 | Loss: 2.6457 | LR: 4.25e-05 | 3119.2ms/step | 19.2 samples/s +Step 41110/50462 | Loss: 2.6486 | LR: 4.24e-05 | 3122.7ms/step | 19.2 samples/s +Step 41120/50462 | Loss: 2.6409 | LR: 4.23e-05 | 3176.5ms/step | 18.9 samples/s +Step 41130/50462 | Loss: 2.6642 | LR: 4.22e-05 | 3118.5ms/step | 19.2 samples/s +Step 41140/50462 | Loss: 2.6368 | LR: 4.21e-05 | 3114.5ms/step | 19.3 samples/s +Step 41150/50462 | Loss: 2.6482 | LR: 4.21e-05 | 3118.2ms/step | 19.2 samples/s +Step 41160/50462 | Loss: 2.6491 | LR: 4.20e-05 | 3119.2ms/step | 19.2 samples/s +Step 41170/50462 | Loss: 2.6558 | LR: 4.19e-05 | 3119.3ms/step | 19.2 samples/s +Step 41180/50462 | Loss: 2.6378 | LR: 4.18e-05 | 3187.8ms/step | 18.8 samples/s +Step 41190/50462 | Loss: 2.5967 | LR: 4.17e-05 | 3124.1ms/step | 19.2 samples/s +Step 41200/50462 | Loss: 2.6352 | LR: 4.16e-05 | 3121.4ms/step | 19.2 samples/s +Step 41210/50462 | Loss: 2.6102 | LR: 4.15e-05 | 3120.3ms/step | 19.2 samples/s +Step 41220/50462 | Loss: 2.6255 | LR: 4.14e-05 | 3128.2ms/step | 19.2 samples/s +Step 41230/50462 | Loss: 2.6496 | LR: 4.14e-05 | 3119.9ms/step | 19.2 samples/s +Step 41240/50462 | Loss: 2.6420 | LR: 4.13e-05 | 3126.4ms/step | 19.2 samples/s +Step 41250/50462 | Loss: 2.6535 | LR: 4.12e-05 | 3125.3ms/step | 19.2 samples/s +Validation at step 41250: Loss = 2.6555, Perplexity = 14.23 +Step 41260/50462 | Loss: 2.6492 | LR: 4.11e-05 | 4778.9ms/step | 12.6 samples/s +Step 41270/50462 | Loss: 2.6237 | LR: 4.10e-05 | 3120.8ms/step | 19.2 samples/s +Step 41280/50462 | Loss: 2.6025 | LR: 4.09e-05 | 3117.6ms/step | 19.2 samples/s +Step 41290/50462 | Loss: 2.6645 | LR: 4.08e-05 | 3118.0ms/step | 19.2 samples/s +Step 41300/50462 | Loss: 2.6537 | LR: 4.07e-05 | 3121.5ms/step | 19.2 samples/s +Step 41310/50462 | Loss: 2.6175 | LR: 4.07e-05 | 3119.6ms/step | 19.2 samples/s +Step 41320/50462 | Loss: 2.6966 | LR: 4.06e-05 | 3124.1ms/step | 19.2 samples/s +Step 41330/50462 | Loss: 2.6337 | LR: 4.05e-05 | 3118.4ms/step | 19.2 samples/s +Step 41340/50462 | Loss: 2.6287 | LR: 4.04e-05 | 3114.9ms/step | 19.3 samples/s +Step 41350/50462 | Loss: 2.6473 | LR: 4.03e-05 | 3114.1ms/step | 19.3 samples/s +Step 41360/50462 | Loss: 2.6744 | LR: 4.02e-05 | 3117.6ms/step | 19.2 samples/s +Step 41370/50462 | Loss: 2.6453 | LR: 4.01e-05 | 3123.1ms/step | 19.2 samples/s +Step 41380/50462 | Loss: 2.6421 | LR: 4.01e-05 | 3127.2ms/step | 19.2 samples/s +Step 41390/50462 | Loss: 2.6142 | LR: 4.00e-05 | 3125.8ms/step | 19.2 samples/s +Step 41400/50462 | Loss: 2.6553 | LR: 3.99e-05 | 3116.1ms/step | 19.3 samples/s +Step 41410/50462 | Loss: 2.6578 | LR: 3.98e-05 | 3115.0ms/step | 19.3 samples/s +Step 41420/50462 | Loss: 2.6219 | LR: 3.97e-05 | 3114.0ms/step | 19.3 samples/s +Step 41430/50462 | Loss: 2.6545 | LR: 3.96e-05 | 3119.6ms/step | 19.2 samples/s +Step 41440/50462 | Loss: 2.6243 | LR: 3.95e-05 | 3178.2ms/step | 18.9 samples/s +Step 41450/50462 | Loss: 2.6441 | LR: 3.95e-05 | 3125.2ms/step | 19.2 samples/s +Step 41460/50462 | Loss: 2.6277 | LR: 3.94e-05 | 3112.1ms/step | 19.3 samples/s +Step 41470/50462 | Loss: 2.6052 | LR: 3.93e-05 | 3118.4ms/step | 19.2 samples/s +Step 41480/50462 | Loss: 2.6624 | LR: 3.92e-05 | 3120.4ms/step | 19.2 samples/s +Step 41490/50462 | Loss: 2.6160 | LR: 3.91e-05 | 3112.6ms/step | 19.3 samples/s +Step 41500/50462 | Loss: 2.6604 | LR: 3.90e-05 | 3179.6ms/step | 18.9 samples/s +Validation at step 41500: Loss = 2.6550, Perplexity = 14.22 +Step 41510/50462 | Loss: 2.6293 | LR: 3.90e-05 | 4768.7ms/step | 12.6 samples/s +Step 41520/50462 | Loss: 2.6439 | LR: 3.89e-05 | 3119.3ms/step | 19.2 samples/s +Step 41530/50462 | Loss: 2.6562 | LR: 3.88e-05 | 3114.5ms/step | 19.3 samples/s +Step 41540/50462 | Loss: 2.6264 | LR: 3.87e-05 | 3121.4ms/step | 19.2 samples/s +Step 41550/50462 | Loss: 2.6536 | LR: 3.86e-05 | 3120.9ms/step | 19.2 samples/s +Step 41560/50462 | Loss: 2.6630 | LR: 3.85e-05 | 3117.2ms/step | 19.2 samples/s +Step 41570/50462 | Loss: 2.6510 | LR: 3.84e-05 | 3113.5ms/step | 19.3 samples/s +Step 41580/50462 | Loss: 2.6558 | LR: 3.84e-05 | 3125.3ms/step | 19.2 samples/s +Step 41590/50462 | Loss: 2.6374 | LR: 3.83e-05 | 3120.8ms/step | 19.2 samples/s +Step 41600/50462 | Loss: 2.6522 | LR: 3.82e-05 | 3118.4ms/step | 19.2 samples/s +Step 41610/50462 | Loss: 2.6356 | LR: 3.81e-05 | 3114.3ms/step | 19.3 samples/s +Step 41620/50462 | Loss: 2.6379 | LR: 3.80e-05 | 3119.7ms/step | 19.2 samples/s +Step 41630/50462 | Loss: 2.6227 | LR: 3.79e-05 | 3115.6ms/step | 19.3 samples/s +Step 41640/50462 | Loss: 2.6370 | LR: 3.79e-05 | 3115.8ms/step | 19.3 samples/s +Step 41650/50462 | Loss: 2.6254 | LR: 3.78e-05 | 3121.4ms/step | 19.2 samples/s +Step 41660/50462 | Loss: 2.6524 | LR: 3.77e-05 | 3119.4ms/step | 19.2 samples/s +Step 41670/50462 | Loss: 2.6448 | LR: 3.76e-05 | 3120.1ms/step | 19.2 samples/s +Step 41680/50462 | Loss: 2.6290 | LR: 3.75e-05 | 3118.5ms/step | 19.2 samples/s +Step 41690/50462 | Loss: 2.6264 | LR: 3.74e-05 | 3119.1ms/step | 19.2 samples/s +Step 41700/50462 | Loss: 2.6406 | LR: 3.74e-05 | 3117.3ms/step | 19.2 samples/s +Step 41710/50462 | Loss: 2.6284 | LR: 3.73e-05 | 3123.0ms/step | 19.2 samples/s +Step 41720/50462 | Loss: 2.6261 | LR: 3.72e-05 | 3118.6ms/step | 19.2 samples/s +Step 41730/50462 | Loss: 2.6323 | LR: 3.71e-05 | 3120.3ms/step | 19.2 samples/s +Step 41740/50462 | Loss: 2.6262 | LR: 3.70e-05 | 3122.9ms/step | 19.2 samples/s +Step 41750/50462 | Loss: 2.6356 | LR: 3.69e-05 | 3123.3ms/step | 19.2 samples/s +Validation at step 41750: Loss = 2.6546, Perplexity = 14.22 +Step 41760/50462 | Loss: 2.6432 | LR: 3.69e-05 | 4834.5ms/step | 12.4 samples/s +Step 41770/50462 | Loss: 2.6514 | LR: 3.68e-05 | 3118.8ms/step | 19.2 samples/s +Step 41780/50462 | Loss: 2.6389 | LR: 3.67e-05 | 3120.6ms/step | 19.2 samples/s +Step 41790/50462 | Loss: 2.6246 | LR: 3.66e-05 | 3123.6ms/step | 19.2 samples/s +Step 41800/50462 | Loss: 2.6269 | LR: 3.65e-05 | 3123.2ms/step | 19.2 samples/s +Step 41810/50462 | Loss: 2.6593 | LR: 3.64e-05 | 3118.7ms/step | 19.2 samples/s +Step 41820/50462 | Loss: 2.5862 | LR: 3.64e-05 | 3238.4ms/step | 18.5 samples/s +Step 41830/50462 | Loss: 2.6374 | LR: 3.63e-05 | 3123.9ms/step | 19.2 samples/s +Step 41840/50462 | Loss: 2.6123 | LR: 3.62e-05 | 3123.1ms/step | 19.2 samples/s +Step 41850/50462 | Loss: 2.6531 | LR: 3.61e-05 | 3114.7ms/step | 19.3 samples/s +Step 41860/50462 | Loss: 2.6394 | LR: 3.60e-05 | 3122.3ms/step | 19.2 samples/s +Step 41870/50462 | Loss: 2.6352 | LR: 3.60e-05 | 3130.3ms/step | 19.2 samples/s +Step 41880/50462 | Loss: 2.6490 | LR: 3.59e-05 | 3123.9ms/step | 19.2 samples/s +Step 41890/50462 | Loss: 2.6374 | LR: 3.58e-05 | 3122.2ms/step | 19.2 samples/s +Step 41900/50462 | Loss: 2.6636 | LR: 3.57e-05 | 3121.2ms/step | 19.2 samples/s +Step 41910/50462 | Loss: 2.6514 | LR: 3.56e-05 | 3117.5ms/step | 19.2 samples/s +Step 41920/50462 | Loss: 2.6478 | LR: 3.55e-05 | 3114.6ms/step | 19.3 samples/s +Step 41930/50462 | Loss: 2.6263 | LR: 3.55e-05 | 3123.8ms/step | 19.2 samples/s +Step 41940/50462 | Loss: 2.6554 | LR: 3.54e-05 | 3121.5ms/step | 19.2 samples/s +Step 41950/50462 | Loss: 2.6249 | LR: 3.53e-05 | 3116.5ms/step | 19.3 samples/s +Step 41960/50462 | Loss: 2.6470 | LR: 3.52e-05 | 3120.5ms/step | 19.2 samples/s +Step 41970/50462 | Loss: 2.6401 | LR: 3.51e-05 | 3123.1ms/step | 19.2 samples/s +Step 41980/50462 | Loss: 2.6362 | LR: 3.51e-05 | 3127.3ms/step | 19.2 samples/s +Step 41990/50462 | Loss: 2.6687 | LR: 3.50e-05 | 3127.9ms/step | 19.2 samples/s +Step 42000/50462 | Loss: 2.6465 | LR: 3.49e-05 | 3129.7ms/step | 19.2 samples/s +Validation at step 42000: Loss = 2.6542, Perplexity = 14.21 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-42000 +Step 42010/50462 | Loss: 2.6162 | LR: 3.48e-05 | 5655.4ms/step | 10.6 samples/s +Step 42020/50462 | Loss: 2.6599 | LR: 3.47e-05 | 3125.4ms/step | 19.2 samples/s +Step 42030/50462 | Loss: 2.6789 | LR: 3.47e-05 | 3124.6ms/step | 19.2 samples/s +Step 42040/50462 | Loss: 2.6435 | LR: 3.46e-05 | 3120.9ms/step | 19.2 samples/s +Step 42050/50462 | Loss: 2.6428 | LR: 3.45e-05 | 3124.6ms/step | 19.2 samples/s +Step 42060/50462 | Loss: 2.6787 | LR: 3.44e-05 | 3117.7ms/step | 19.2 samples/s +Step 42070/50462 | Loss: 2.6605 | LR: 3.43e-05 | 3122.4ms/step | 19.2 samples/s +Step 42080/50462 | Loss: 2.6295 | LR: 3.43e-05 | 3176.1ms/step | 18.9 samples/s +Step 42090/50462 | Loss: 2.6584 | LR: 3.42e-05 | 3113.5ms/step | 19.3 samples/s +Step 42100/50462 | Loss: 2.6443 | LR: 3.41e-05 | 3112.2ms/step | 19.3 samples/s +Step 42110/50462 | Loss: 2.6121 | LR: 3.40e-05 | 3122.5ms/step | 19.2 samples/s +Step 42120/50462 | Loss: 2.6468 | LR: 3.39e-05 | 3119.8ms/step | 19.2 samples/s +Step 42130/50462 | Loss: 2.6568 | LR: 3.39e-05 | 3116.8ms/step | 19.3 samples/s +Step 42140/50462 | Loss: 2.6541 | LR: 3.38e-05 | 3245.7ms/step | 18.5 samples/s +Step 42150/50462 | Loss: 2.6477 | LR: 3.37e-05 | 3126.5ms/step | 19.2 samples/s +Step 42160/50462 | Loss: 2.6088 | LR: 3.36e-05 | 3130.1ms/step | 19.2 samples/s +Step 42170/50462 | Loss: 2.6365 | LR: 3.35e-05 | 3122.9ms/step | 19.2 samples/s +Step 42180/50462 | Loss: 2.6434 | LR: 3.35e-05 | 3124.2ms/step | 19.2 samples/s +Step 42190/50462 | Loss: 2.6717 | LR: 3.34e-05 | 3127.9ms/step | 19.2 samples/s +Step 42200/50462 | Loss: 2.6297 | LR: 3.33e-05 | 3123.8ms/step | 19.2 samples/s +Step 42210/50462 | Loss: 2.6343 | LR: 3.32e-05 | 3124.7ms/step | 19.2 samples/s +Step 42220/50462 | Loss: 2.6607 | LR: 3.32e-05 | 3121.9ms/step | 19.2 samples/s +Step 42230/50462 | Loss: 2.6485 | LR: 3.31e-05 | 3124.7ms/step | 19.2 samples/s +Step 42240/50462 | Loss: 2.6027 | LR: 3.30e-05 | 3131.8ms/step | 19.2 samples/s +Step 42250/50462 | Loss: 2.6541 | LR: 3.29e-05 | 3128.0ms/step | 19.2 samples/s +Validation at step 42250: Loss = 2.6539, Perplexity = 14.21 +Step 42260/50462 | Loss: 2.6599 | LR: 3.28e-05 | 4787.6ms/step | 12.5 samples/s +Step 42270/50462 | Loss: 2.6695 | LR: 3.28e-05 | 3118.7ms/step | 19.2 samples/s +Step 42280/50462 | Loss: 2.6403 | LR: 3.27e-05 | 3127.8ms/step | 19.2 samples/s +Step 42290/50462 | Loss: 2.6557 | LR: 3.26e-05 | 3120.9ms/step | 19.2 samples/s +Step 42300/50462 | Loss: 2.6404 | LR: 3.25e-05 | 3126.7ms/step | 19.2 samples/s +Step 42310/50462 | Loss: 2.6336 | LR: 3.24e-05 | 3125.6ms/step | 19.2 samples/s +Step 42320/50462 | Loss: 2.6158 | LR: 3.24e-05 | 3122.8ms/step | 19.2 samples/s +Step 42330/50462 | Loss: 2.6625 | LR: 3.23e-05 | 3121.2ms/step | 19.2 samples/s +Step 42340/50462 | Loss: 2.6699 | LR: 3.22e-05 | 3124.7ms/step | 19.2 samples/s +Step 42350/50462 | Loss: 2.6189 | LR: 3.21e-05 | 3127.4ms/step | 19.2 samples/s +Step 42360/50462 | Loss: 2.6298 | LR: 3.21e-05 | 3125.2ms/step | 19.2 samples/s +Step 42370/50462 | Loss: 2.6398 | LR: 3.20e-05 | 3122.6ms/step | 19.2 samples/s +Step 42380/50462 | Loss: 2.6261 | LR: 3.19e-05 | 3119.1ms/step | 19.2 samples/s +Step 42390/50462 | Loss: 2.6554 | LR: 3.18e-05 | 3129.4ms/step | 19.2 samples/s +Step 42400/50462 | Loss: 2.6625 | LR: 3.18e-05 | 3189.9ms/step | 18.8 samples/s +Step 42410/50462 | Loss: 2.6530 | LR: 3.17e-05 | 3116.4ms/step | 19.3 samples/s +Step 42420/50462 | Loss: 2.6481 | LR: 3.16e-05 | 3119.4ms/step | 19.2 samples/s +Step 42430/50462 | Loss: 2.6348 | LR: 3.15e-05 | 3123.3ms/step | 19.2 samples/s +Step 42440/50462 | Loss: 2.6361 | LR: 3.14e-05 | 3124.4ms/step | 19.2 samples/s +Step 42450/50462 | Loss: 2.6534 | LR: 3.14e-05 | 3129.6ms/step | 19.2 samples/s +Step 42460/50462 | Loss: 2.6343 | LR: 3.13e-05 | 3186.6ms/step | 18.8 samples/s +Step 42470/50462 | Loss: 2.6440 | LR: 3.12e-05 | 3113.2ms/step | 19.3 samples/s +Step 42480/50462 | Loss: 2.6028 | LR: 3.11e-05 | 3115.4ms/step | 19.3 samples/s +Step 42490/50462 | Loss: 2.6471 | LR: 3.11e-05 | 3111.4ms/step | 19.3 samples/s +Step 42500/50462 | Loss: 2.6586 | LR: 3.10e-05 | 3117.4ms/step | 19.2 samples/s +Validation at step 42500: Loss = 2.6536, Perplexity = 14.20 +Step 42510/50462 | Loss: 2.6556 | LR: 3.09e-05 | 4778.5ms/step | 12.6 samples/s +Step 42520/50462 | Loss: 2.6379 | LR: 3.08e-05 | 3120.4ms/step | 19.2 samples/s +Step 42530/50462 | Loss: 2.6319 | LR: 3.08e-05 | 3119.7ms/step | 19.2 samples/s +Step 42540/50462 | Loss: 2.6368 | LR: 3.07e-05 | 3120.7ms/step | 19.2 samples/s +Step 42550/50462 | Loss: 2.6644 | LR: 3.06e-05 | 3118.8ms/step | 19.2 samples/s +Step 42560/50462 | Loss: 2.7230 | LR: 3.05e-05 | 3120.6ms/step | 19.2 samples/s +Step 42570/50462 | Loss: 2.6533 | LR: 3.05e-05 | 3121.3ms/step | 19.2 samples/s +Step 42580/50462 | Loss: 2.6452 | LR: 3.04e-05 | 3120.2ms/step | 19.2 samples/s +Step 42590/50462 | Loss: 2.6361 | LR: 3.03e-05 | 3113.7ms/step | 19.3 samples/s +Step 42600/50462 | Loss: 2.6867 | LR: 3.02e-05 | 3119.7ms/step | 19.2 samples/s +Step 42610/50462 | Loss: 2.6361 | LR: 3.02e-05 | 3121.0ms/step | 19.2 samples/s +Step 42620/50462 | Loss: 2.6302 | LR: 3.01e-05 | 3122.2ms/step | 19.2 samples/s +Step 42630/50462 | Loss: 2.6608 | LR: 3.00e-05 | 3126.2ms/step | 19.2 samples/s +Step 42640/50462 | Loss: 2.6884 | LR: 2.99e-05 | 3124.8ms/step | 19.2 samples/s +Step 42650/50462 | Loss: 2.6558 | LR: 2.99e-05 | 3124.5ms/step | 19.2 samples/s +Step 42660/50462 | Loss: 2.6590 | LR: 2.98e-05 | 3120.1ms/step | 19.2 samples/s +Step 42670/50462 | Loss: 2.6311 | LR: 2.97e-05 | 3119.6ms/step | 19.2 samples/s +Step 42680/50462 | Loss: 2.6294 | LR: 2.96e-05 | 3118.4ms/step | 19.2 samples/s +Step 42690/50462 | Loss: 2.6489 | LR: 2.96e-05 | 3118.6ms/step | 19.2 samples/s +Step 42700/50462 | Loss: 2.6556 | LR: 2.95e-05 | 3124.8ms/step | 19.2 samples/s +Step 42710/50462 | Loss: 2.6357 | LR: 2.94e-05 | 3124.0ms/step | 19.2 samples/s +Step 42720/50462 | Loss: 2.6362 | LR: 2.93e-05 | 3174.9ms/step | 18.9 samples/s +Step 42730/50462 | Loss: 2.6372 | LR: 2.93e-05 | 3117.7ms/step | 19.2 samples/s +Step 42740/50462 | Loss: 2.6438 | LR: 2.92e-05 | 3119.0ms/step | 19.2 samples/s +Step 42750/50462 | Loss: 2.6380 | LR: 2.91e-05 | 3121.9ms/step | 19.2 samples/s +Validation at step 42750: Loss = 2.6534, Perplexity = 14.20 +Step 42760/50462 | Loss: 2.6560 | LR: 2.90e-05 | 4773.0ms/step | 12.6 samples/s +Step 42770/50462 | Loss: 2.6430 | LR: 2.90e-05 | 3122.2ms/step | 19.2 samples/s +Step 42780/50462 | Loss: 2.6439 | LR: 2.89e-05 | 3240.5ms/step | 18.5 samples/s +Step 42790/50462 | Loss: 2.6602 | LR: 2.88e-05 | 3117.8ms/step | 19.2 samples/s +Step 42800/50462 | Loss: 2.6607 | LR: 2.87e-05 | 3120.6ms/step | 19.2 samples/s +Step 42810/50462 | Loss: 2.6652 | LR: 2.87e-05 | 3121.9ms/step | 19.2 samples/s +Step 42820/50462 | Loss: 2.6339 | LR: 2.86e-05 | 3125.1ms/step | 19.2 samples/s +Step 42830/50462 | Loss: 2.6290 | LR: 2.85e-05 | 3128.6ms/step | 19.2 samples/s +Step 42840/50462 | Loss: 2.6553 | LR: 2.84e-05 | 3127.9ms/step | 19.2 samples/s +Step 42850/50462 | Loss: 2.6450 | LR: 2.84e-05 | 3122.1ms/step | 19.2 samples/s +Step 42860/50462 | Loss: 2.6747 | LR: 2.83e-05 | 3130.7ms/step | 19.2 samples/s +Step 42870/50462 | Loss: 2.6556 | LR: 2.82e-05 | 3126.1ms/step | 19.2 samples/s +Step 42880/50462 | Loss: 2.6519 | LR: 2.82e-05 | 3125.8ms/step | 19.2 samples/s +Step 42890/50462 | Loss: 2.6357 | LR: 2.81e-05 | 3126.6ms/step | 19.2 samples/s +Step 42900/50462 | Loss: 2.6451 | LR: 2.80e-05 | 3124.3ms/step | 19.2 samples/s +Step 42910/50462 | Loss: 2.6729 | LR: 2.79e-05 | 3128.7ms/step | 19.2 samples/s +Step 42920/50462 | Loss: 2.6322 | LR: 2.79e-05 | 3124.5ms/step | 19.2 samples/s +Step 42930/50462 | Loss: 2.6310 | LR: 2.78e-05 | 3127.0ms/step | 19.2 samples/s +Step 42940/50462 | Loss: 2.6238 | LR: 2.77e-05 | 3123.4ms/step | 19.2 samples/s +Step 42950/50462 | Loss: 2.6717 | LR: 2.76e-05 | 3118.1ms/step | 19.2 samples/s +Step 42960/50462 | Loss: 2.6554 | LR: 2.76e-05 | 3117.1ms/step | 19.2 samples/s +Step 42970/50462 | Loss: 2.6263 | LR: 2.75e-05 | 3119.8ms/step | 19.2 samples/s +Step 42980/50462 | Loss: 2.6405 | LR: 2.74e-05 | 3119.2ms/step | 19.2 samples/s +Step 42990/50462 | Loss: 2.6477 | LR: 2.74e-05 | 3121.5ms/step | 19.2 samples/s +Step 43000/50462 | Loss: 2.6276 | LR: 2.73e-05 | 3123.3ms/step | 19.2 samples/s +Validation at step 43000: Loss = 2.6532, Perplexity = 14.20 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-43000 +Step 43010/50462 | Loss: 2.6672 | LR: 2.72e-05 | 5625.8ms/step | 10.7 samples/s +Step 43020/50462 | Loss: 2.6384 | LR: 2.71e-05 | 3110.7ms/step | 19.3 samples/s +Step 43030/50462 | Loss: 2.6455 | LR: 2.71e-05 | 3116.9ms/step | 19.2 samples/s +Step 43040/50462 | Loss: 2.6295 | LR: 2.70e-05 | 3174.7ms/step | 18.9 samples/s +Step 43050/50462 | Loss: 2.6515 | LR: 2.69e-05 | 3122.2ms/step | 19.2 samples/s +Step 43060/50462 | Loss: 2.6290 | LR: 2.69e-05 | 3126.8ms/step | 19.2 samples/s +Step 43070/50462 | Loss: 2.6458 | LR: 2.68e-05 | 3123.8ms/step | 19.2 samples/s +Step 43080/50462 | Loss: 2.6249 | LR: 2.67e-05 | 3115.0ms/step | 19.3 samples/s +Step 43090/50462 | Loss: 2.6258 | LR: 2.66e-05 | 3119.0ms/step | 19.2 samples/s +Step 43100/50462 | Loss: 2.6190 | LR: 2.66e-05 | 3246.7ms/step | 18.5 samples/s +Step 43110/50462 | Loss: 2.6276 | LR: 2.65e-05 | 3119.3ms/step | 19.2 samples/s +Step 43120/50462 | Loss: 2.6418 | LR: 2.64e-05 | 3120.8ms/step | 19.2 samples/s +Step 43130/50462 | Loss: 2.6540 | LR: 2.64e-05 | 3118.8ms/step | 19.2 samples/s +Step 43140/50462 | Loss: 2.6232 | LR: 2.63e-05 | 3121.0ms/step | 19.2 samples/s +Step 43150/50462 | Loss: 2.6256 | LR: 2.62e-05 | 3122.7ms/step | 19.2 samples/s +Step 43160/50462 | Loss: 2.6627 | LR: 2.61e-05 | 3127.5ms/step | 19.2 samples/s +Step 43170/50462 | Loss: 2.6555 | LR: 2.61e-05 | 3119.3ms/step | 19.2 samples/s +Step 43180/50462 | Loss: 2.6102 | LR: 2.60e-05 | 3120.8ms/step | 19.2 samples/s +Step 43190/50462 | Loss: 2.6634 | LR: 2.59e-05 | 3125.8ms/step | 19.2 samples/s +Step 43200/50462 | Loss: 2.6426 | LR: 2.59e-05 | 3120.1ms/step | 19.2 samples/s +Step 43210/50462 | Loss: 2.6519 | LR: 2.58e-05 | 3123.0ms/step | 19.2 samples/s +Step 43220/50462 | Loss: 2.6552 | LR: 2.57e-05 | 3124.4ms/step | 19.2 samples/s +Step 43230/50462 | Loss: 2.6408 | LR: 2.57e-05 | 3124.8ms/step | 19.2 samples/s +Step 43240/50462 | Loss: 2.6663 | LR: 2.56e-05 | 3122.6ms/step | 19.2 samples/s +Step 43250/50462 | Loss: 2.6169 | LR: 2.55e-05 | 3122.2ms/step | 19.2 samples/s +Validation at step 43250: Loss = 2.6530, Perplexity = 14.20 +Step 43260/50462 | Loss: 2.6163 | LR: 2.54e-05 | 4773.7ms/step | 12.6 samples/s +Step 43270/50462 | Loss: 2.6259 | LR: 2.54e-05 | 3122.7ms/step | 19.2 samples/s +Step 43280/50462 | Loss: 2.6470 | LR: 2.53e-05 | 3123.1ms/step | 19.2 samples/s +Step 43290/50462 | Loss: 2.6260 | LR: 2.52e-05 | 3124.9ms/step | 19.2 samples/s +Step 43300/50462 | Loss: 2.6699 | LR: 2.52e-05 | 3118.3ms/step | 19.2 samples/s +Step 43310/50462 | Loss: 2.6348 | LR: 2.51e-05 | 3120.2ms/step | 19.2 samples/s +Step 43320/50462 | Loss: 2.6403 | LR: 2.50e-05 | 3119.1ms/step | 19.2 samples/s +Step 43330/50462 | Loss: 2.6286 | LR: 2.50e-05 | 3116.3ms/step | 19.3 samples/s +Step 43340/50462 | Loss: 2.6372 | LR: 2.49e-05 | 3118.0ms/step | 19.2 samples/s +Step 43350/50462 | Loss: 2.6228 | LR: 2.48e-05 | 3120.2ms/step | 19.2 samples/s +Step 43360/50462 | Loss: 2.6562 | LR: 2.48e-05 | 3177.8ms/step | 18.9 samples/s +Step 43370/50462 | Loss: 2.6469 | LR: 2.47e-05 | 3124.8ms/step | 19.2 samples/s +Step 43380/50462 | Loss: 2.6286 | LR: 2.46e-05 | 3125.2ms/step | 19.2 samples/s +Step 43390/50462 | Loss: 2.6575 | LR: 2.46e-05 | 3118.7ms/step | 19.2 samples/s +Step 43400/50462 | Loss: 2.6369 | LR: 2.45e-05 | 3130.0ms/step | 19.2 samples/s +Step 43410/50462 | Loss: 2.6576 | LR: 2.44e-05 | 3123.4ms/step | 19.2 samples/s +Step 43420/50462 | Loss: 2.6389 | LR: 2.44e-05 | 3186.9ms/step | 18.8 samples/s +Step 43430/50462 | Loss: 2.6412 | LR: 2.43e-05 | 3124.0ms/step | 19.2 samples/s +Step 43440/50462 | Loss: 2.6401 | LR: 2.42e-05 | 3128.7ms/step | 19.2 samples/s +Step 43450/50462 | Loss: 2.6636 | LR: 2.41e-05 | 3125.3ms/step | 19.2 samples/s +Step 43460/50462 | Loss: 2.6508 | LR: 2.41e-05 | 3123.0ms/step | 19.2 samples/s +Step 43470/50462 | Loss: 2.6629 | LR: 2.40e-05 | 3129.5ms/step | 19.2 samples/s +Step 43480/50462 | Loss: 2.6653 | LR: 2.39e-05 | 3131.8ms/step | 19.2 samples/s +Step 43490/50462 | Loss: 2.6339 | LR: 2.39e-05 | 3123.5ms/step | 19.2 samples/s +Step 43500/50462 | Loss: 2.6511 | LR: 2.38e-05 | 3126.3ms/step | 19.2 samples/s +Validation at step 43500: Loss = 2.6529, Perplexity = 14.20 +Step 43510/50462 | Loss: 2.6538 | LR: 2.37e-05 | 4782.0ms/step | 12.5 samples/s +Step 43520/50462 | Loss: 2.6199 | LR: 2.37e-05 | 3123.8ms/step | 19.2 samples/s +Step 43530/50462 | Loss: 2.6418 | LR: 2.36e-05 | 3125.9ms/step | 19.2 samples/s +Step 43540/50462 | Loss: 2.6346 | LR: 2.35e-05 | 3118.6ms/step | 19.2 samples/s +Step 43550/50462 | Loss: 2.6438 | LR: 2.35e-05 | 3120.2ms/step | 19.2 samples/s +Step 43560/50462 | Loss: 2.6226 | LR: 2.34e-05 | 3112.5ms/step | 19.3 samples/s +Step 43570/50462 | Loss: 2.6433 | LR: 2.33e-05 | 3115.9ms/step | 19.3 samples/s +Step 43580/50462 | Loss: 2.6707 | LR: 2.33e-05 | 3121.3ms/step | 19.2 samples/s +Step 43590/50462 | Loss: 2.6320 | LR: 2.32e-05 | 3127.0ms/step | 19.2 samples/s +Step 43600/50462 | Loss: 2.6650 | LR: 2.31e-05 | 3117.4ms/step | 19.2 samples/s +Step 43610/50462 | Loss: 2.6471 | LR: 2.31e-05 | 3111.6ms/step | 19.3 samples/s +Step 43620/50462 | Loss: 2.6289 | LR: 2.30e-05 | 3121.6ms/step | 19.2 samples/s +Step 43630/50462 | Loss: 2.6680 | LR: 2.29e-05 | 3118.1ms/step | 19.2 samples/s +Step 43640/50462 | Loss: 2.6310 | LR: 2.29e-05 | 3119.2ms/step | 19.2 samples/s +Step 43650/50462 | Loss: 2.6486 | LR: 2.28e-05 | 3120.6ms/step | 19.2 samples/s +Step 43660/50462 | Loss: 2.6621 | LR: 2.27e-05 | 3124.0ms/step | 19.2 samples/s +Step 43670/50462 | Loss: 2.6513 | LR: 2.27e-05 | 3128.9ms/step | 19.2 samples/s +Step 43680/50462 | Loss: 2.6565 | LR: 2.26e-05 | 3187.4ms/step | 18.8 samples/s +Step 43690/50462 | Loss: 2.6491 | LR: 2.25e-05 | 3123.4ms/step | 19.2 samples/s +Step 43700/50462 | Loss: 2.6649 | LR: 2.25e-05 | 3124.6ms/step | 19.2 samples/s +Step 43710/50462 | Loss: 2.6682 | LR: 2.24e-05 | 3128.2ms/step | 19.2 samples/s +Step 43720/50462 | Loss: 2.6275 | LR: 2.24e-05 | 3126.5ms/step | 19.2 samples/s +Step 43730/50462 | Loss: 2.6170 | LR: 2.23e-05 | 3128.5ms/step | 19.2 samples/s +Step 43740/50462 | Loss: 2.6403 | LR: 2.22e-05 | 3230.5ms/step | 18.6 samples/s +Step 43750/50462 | Loss: 2.6419 | LR: 2.22e-05 | 3127.3ms/step | 19.2 samples/s +Validation at step 43750: Loss = 2.6528, Perplexity = 14.19 +Step 43760/50462 | Loss: 2.6455 | LR: 2.21e-05 | 4796.0ms/step | 12.5 samples/s +Step 43770/50462 | Loss: 2.6334 | LR: 2.20e-05 | 3126.1ms/step | 19.2 samples/s +Step 43780/50462 | Loss: 2.6376 | LR: 2.20e-05 | 3126.3ms/step | 19.2 samples/s +Step 43790/50462 | Loss: 2.6444 | LR: 2.19e-05 | 3127.2ms/step | 19.2 samples/s +Step 43800/50462 | Loss: 2.6608 | LR: 2.18e-05 | 3128.1ms/step | 19.2 samples/s +Step 43810/50462 | Loss: 2.6069 | LR: 2.18e-05 | 3120.6ms/step | 19.2 samples/s +Step 43820/50462 | Loss: 2.6109 | LR: 2.17e-05 | 3125.0ms/step | 19.2 samples/s +Step 43830/50462 | Loss: 2.6521 | LR: 2.16e-05 | 3131.5ms/step | 19.2 samples/s +Step 43840/50462 | Loss: 2.6712 | LR: 2.16e-05 | 3118.5ms/step | 19.2 samples/s +Step 43850/50462 | Loss: 2.6629 | LR: 2.15e-05 | 3118.3ms/step | 19.2 samples/s +Step 43860/50462 | Loss: 2.6467 | LR: 2.14e-05 | 3121.2ms/step | 19.2 samples/s +Step 43870/50462 | Loss: 2.6314 | LR: 2.14e-05 | 3123.1ms/step | 19.2 samples/s +Step 43880/50462 | Loss: 2.6549 | LR: 2.13e-05 | 3130.1ms/step | 19.2 samples/s +Step 43890/50462 | Loss: 2.6471 | LR: 2.13e-05 | 3124.9ms/step | 19.2 samples/s +Step 43900/50462 | Loss: 2.7011 | LR: 2.12e-05 | 3116.7ms/step | 19.3 samples/s +Step 43910/50462 | Loss: 2.6651 | LR: 2.11e-05 | 3118.4ms/step | 19.2 samples/s +Step 43920/50462 | Loss: 2.6406 | LR: 2.11e-05 | 3127.1ms/step | 19.2 samples/s +Step 43930/50462 | Loss: 2.6687 | LR: 2.10e-05 | 3124.3ms/step | 19.2 samples/s +Step 43940/50462 | Loss: 2.6416 | LR: 2.09e-05 | 3129.3ms/step | 19.2 samples/s +Step 43950/50462 | Loss: 2.6566 | LR: 2.09e-05 | 3126.6ms/step | 19.2 samples/s +Step 43960/50462 | Loss: 2.6367 | LR: 2.08e-05 | 3128.0ms/step | 19.2 samples/s +Step 43970/50462 | Loss: 2.6326 | LR: 2.07e-05 | 3125.1ms/step | 19.2 samples/s +Step 43980/50462 | Loss: 2.6247 | LR: 2.07e-05 | 3129.2ms/step | 19.2 samples/s +Step 43990/50462 | Loss: 2.6598 | LR: 2.06e-05 | 3122.9ms/step | 19.2 samples/s +Step 44000/50462 | Loss: 2.6644 | LR: 2.06e-05 | 3177.1ms/step | 18.9 samples/s +Validation at step 44000: Loss = 2.6527, Perplexity = 14.19 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-44000 +Step 44010/50462 | Loss: 2.6519 | LR: 2.05e-05 | 5664.1ms/step | 10.6 samples/s +Step 44020/50462 | Loss: 2.6434 | LR: 2.04e-05 | 3116.2ms/step | 19.3 samples/s +Step 44030/50462 | Loss: 2.6382 | LR: 2.04e-05 | 3117.4ms/step | 19.2 samples/s +Step 44040/50462 | Loss: 2.6327 | LR: 2.03e-05 | 3124.7ms/step | 19.2 samples/s +Step 44050/50462 | Loss: 2.6662 | LR: 2.02e-05 | 3119.5ms/step | 19.2 samples/s +Step 44060/50462 | Loss: 2.6640 | LR: 2.02e-05 | 3181.3ms/step | 18.9 samples/s +Step 44070/50462 | Loss: 2.6664 | LR: 2.01e-05 | 3180.0ms/step | 18.9 samples/s +Step 44080/50462 | Loss: 2.6473 | LR: 2.01e-05 | 3119.2ms/step | 19.2 samples/s +Step 44090/50462 | Loss: 2.6141 | LR: 2.00e-05 | 3117.4ms/step | 19.2 samples/s +Step 44100/50462 | Loss: 2.6346 | LR: 1.99e-05 | 3120.1ms/step | 19.2 samples/s +Step 44110/50462 | Loss: 2.6359 | LR: 1.99e-05 | 3123.8ms/step | 19.2 samples/s +Step 44120/50462 | Loss: 2.6536 | LR: 1.98e-05 | 3109.6ms/step | 19.3 samples/s +Step 44130/50462 | Loss: 2.6589 | LR: 1.97e-05 | 3118.6ms/step | 19.2 samples/s +Step 44140/50462 | Loss: 2.6311 | LR: 1.97e-05 | 3119.4ms/step | 19.2 samples/s +Step 44150/50462 | Loss: 2.6248 | LR: 1.96e-05 | 3120.3ms/step | 19.2 samples/s +Step 44160/50462 | Loss: 2.6519 | LR: 1.96e-05 | 3120.6ms/step | 19.2 samples/s +Step 44170/50462 | Loss: 2.6592 | LR: 1.95e-05 | 3118.6ms/step | 19.2 samples/s +Step 44180/50462 | Loss: 2.6265 | LR: 1.94e-05 | 3117.8ms/step | 19.2 samples/s +Step 44190/50462 | Loss: 2.6306 | LR: 1.94e-05 | 3119.9ms/step | 19.2 samples/s +Step 44200/50462 | Loss: 2.6529 | LR: 1.93e-05 | 3125.0ms/step | 19.2 samples/s +Step 44210/50462 | Loss: 2.6424 | LR: 1.93e-05 | 3114.0ms/step | 19.3 samples/s +Step 44220/50462 | Loss: 2.6446 | LR: 1.92e-05 | 3115.3ms/step | 19.3 samples/s +Step 44230/50462 | Loss: 2.6575 | LR: 1.91e-05 | 3111.2ms/step | 19.3 samples/s +Step 44240/50462 | Loss: 2.6467 | LR: 1.91e-05 | 3114.8ms/step | 19.3 samples/s +Step 44250/50462 | Loss: 2.6583 | LR: 1.90e-05 | 3120.8ms/step | 19.2 samples/s +Validation at step 44250: Loss = 2.6527, Perplexity = 14.19 +Step 44260/50462 | Loss: 2.6532 | LR: 1.90e-05 | 4770.5ms/step | 12.6 samples/s +Step 44270/50462 | Loss: 2.6262 | LR: 1.89e-05 | 3119.7ms/step | 19.2 samples/s +Step 44280/50462 | Loss: 2.6449 | LR: 1.88e-05 | 3124.3ms/step | 19.2 samples/s +Step 44290/50462 | Loss: 2.6450 | LR: 1.88e-05 | 3123.9ms/step | 19.2 samples/s +Step 44300/50462 | Loss: 2.5956 | LR: 1.87e-05 | 3125.3ms/step | 19.2 samples/s +Step 44310/50462 | Loss: 2.6221 | LR: 1.87e-05 | 3122.9ms/step | 19.2 samples/s +Step 44320/50462 | Loss: 2.6575 | LR: 1.86e-05 | 3179.7ms/step | 18.9 samples/s +Step 44330/50462 | Loss: 2.6486 | LR: 1.85e-05 | 3134.6ms/step | 19.1 samples/s +Step 44340/50462 | Loss: 2.6429 | LR: 1.85e-05 | 3126.6ms/step | 19.2 samples/s +Step 44350/50462 | Loss: 2.6492 | LR: 1.84e-05 | 3127.4ms/step | 19.2 samples/s +Step 44360/50462 | Loss: 2.6212 | LR: 1.84e-05 | 3125.8ms/step | 19.2 samples/s +Step 44370/50462 | Loss: 2.6468 | LR: 1.83e-05 | 3133.7ms/step | 19.1 samples/s +Step 44380/50462 | Loss: 2.6375 | LR: 1.82e-05 | 3189.3ms/step | 18.8 samples/s +Step 44390/50462 | Loss: 2.6574 | LR: 1.82e-05 | 3179.5ms/step | 18.9 samples/s +Step 44400/50462 | Loss: 2.6267 | LR: 1.81e-05 | 3126.4ms/step | 19.2 samples/s +Step 44410/50462 | Loss: 2.6343 | LR: 1.81e-05 | 3129.2ms/step | 19.2 samples/s +Step 44420/50462 | Loss: 2.6435 | LR: 1.80e-05 | 3130.9ms/step | 19.2 samples/s +Step 44430/50462 | Loss: 2.6519 | LR: 1.79e-05 | 3129.9ms/step | 19.2 samples/s +Step 44440/50462 | Loss: 2.6403 | LR: 1.79e-05 | 3129.3ms/step | 19.2 samples/s +Step 44450/50462 | Loss: 2.6274 | LR: 1.78e-05 | 3124.2ms/step | 19.2 samples/s +Step 44460/50462 | Loss: 2.6445 | LR: 1.78e-05 | 3120.1ms/step | 19.2 samples/s +Step 44470/50462 | Loss: 2.6566 | LR: 1.77e-05 | 3119.6ms/step | 19.2 samples/s +Step 44480/50462 | Loss: 2.6734 | LR: 1.77e-05 | 3121.4ms/step | 19.2 samples/s +Step 44490/50462 | Loss: 2.6683 | LR: 1.76e-05 | 3122.9ms/step | 19.2 samples/s +Step 44500/50462 | Loss: 2.6299 | LR: 1.75e-05 | 3120.7ms/step | 19.2 samples/s +Validation at step 44500: Loss = 2.6526, Perplexity = 14.19 +Step 44510/50462 | Loss: 2.6174 | LR: 1.75e-05 | 4782.3ms/step | 12.5 samples/s +Step 44520/50462 | Loss: 2.6185 | LR: 1.74e-05 | 3126.6ms/step | 19.2 samples/s +Step 44530/50462 | Loss: 2.6422 | LR: 1.74e-05 | 3126.6ms/step | 19.2 samples/s +Step 44540/50462 | Loss: 2.6312 | LR: 1.73e-05 | 3125.4ms/step | 19.2 samples/s +Step 44550/50462 | Loss: 2.6484 | LR: 1.72e-05 | 3115.5ms/step | 19.3 samples/s +Step 44560/50462 | Loss: 2.6500 | LR: 1.72e-05 | 3119.9ms/step | 19.2 samples/s +Step 44570/50462 | Loss: 2.6395 | LR: 1.71e-05 | 3123.4ms/step | 19.2 samples/s +Step 44580/50462 | Loss: 2.6176 | LR: 1.71e-05 | 3127.1ms/step | 19.2 samples/s +Step 44590/50462 | Loss: 2.6349 | LR: 1.70e-05 | 3125.2ms/step | 19.2 samples/s +Step 44600/50462 | Loss: 2.6571 | LR: 1.70e-05 | 3120.4ms/step | 19.2 samples/s +Step 44610/50462 | Loss: 2.6418 | LR: 1.69e-05 | 3124.6ms/step | 19.2 samples/s +Step 44620/50462 | Loss: 2.6567 | LR: 1.68e-05 | 3127.6ms/step | 19.2 samples/s +Step 44630/50462 | Loss: 2.6412 | LR: 1.68e-05 | 3121.5ms/step | 19.2 samples/s +Step 44640/50462 | Loss: 2.6331 | LR: 1.67e-05 | 3112.8ms/step | 19.3 samples/s +Step 44650/50462 | Loss: 2.6245 | LR: 1.67e-05 | 3174.1ms/step | 18.9 samples/s +Step 44660/50462 | Loss: 2.6578 | LR: 1.66e-05 | 3123.7ms/step | 19.2 samples/s +Step 44670/50462 | Loss: 2.6667 | LR: 1.66e-05 | 3128.1ms/step | 19.2 samples/s +Step 44680/50462 | Loss: 2.6539 | LR: 1.65e-05 | 3118.5ms/step | 19.2 samples/s +Step 44690/50462 | Loss: 2.6505 | LR: 1.64e-05 | 3119.2ms/step | 19.2 samples/s +Step 44700/50462 | Loss: 2.6579 | LR: 1.64e-05 | 3179.8ms/step | 18.9 samples/s +Step 44710/50462 | Loss: 2.6407 | LR: 1.63e-05 | 3179.5ms/step | 18.9 samples/s +Step 44720/50462 | Loss: 2.6393 | LR: 1.63e-05 | 3126.8ms/step | 19.2 samples/s +Step 44730/50462 | Loss: 2.6284 | LR: 1.62e-05 | 3121.6ms/step | 19.2 samples/s +Step 44740/50462 | Loss: 2.6266 | LR: 1.62e-05 | 3123.8ms/step | 19.2 samples/s +Step 44750/50462 | Loss: 2.6495 | LR: 1.61e-05 | 3126.3ms/step | 19.2 samples/s +Validation at step 44750: Loss = 2.6525, Perplexity = 14.19 +Step 44760/50462 | Loss: 2.6655 | LR: 1.61e-05 | 4794.8ms/step | 12.5 samples/s +Step 44770/50462 | Loss: 2.6174 | LR: 1.60e-05 | 3128.0ms/step | 19.2 samples/s +Step 44780/50462 | Loss: 2.6460 | LR: 1.59e-05 | 3126.7ms/step | 19.2 samples/s +Step 44790/50462 | Loss: 2.6323 | LR: 1.59e-05 | 3118.3ms/step | 19.2 samples/s +Step 44800/50462 | Loss: 2.6329 | LR: 1.58e-05 | 3120.8ms/step | 19.2 samples/s +Step 44810/50462 | Loss: 2.6607 | LR: 1.58e-05 | 3119.4ms/step | 19.2 samples/s +Step 44820/50462 | Loss: 2.6507 | LR: 1.57e-05 | 3119.4ms/step | 19.2 samples/s +Step 44830/50462 | Loss: 2.6731 | LR: 1.57e-05 | 3126.0ms/step | 19.2 samples/s +Step 44840/50462 | Loss: 2.6265 | LR: 1.56e-05 | 3120.3ms/step | 19.2 samples/s +Step 44850/50462 | Loss: 2.6352 | LR: 1.56e-05 | 3125.9ms/step | 19.2 samples/s +Step 44860/50462 | Loss: 2.6321 | LR: 1.55e-05 | 3126.2ms/step | 19.2 samples/s +Step 44870/50462 | Loss: 2.6226 | LR: 1.54e-05 | 3119.1ms/step | 19.2 samples/s +Step 44880/50462 | Loss: 2.6433 | LR: 1.54e-05 | 3127.0ms/step | 19.2 samples/s +Step 44890/50462 | Loss: 2.6033 | LR: 1.53e-05 | 3129.0ms/step | 19.2 samples/s +Step 44900/50462 | Loss: 2.6031 | LR: 1.53e-05 | 3122.8ms/step | 19.2 samples/s +Step 44910/50462 | Loss: 2.6266 | LR: 1.52e-05 | 3118.0ms/step | 19.2 samples/s +Step 44920/50462 | Loss: 2.6308 | LR: 1.52e-05 | 3123.0ms/step | 19.2 samples/s +Step 44930/50462 | Loss: 2.6344 | LR: 1.51e-05 | 3121.6ms/step | 19.2 samples/s +Step 44940/50462 | Loss: 2.6354 | LR: 1.51e-05 | 3123.6ms/step | 19.2 samples/s +Step 44950/50462 | Loss: 2.6002 | LR: 1.50e-05 | 3121.2ms/step | 19.2 samples/s +Step 44960/50462 | Loss: 2.6741 | LR: 1.50e-05 | 3178.2ms/step | 18.9 samples/s +Step 44970/50462 | Loss: 2.6249 | LR: 1.49e-05 | 3125.8ms/step | 19.2 samples/s +Step 44980/50462 | Loss: 2.6391 | LR: 1.49e-05 | 3130.0ms/step | 19.2 samples/s +Step 44990/50462 | Loss: 2.6632 | LR: 1.48e-05 | 3125.3ms/step | 19.2 samples/s +Step 45000/50462 | Loss: 2.6583 | LR: 1.47e-05 | 3119.1ms/step | 19.2 samples/s +Validation at step 45000: Loss = 2.6525, Perplexity = 14.19 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-45000 +Step 45010/50462 | Loss: 2.6467 | LR: 1.47e-05 | 5661.0ms/step | 10.6 samples/s +Step 45020/50462 | Loss: 2.6336 | LR: 1.46e-05 | 3183.0ms/step | 18.9 samples/s +Step 45030/50462 | Loss: 2.6388 | LR: 1.46e-05 | 3189.5ms/step | 18.8 samples/s +Step 45040/50462 | Loss: 2.6527 | LR: 1.45e-05 | 3121.3ms/step | 19.2 samples/s +Step 45050/50462 | Loss: 2.6243 | LR: 1.45e-05 | 3121.3ms/step | 19.2 samples/s +Step 45060/50462 | Loss: 2.6482 | LR: 1.44e-05 | 3117.9ms/step | 19.2 samples/s +Step 45070/50462 | Loss: 2.6426 | LR: 1.44e-05 | 3125.8ms/step | 19.2 samples/s +Step 45080/50462 | Loss: 2.6334 | LR: 1.43e-05 | 3124.2ms/step | 19.2 samples/s +Step 45090/50462 | Loss: 2.6339 | LR: 1.43e-05 | 3125.9ms/step | 19.2 samples/s +Step 45100/50462 | Loss: 2.6682 | LR: 1.42e-05 | 3126.4ms/step | 19.2 samples/s +Step 45110/50462 | Loss: 2.6683 | LR: 1.42e-05 | 3125.6ms/step | 19.2 samples/s +Step 45120/50462 | Loss: 2.6508 | LR: 1.41e-05 | 3129.1ms/step | 19.2 samples/s +Step 45130/50462 | Loss: 2.6573 | LR: 1.41e-05 | 3125.2ms/step | 19.2 samples/s +Step 45140/50462 | Loss: 2.6273 | LR: 1.40e-05 | 3130.0ms/step | 19.2 samples/s +Step 45150/50462 | Loss: 2.6521 | LR: 1.40e-05 | 3121.6ms/step | 19.2 samples/s +Step 45160/50462 | Loss: 2.6577 | LR: 1.39e-05 | 3125.7ms/step | 19.2 samples/s +Step 45170/50462 | Loss: 2.6237 | LR: 1.39e-05 | 3117.7ms/step | 19.2 samples/s +Step 45180/50462 | Loss: 2.6498 | LR: 1.38e-05 | 3118.4ms/step | 19.2 samples/s +Step 45190/50462 | Loss: 2.6356 | LR: 1.37e-05 | 3114.7ms/step | 19.3 samples/s +Step 45200/50462 | Loss: 2.6613 | LR: 1.37e-05 | 3122.4ms/step | 19.2 samples/s +Step 45210/50462 | Loss: 2.6433 | LR: 1.36e-05 | 3124.6ms/step | 19.2 samples/s +Step 45220/50462 | Loss: 2.6317 | LR: 1.36e-05 | 3126.8ms/step | 19.2 samples/s +Step 45230/50462 | Loss: 2.6234 | LR: 1.35e-05 | 3123.8ms/step | 19.2 samples/s +Step 45240/50462 | Loss: 2.6574 | LR: 1.35e-05 | 3123.8ms/step | 19.2 samples/s +Step 45250/50462 | Loss: 2.6383 | LR: 1.34e-05 | 3125.4ms/step | 19.2 samples/s +Validation at step 45250: Loss = 2.6524, Perplexity = 14.19 +Step 45260/50462 | Loss: 2.6510 | LR: 1.34e-05 | 4783.7ms/step | 12.5 samples/s +Step 45270/50462 | Loss: 2.6575 | LR: 1.33e-05 | 3123.7ms/step | 19.2 samples/s +Step 45280/50462 | Loss: 2.6163 | LR: 1.33e-05 | 3174.6ms/step | 18.9 samples/s +Step 45290/50462 | Loss: 2.6414 | LR: 1.32e-05 | 3129.1ms/step | 19.2 samples/s +Step 45300/50462 | Loss: 2.6563 | LR: 1.32e-05 | 3122.0ms/step | 19.2 samples/s +Step 45310/50462 | Loss: 2.6317 | LR: 1.31e-05 | 3125.6ms/step | 19.2 samples/s +Step 45320/50462 | Loss: 2.6364 | LR: 1.31e-05 | 3126.2ms/step | 19.2 samples/s +Step 45330/50462 | Loss: 2.6513 | LR: 1.30e-05 | 3121.0ms/step | 19.2 samples/s +Step 45340/50462 | Loss: 2.6415 | LR: 1.30e-05 | 3180.1ms/step | 18.9 samples/s +Step 45350/50462 | Loss: 2.6393 | LR: 1.29e-05 | 3187.9ms/step | 18.8 samples/s +Step 45360/50462 | Loss: 2.6647 | LR: 1.29e-05 | 3125.1ms/step | 19.2 samples/s +Step 45370/50462 | Loss: 2.6362 | LR: 1.28e-05 | 3124.1ms/step | 19.2 samples/s +Step 45380/50462 | Loss: 2.6293 | LR: 1.28e-05 | 3118.5ms/step | 19.2 samples/s +Step 45390/50462 | Loss: 2.6398 | LR: 1.27e-05 | 3113.1ms/step | 19.3 samples/s +Step 45400/50462 | Loss: 2.6205 | LR: 1.27e-05 | 3126.1ms/step | 19.2 samples/s +Step 45410/50462 | Loss: 2.6635 | LR: 1.26e-05 | 3122.6ms/step | 19.2 samples/s +Step 45420/50462 | Loss: 2.6353 | LR: 1.26e-05 | 3118.0ms/step | 19.2 samples/s +Step 45430/50462 | Loss: 2.6098 | LR: 1.25e-05 | 3127.1ms/step | 19.2 samples/s +Step 45440/50462 | Loss: 2.6527 | LR: 1.25e-05 | 3125.1ms/step | 19.2 samples/s +Step 45450/50462 | Loss: 2.6444 | LR: 1.24e-05 | 3118.1ms/step | 19.2 samples/s +Step 45460/50462 | Loss: 2.6295 | LR: 1.24e-05 | 3116.5ms/step | 19.3 samples/s +Step 45470/50462 | Loss: 2.6431 | LR: 1.23e-05 | 3116.4ms/step | 19.3 samples/s +Step 45480/50462 | Loss: 2.6320 | LR: 1.23e-05 | 3116.8ms/step | 19.3 samples/s +Step 45490/50462 | Loss: 2.6530 | LR: 1.22e-05 | 3127.2ms/step | 19.2 samples/s +Step 45500/50462 | Loss: 2.6543 | LR: 1.22e-05 | 3123.4ms/step | 19.2 samples/s +Validation at step 45500: Loss = 2.6524, Perplexity = 14.19 +Step 45510/50462 | Loss: 2.6288 | LR: 1.21e-05 | 4773.0ms/step | 12.6 samples/s +Step 45520/50462 | Loss: 2.6721 | LR: 1.21e-05 | 3120.2ms/step | 19.2 samples/s +Step 45530/50462 | Loss: 2.6207 | LR: 1.20e-05 | 3116.2ms/step | 19.3 samples/s +Step 45540/50462 | Loss: 2.6691 | LR: 1.20e-05 | 3119.9ms/step | 19.2 samples/s +Step 45550/50462 | Loss: 2.6300 | LR: 1.19e-05 | 3123.3ms/step | 19.2 samples/s +Step 45560/50462 | Loss: 2.6519 | LR: 1.19e-05 | 3124.5ms/step | 19.2 samples/s +Step 45570/50462 | Loss: 2.6564 | LR: 1.19e-05 | 3127.5ms/step | 19.2 samples/s +Step 45580/50462 | Loss: 2.6345 | LR: 1.18e-05 | 3124.3ms/step | 19.2 samples/s +Step 45590/50462 | Loss: 2.6208 | LR: 1.18e-05 | 3123.9ms/step | 19.2 samples/s +Step 45600/50462 | Loss: 2.6310 | LR: 1.17e-05 | 3180.4ms/step | 18.9 samples/s +Step 45610/50462 | Loss: 2.6121 | LR: 1.17e-05 | 3130.3ms/step | 19.2 samples/s +Step 45620/50462 | Loss: 2.6426 | LR: 1.16e-05 | 3113.3ms/step | 19.3 samples/s +Step 45630/50462 | Loss: 2.6592 | LR: 1.16e-05 | 3120.6ms/step | 19.2 samples/s +Step 45640/50462 | Loss: 2.6359 | LR: 1.15e-05 | 3122.6ms/step | 19.2 samples/s +Step 45650/50462 | Loss: 2.6619 | LR: 1.15e-05 | 3125.6ms/step | 19.2 samples/s +Step 45660/50462 | Loss: 2.6447 | LR: 1.14e-05 | 3180.6ms/step | 18.9 samples/s +Step 45670/50462 | Loss: 2.6137 | LR: 1.14e-05 | 3175.0ms/step | 18.9 samples/s +Step 45680/50462 | Loss: 2.6313 | LR: 1.13e-05 | 3116.0ms/step | 19.3 samples/s +Step 45690/50462 | Loss: 2.6519 | LR: 1.13e-05 | 3116.4ms/step | 19.3 samples/s +Step 45700/50462 | Loss: 2.6495 | LR: 1.12e-05 | 3119.1ms/step | 19.2 samples/s +Step 45710/50462 | Loss: 2.6471 | LR: 1.12e-05 | 3124.7ms/step | 19.2 samples/s +Step 45720/50462 | Loss: 2.6389 | LR: 1.11e-05 | 3124.9ms/step | 19.2 samples/s +Step 45730/50462 | Loss: 2.6195 | LR: 1.11e-05 | 3126.3ms/step | 19.2 samples/s +Step 45740/50462 | Loss: 2.6357 | LR: 1.10e-05 | 3112.3ms/step | 19.3 samples/s +Step 45750/50462 | Loss: 2.6541 | LR: 1.10e-05 | 3116.4ms/step | 19.3 samples/s +Validation at step 45750: Loss = 2.6524, Perplexity = 14.19 +Step 45760/50462 | Loss: 2.6054 | LR: 1.10e-05 | 4785.8ms/step | 12.5 samples/s +Step 45770/50462 | Loss: 2.6391 | LR: 1.09e-05 | 3122.6ms/step | 19.2 samples/s +Step 45780/50462 | Loss: 2.6321 | LR: 1.09e-05 | 3120.0ms/step | 19.2 samples/s +Step 45790/50462 | Loss: 2.6691 | LR: 1.08e-05 | 3117.1ms/step | 19.2 samples/s +Step 45800/50462 | Loss: 2.5859 | LR: 1.08e-05 | 3120.3ms/step | 19.2 samples/s +Step 45810/50462 | Loss: 2.6427 | LR: 1.07e-05 | 3118.1ms/step | 19.2 samples/s +Step 45820/50462 | Loss: 2.6408 | LR: 1.07e-05 | 3120.2ms/step | 19.2 samples/s +Step 45830/50462 | Loss: 2.6485 | LR: 1.06e-05 | 3122.3ms/step | 19.2 samples/s +Step 45840/50462 | Loss: 2.6618 | LR: 1.06e-05 | 3126.4ms/step | 19.2 samples/s +Step 45850/50462 | Loss: 2.6432 | LR: 1.05e-05 | 3120.4ms/step | 19.2 samples/s +Step 45860/50462 | Loss: 2.6647 | LR: 1.05e-05 | 3122.2ms/step | 19.2 samples/s +Step 45870/50462 | Loss: 2.6492 | LR: 1.05e-05 | 3123.9ms/step | 19.2 samples/s +Step 45880/50462 | Loss: 2.6528 | LR: 1.04e-05 | 3121.2ms/step | 19.2 samples/s +Step 45890/50462 | Loss: 2.6163 | LR: 1.04e-05 | 3121.0ms/step | 19.2 samples/s +Step 45900/50462 | Loss: 2.6391 | LR: 1.03e-05 | 3121.9ms/step | 19.2 samples/s +Step 45910/50462 | Loss: 2.6680 | LR: 1.03e-05 | 3116.5ms/step | 19.3 samples/s +Step 45920/50462 | Loss: 2.6210 | LR: 1.02e-05 | 3115.5ms/step | 19.3 samples/s +Step 45930/50462 | Loss: 2.6337 | LR: 1.02e-05 | 3177.4ms/step | 18.9 samples/s +Step 45940/50462 | Loss: 2.6526 | LR: 1.01e-05 | 3116.7ms/step | 19.3 samples/s +Step 45950/50462 | Loss: 2.6380 | LR: 1.01e-05 | 3118.8ms/step | 19.2 samples/s +Step 45960/50462 | Loss: 2.6236 | LR: 1.01e-05 | 3127.6ms/step | 19.2 samples/s +Step 45970/50462 | Loss: 2.6295 | LR: 1.00e-05 | 3117.6ms/step | 19.2 samples/s +Step 45980/50462 | Loss: 2.6749 | LR: 9.96e-06 | 3114.3ms/step | 19.3 samples/s +Step 45990/50462 | Loss: 2.6264 | LR: 9.92e-06 | 3230.6ms/step | 18.6 samples/s +Step 46000/50462 | Loss: 2.6434 | LR: 9.87e-06 | 3116.9ms/step | 19.2 samples/s +Validation at step 46000: Loss = 2.6524, Perplexity = 14.19 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-46000 +Step 46010/50462 | Loss: 2.6455 | LR: 9.83e-06 | 5650.3ms/step | 10.6 samples/s +Step 46020/50462 | Loss: 2.6332 | LR: 9.79e-06 | 3125.7ms/step | 19.2 samples/s +Step 46030/50462 | Loss: 2.6555 | LR: 9.74e-06 | 3122.9ms/step | 19.2 samples/s +Step 46040/50462 | Loss: 2.6554 | LR: 9.70e-06 | 3123.1ms/step | 19.2 samples/s +Step 46050/50462 | Loss: 2.6350 | LR: 9.65e-06 | 3118.4ms/step | 19.2 samples/s +Step 46060/50462 | Loss: 2.6311 | LR: 9.61e-06 | 3117.5ms/step | 19.2 samples/s +Step 46070/50462 | Loss: 2.6547 | LR: 9.57e-06 | 3126.8ms/step | 19.2 samples/s +Step 46080/50462 | Loss: 2.6444 | LR: 9.52e-06 | 3120.1ms/step | 19.2 samples/s +Step 46090/50462 | Loss: 2.6151 | LR: 9.48e-06 | 3120.2ms/step | 19.2 samples/s +Step 46100/50462 | Loss: 2.6452 | LR: 9.44e-06 | 3127.6ms/step | 19.2 samples/s +Step 46110/50462 | Loss: 2.6400 | LR: 9.40e-06 | 3126.3ms/step | 19.2 samples/s +Step 46120/50462 | Loss: 2.6346 | LR: 9.35e-06 | 3119.9ms/step | 19.2 samples/s +Step 46130/50462 | Loss: 2.6356 | LR: 9.31e-06 | 3124.2ms/step | 19.2 samples/s +Step 46140/50462 | Loss: 2.6333 | LR: 9.27e-06 | 3128.1ms/step | 19.2 samples/s +Step 46150/50462 | Loss: 2.6333 | LR: 9.22e-06 | 3126.6ms/step | 19.2 samples/s +Step 46160/50462 | Loss: 2.6514 | LR: 9.18e-06 | 3122.4ms/step | 19.2 samples/s +Step 46170/50462 | Loss: 2.6526 | LR: 9.14e-06 | 3123.7ms/step | 19.2 samples/s +Step 46180/50462 | Loss: 2.6226 | LR: 9.10e-06 | 3121.0ms/step | 19.2 samples/s +Step 46190/50462 | Loss: 2.6235 | LR: 9.06e-06 | 3124.7ms/step | 19.2 samples/s +Step 46200/50462 | Loss: 2.6295 | LR: 9.01e-06 | 3126.3ms/step | 19.2 samples/s +Step 46210/50462 | Loss: 2.6495 | LR: 8.97e-06 | 3117.4ms/step | 19.2 samples/s +Step 46220/50462 | Loss: 2.6492 | LR: 8.93e-06 | 3120.4ms/step | 19.2 samples/s +Step 46230/50462 | Loss: 2.6560 | LR: 8.89e-06 | 3126.0ms/step | 19.2 samples/s +Step 46240/50462 | Loss: 2.6639 | LR: 8.85e-06 | 3178.0ms/step | 18.9 samples/s +Step 46250/50462 | Loss: 2.6592 | LR: 8.80e-06 | 3120.4ms/step | 19.2 samples/s +Validation at step 46250: Loss = 2.6524, Perplexity = 14.19 +Step 46260/50462 | Loss: 2.6621 | LR: 8.76e-06 | 4778.3ms/step | 12.6 samples/s +Step 46270/50462 | Loss: 2.6485 | LR: 8.72e-06 | 3122.0ms/step | 19.2 samples/s +Step 46280/50462 | Loss: 2.6795 | LR: 8.68e-06 | 3121.4ms/step | 19.2 samples/s +Step 46290/50462 | Loss: 2.6684 | LR: 8.64e-06 | 3114.9ms/step | 19.3 samples/s +Step 46300/50462 | Loss: 2.6358 | LR: 8.60e-06 | 3118.1ms/step | 19.2 samples/s +Step 46310/50462 | Loss: 2.6428 | LR: 8.56e-06 | 3234.6ms/step | 18.5 samples/s +Step 46320/50462 | Loss: 2.6702 | LR: 8.52e-06 | 3126.7ms/step | 19.2 samples/s +Step 46330/50462 | Loss: 2.6513 | LR: 8.47e-06 | 3118.8ms/step | 19.2 samples/s +Step 46340/50462 | Loss: 2.6489 | LR: 8.43e-06 | 3122.9ms/step | 19.2 samples/s +Step 46350/50462 | Loss: 2.6704 | LR: 8.39e-06 | 3122.8ms/step | 19.2 samples/s +Step 46360/50462 | Loss: 2.6356 | LR: 8.35e-06 | 3123.0ms/step | 19.2 samples/s +Step 46370/50462 | Loss: 2.6178 | LR: 8.31e-06 | 3123.5ms/step | 19.2 samples/s +Step 46380/50462 | Loss: 2.6456 | LR: 8.27e-06 | 3111.1ms/step | 19.3 samples/s +Step 46390/50462 | Loss: 2.6341 | LR: 8.23e-06 | 3120.5ms/step | 19.2 samples/s +Step 46400/50462 | Loss: 2.6258 | LR: 8.19e-06 | 3123.6ms/step | 19.2 samples/s +Step 46410/50462 | Loss: 2.6369 | LR: 8.15e-06 | 3121.9ms/step | 19.2 samples/s +Step 46420/50462 | Loss: 2.6336 | LR: 8.11e-06 | 3123.3ms/step | 19.2 samples/s +Step 46430/50462 | Loss: 2.6595 | LR: 8.07e-06 | 3118.5ms/step | 19.2 samples/s +Step 46440/50462 | Loss: 2.6394 | LR: 8.03e-06 | 3116.4ms/step | 19.3 samples/s +Step 46450/50462 | Loss: 2.6675 | LR: 7.99e-06 | 3126.8ms/step | 19.2 samples/s +Step 46460/50462 | Loss: 2.6435 | LR: 7.95e-06 | 3125.2ms/step | 19.2 samples/s +Step 46470/50462 | Loss: 2.6352 | LR: 7.91e-06 | 3128.2ms/step | 19.2 samples/s +Step 46480/50462 | Loss: 2.6317 | LR: 7.87e-06 | 3125.6ms/step | 19.2 samples/s +Step 46490/50462 | Loss: 2.6266 | LR: 7.83e-06 | 3122.3ms/step | 19.2 samples/s +Step 46500/50462 | Loss: 2.6463 | LR: 7.80e-06 | 3129.4ms/step | 19.2 samples/s +Validation at step 46500: Loss = 2.6524, Perplexity = 14.19 +Step 46510/50462 | Loss: 2.6369 | LR: 7.76e-06 | 4797.7ms/step | 12.5 samples/s +Step 46520/50462 | Loss: 2.6611 | LR: 7.72e-06 | 3126.9ms/step | 19.2 samples/s +Step 46530/50462 | Loss: 2.6400 | LR: 7.68e-06 | 3127.7ms/step | 19.2 samples/s +Step 46540/50462 | Loss: 2.6575 | LR: 7.64e-06 | 3121.9ms/step | 19.2 samples/s +Step 46550/50462 | Loss: 2.6174 | LR: 7.60e-06 | 3118.2ms/step | 19.2 samples/s +Step 46560/50462 | Loss: 2.6320 | LR: 7.56e-06 | 3183.0ms/step | 18.8 samples/s +Step 46570/50462 | Loss: 2.6443 | LR: 7.52e-06 | 3117.4ms/step | 19.2 samples/s +Step 46580/50462 | Loss: 2.6345 | LR: 7.49e-06 | 3118.1ms/step | 19.2 samples/s +Step 46590/50462 | Loss: 2.6274 | LR: 7.45e-06 | 3120.2ms/step | 19.2 samples/s +Step 46600/50462 | Loss: 2.6452 | LR: 7.41e-06 | 3126.6ms/step | 19.2 samples/s +Step 46610/50462 | Loss: 2.6431 | LR: 7.37e-06 | 3123.4ms/step | 19.2 samples/s +Step 46620/50462 | Loss: 2.6227 | LR: 7.33e-06 | 3121.2ms/step | 19.2 samples/s +Step 46630/50462 | Loss: 2.6396 | LR: 7.29e-06 | 3238.6ms/step | 18.5 samples/s +Step 46640/50462 | Loss: 2.6502 | LR: 7.26e-06 | 3116.0ms/step | 19.3 samples/s +Step 46650/50462 | Loss: 2.6440 | LR: 7.22e-06 | 3123.1ms/step | 19.2 samples/s +Step 46660/50462 | Loss: 2.6330 | LR: 7.18e-06 | 3120.6ms/step | 19.2 samples/s +Step 46670/50462 | Loss: 2.6502 | LR: 7.14e-06 | 3120.4ms/step | 19.2 samples/s +Step 46680/50462 | Loss: 2.6561 | LR: 7.11e-06 | 3126.3ms/step | 19.2 samples/s +Step 46690/50462 | Loss: 2.6729 | LR: 7.07e-06 | 3121.2ms/step | 19.2 samples/s +Step 46700/50462 | Loss: 2.6335 | LR: 7.03e-06 | 3122.3ms/step | 19.2 samples/s +Step 46710/50462 | Loss: 2.6667 | LR: 6.99e-06 | 3125.8ms/step | 19.2 samples/s +Step 46720/50462 | Loss: 2.6506 | LR: 6.96e-06 | 3118.0ms/step | 19.2 samples/s +Step 46730/50462 | Loss: 2.6304 | LR: 6.92e-06 | 3125.7ms/step | 19.2 samples/s +Step 46740/50462 | Loss: 2.6540 | LR: 6.88e-06 | 3128.3ms/step | 19.2 samples/s +Step 46750/50462 | Loss: 2.6371 | LR: 6.85e-06 | 3116.0ms/step | 19.3 samples/s +Validation at step 46750: Loss = 2.6524, Perplexity = 14.19 +Step 46760/50462 | Loss: 2.6531 | LR: 6.81e-06 | 4785.0ms/step | 12.5 samples/s +Step 46770/50462 | Loss: 2.6433 | LR: 6.77e-06 | 3113.2ms/step | 19.3 samples/s +Step 46780/50462 | Loss: 2.6712 | LR: 6.74e-06 | 3116.6ms/step | 19.3 samples/s +Step 46790/50462 | Loss: 2.6242 | LR: 6.70e-06 | 3126.1ms/step | 19.2 samples/s +Step 46800/50462 | Loss: 2.6625 | LR: 6.66e-06 | 3122.0ms/step | 19.2 samples/s +Step 46810/50462 | Loss: 2.6563 | LR: 6.63e-06 | 3112.3ms/step | 19.3 samples/s +Step 46820/50462 | Loss: 2.6226 | LR: 6.59e-06 | 3116.1ms/step | 19.3 samples/s +Step 46830/50462 | Loss: 2.6232 | LR: 6.56e-06 | 3118.1ms/step | 19.2 samples/s +Step 46840/50462 | Loss: 2.6475 | LR: 6.52e-06 | 3122.7ms/step | 19.2 samples/s +Step 46850/50462 | Loss: 2.6402 | LR: 6.48e-06 | 3119.4ms/step | 19.2 samples/s +Step 46860/50462 | Loss: 2.6408 | LR: 6.45e-06 | 3115.9ms/step | 19.3 samples/s +Step 46870/50462 | Loss: 2.6002 | LR: 6.41e-06 | 3110.4ms/step | 19.3 samples/s +Step 46880/50462 | Loss: 2.6365 | LR: 6.38e-06 | 3173.7ms/step | 18.9 samples/s +Step 46890/50462 | Loss: 2.6423 | LR: 6.34e-06 | 3120.6ms/step | 19.2 samples/s +Step 46900/50462 | Loss: 2.6334 | LR: 6.31e-06 | 3115.6ms/step | 19.3 samples/s +Step 46910/50462 | Loss: 2.6248 | LR: 6.27e-06 | 3125.8ms/step | 19.2 samples/s +Step 46920/50462 | Loss: 2.6648 | LR: 6.24e-06 | 3121.7ms/step | 19.2 samples/s +Step 46930/50462 | Loss: 2.6395 | LR: 6.20e-06 | 3118.9ms/step | 19.2 samples/s +Step 46940/50462 | Loss: 2.6443 | LR: 6.17e-06 | 3123.9ms/step | 19.2 samples/s +Step 46950/50462 | Loss: 2.6299 | LR: 6.13e-06 | 3229.3ms/step | 18.6 samples/s +Step 46960/50462 | Loss: 2.6550 | LR: 6.10e-06 | 3118.3ms/step | 19.2 samples/s +Step 46970/50462 | Loss: 2.6536 | LR: 6.06e-06 | 3115.8ms/step | 19.3 samples/s +Step 46980/50462 | Loss: 2.6497 | LR: 6.03e-06 | 3111.4ms/step | 19.3 samples/s +Step 46990/50462 | Loss: 2.6708 | LR: 5.99e-06 | 3118.8ms/step | 19.2 samples/s +Step 47000/50462 | Loss: 2.6534 | LR: 5.96e-06 | 3115.1ms/step | 19.3 samples/s +Validation at step 47000: Loss = 2.6523, Perplexity = 14.19 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-47000 +Step 47010/50462 | Loss: 2.6653 | LR: 5.93e-06 | 5637.8ms/step | 10.6 samples/s +Step 47020/50462 | Loss: 2.6405 | LR: 5.89e-06 | 3119.9ms/step | 19.2 samples/s +Step 47030/50462 | Loss: 2.6616 | LR: 5.86e-06 | 3123.1ms/step | 19.2 samples/s +Step 47040/50462 | Loss: 2.6074 | LR: 5.82e-06 | 3125.0ms/step | 19.2 samples/s +Step 47050/50462 | Loss: 2.6474 | LR: 5.79e-06 | 3130.5ms/step | 19.2 samples/s +Step 47060/50462 | Loss: 2.6437 | LR: 5.76e-06 | 3116.5ms/step | 19.3 samples/s +Step 47070/50462 | Loss: 2.6326 | LR: 5.72e-06 | 3123.9ms/step | 19.2 samples/s +Step 47080/50462 | Loss: 2.6425 | LR: 5.69e-06 | 3118.8ms/step | 19.2 samples/s +Step 47090/50462 | Loss: 2.6362 | LR: 5.65e-06 | 3120.2ms/step | 19.2 samples/s +Step 47100/50462 | Loss: 2.6426 | LR: 5.62e-06 | 3116.4ms/step | 19.3 samples/s +Step 47110/50462 | Loss: 2.6460 | LR: 5.59e-06 | 3123.6ms/step | 19.2 samples/s +Step 47120/50462 | Loss: 2.5977 | LR: 5.56e-06 | 3120.8ms/step | 19.2 samples/s +Step 47130/50462 | Loss: 2.6408 | LR: 5.52e-06 | 3122.3ms/step | 19.2 samples/s +Step 47140/50462 | Loss: 2.6546 | LR: 5.49e-06 | 3117.1ms/step | 19.2 samples/s +Step 47150/50462 | Loss: 2.6147 | LR: 5.46e-06 | 3121.3ms/step | 19.2 samples/s +Step 47160/50462 | Loss: 2.6537 | LR: 5.42e-06 | 3123.7ms/step | 19.2 samples/s +Step 47170/50462 | Loss: 2.6370 | LR: 5.39e-06 | 3123.0ms/step | 19.2 samples/s +Step 47180/50462 | Loss: 2.6466 | LR: 5.36e-06 | 3120.7ms/step | 19.2 samples/s +Step 47190/50462 | Loss: 2.6405 | LR: 5.33e-06 | 3119.6ms/step | 19.2 samples/s +Step 47200/50462 | Loss: 2.6355 | LR: 5.29e-06 | 3183.6ms/step | 18.8 samples/s +Step 47210/50462 | Loss: 2.6123 | LR: 5.26e-06 | 3130.9ms/step | 19.2 samples/s +Step 47220/50462 | Loss: 2.6538 | LR: 5.23e-06 | 3128.4ms/step | 19.2 samples/s +Step 47230/50462 | Loss: 2.6482 | LR: 5.20e-06 | 3122.0ms/step | 19.2 samples/s +Step 47240/50462 | Loss: 2.6194 | LR: 5.16e-06 | 3114.7ms/step | 19.3 samples/s +Step 47250/50462 | Loss: 2.6486 | LR: 5.13e-06 | 3115.5ms/step | 19.3 samples/s +Validation at step 47250: Loss = 2.6523, Perplexity = 14.19 +Step 47260/50462 | Loss: 2.6418 | LR: 5.10e-06 | 4774.2ms/step | 12.6 samples/s +Step 47270/50462 | Loss: 2.6571 | LR: 5.07e-06 | 3181.2ms/step | 18.9 samples/s +Step 47280/50462 | Loss: 2.6312 | LR: 5.04e-06 | 3176.0ms/step | 18.9 samples/s +Step 47290/50462 | Loss: 2.6577 | LR: 5.01e-06 | 3113.8ms/step | 19.3 samples/s +Step 47300/50462 | Loss: 2.6564 | LR: 4.97e-06 | 3120.7ms/step | 19.2 samples/s +Step 47310/50462 | Loss: 2.6197 | LR: 4.94e-06 | 3118.1ms/step | 19.2 samples/s +Step 47320/50462 | Loss: 2.6349 | LR: 4.91e-06 | 3114.5ms/step | 19.3 samples/s +Step 47330/50462 | Loss: 2.6379 | LR: 4.88e-06 | 3125.2ms/step | 19.2 samples/s +Step 47340/50462 | Loss: 2.6043 | LR: 4.85e-06 | 3125.8ms/step | 19.2 samples/s +Step 47350/50462 | Loss: 2.6512 | LR: 4.82e-06 | 3125.1ms/step | 19.2 samples/s +Step 47360/50462 | Loss: 2.6176 | LR: 4.79e-06 | 3119.3ms/step | 19.2 samples/s +Step 47370/50462 | Loss: 2.6601 | LR: 4.76e-06 | 3116.5ms/step | 19.3 samples/s +Step 47380/50462 | Loss: 2.6517 | LR: 4.73e-06 | 3121.2ms/step | 19.2 samples/s +Step 47390/50462 | Loss: 2.6373 | LR: 4.70e-06 | 3119.0ms/step | 19.2 samples/s +Step 47400/50462 | Loss: 2.6358 | LR: 4.67e-06 | 3119.4ms/step | 19.2 samples/s +Step 47410/50462 | Loss: 2.6520 | LR: 4.64e-06 | 3119.0ms/step | 19.2 samples/s +Step 47420/50462 | Loss: 2.5991 | LR: 4.61e-06 | 3111.4ms/step | 19.3 samples/s +Step 47430/50462 | Loss: 2.6304 | LR: 4.58e-06 | 3113.5ms/step | 19.3 samples/s +Step 47440/50462 | Loss: 2.6393 | LR: 4.55e-06 | 3122.6ms/step | 19.2 samples/s +Step 47450/50462 | Loss: 2.6111 | LR: 4.52e-06 | 3110.5ms/step | 19.3 samples/s +Step 47460/50462 | Loss: 2.6550 | LR: 4.49e-06 | 3113.9ms/step | 19.3 samples/s +Step 47470/50462 | Loss: 2.6559 | LR: 4.46e-06 | 3106.3ms/step | 19.3 samples/s +Step 47480/50462 | Loss: 2.6510 | LR: 4.43e-06 | 3110.5ms/step | 19.3 samples/s +Step 47490/50462 | Loss: 2.6459 | LR: 4.40e-06 | 3113.3ms/step | 19.3 samples/s +Step 47500/50462 | Loss: 2.6387 | LR: 4.37e-06 | 3120.4ms/step | 19.2 samples/s +Validation at step 47500: Loss = 2.6523, Perplexity = 14.19 +Step 47510/50462 | Loss: 2.6080 | LR: 4.34e-06 | 4777.6ms/step | 12.6 samples/s +Step 47520/50462 | Loss: 2.6558 | LR: 4.31e-06 | 3172.9ms/step | 18.9 samples/s +Step 47530/50462 | Loss: 2.6521 | LR: 4.28e-06 | 3118.2ms/step | 19.2 samples/s +Step 47540/50462 | Loss: 2.6229 | LR: 4.25e-06 | 3114.4ms/step | 19.3 samples/s +Step 47550/50462 | Loss: 2.6509 | LR: 4.22e-06 | 3122.1ms/step | 19.2 samples/s +Step 47560/50462 | Loss: 2.6323 | LR: 4.19e-06 | 3111.9ms/step | 19.3 samples/s +Step 47570/50462 | Loss: 2.6466 | LR: 4.16e-06 | 3120.6ms/step | 19.2 samples/s +Step 47580/50462 | Loss: 2.6808 | LR: 4.13e-06 | 3121.8ms/step | 19.2 samples/s +Step 47590/50462 | Loss: 2.6511 | LR: 4.11e-06 | 3172.5ms/step | 18.9 samples/s +Step 47600/50462 | Loss: 2.6418 | LR: 4.08e-06 | 3174.7ms/step | 18.9 samples/s +Step 47610/50462 | Loss: 2.6203 | LR: 4.05e-06 | 3121.4ms/step | 19.2 samples/s +Step 47620/50462 | Loss: 2.6321 | LR: 4.02e-06 | 3111.2ms/step | 19.3 samples/s +Step 47630/50462 | Loss: 2.6639 | LR: 3.99e-06 | 3113.2ms/step | 19.3 samples/s +Step 47640/50462 | Loss: 2.6168 | LR: 3.97e-06 | 3122.2ms/step | 19.2 samples/s +Step 47650/50462 | Loss: 2.6395 | LR: 3.94e-06 | 3122.0ms/step | 19.2 samples/s +Step 47660/50462 | Loss: 2.6388 | LR: 3.91e-06 | 3121.2ms/step | 19.2 samples/s +Step 47670/50462 | Loss: 2.6278 | LR: 3.88e-06 | 3119.7ms/step | 19.2 samples/s +Step 47680/50462 | Loss: 2.6469 | LR: 3.85e-06 | 3124.7ms/step | 19.2 samples/s +Step 47690/50462 | Loss: 2.6388 | LR: 3.83e-06 | 3120.5ms/step | 19.2 samples/s +Step 47700/50462 | Loss: 2.6508 | LR: 3.80e-06 | 3123.4ms/step | 19.2 samples/s +Step 47710/50462 | Loss: 2.6168 | LR: 3.77e-06 | 3121.9ms/step | 19.2 samples/s +Step 47720/50462 | Loss: 2.6423 | LR: 3.74e-06 | 3115.3ms/step | 19.3 samples/s +Step 47730/50462 | Loss: 2.6421 | LR: 3.72e-06 | 3120.2ms/step | 19.2 samples/s +Step 47740/50462 | Loss: 2.6275 | LR: 3.69e-06 | 3121.5ms/step | 19.2 samples/s +Step 47750/50462 | Loss: 2.6524 | LR: 3.66e-06 | 3122.5ms/step | 19.2 samples/s +Validation at step 47750: Loss = 2.6523, Perplexity = 14.19 +Step 47760/50462 | Loss: 2.6145 | LR: 3.64e-06 | 4775.0ms/step | 12.6 samples/s +Step 47770/50462 | Loss: 2.6788 | LR: 3.61e-06 | 3120.4ms/step | 19.2 samples/s +Step 47780/50462 | Loss: 2.6365 | LR: 3.58e-06 | 3119.7ms/step | 19.2 samples/s +Step 47790/50462 | Loss: 2.6256 | LR: 3.56e-06 | 3119.3ms/step | 19.2 samples/s +Step 47800/50462 | Loss: 2.6191 | LR: 3.53e-06 | 3117.9ms/step | 19.2 samples/s +Step 47810/50462 | Loss: 2.6502 | LR: 3.50e-06 | 3120.1ms/step | 19.2 samples/s +Step 47820/50462 | Loss: 2.6470 | LR: 3.48e-06 | 3118.6ms/step | 19.2 samples/s +Step 47830/50462 | Loss: 2.6396 | LR: 3.45e-06 | 3119.1ms/step | 19.2 samples/s +Step 47840/50462 | Loss: 2.6600 | LR: 3.42e-06 | 3171.4ms/step | 18.9 samples/s +Step 47850/50462 | Loss: 2.6521 | LR: 3.40e-06 | 3110.1ms/step | 19.3 samples/s +Step 47860/50462 | Loss: 2.6568 | LR: 3.37e-06 | 3111.3ms/step | 19.3 samples/s +Step 47870/50462 | Loss: 2.6479 | LR: 3.35e-06 | 3110.2ms/step | 19.3 samples/s +Step 47880/50462 | Loss: 2.6040 | LR: 3.32e-06 | 3112.8ms/step | 19.3 samples/s +Step 47890/50462 | Loss: 2.6606 | LR: 3.30e-06 | 3123.7ms/step | 19.2 samples/s +Step 47900/50462 | Loss: 2.6517 | LR: 3.27e-06 | 3114.0ms/step | 19.3 samples/s +Step 47910/50462 | Loss: 2.6486 | LR: 3.24e-06 | 3170.9ms/step | 18.9 samples/s +Step 47920/50462 | Loss: 2.6228 | LR: 3.22e-06 | 3177.2ms/step | 18.9 samples/s +Step 47930/50462 | Loss: 2.6619 | LR: 3.19e-06 | 3120.7ms/step | 19.2 samples/s +Step 47940/50462 | Loss: 2.6349 | LR: 3.17e-06 | 3116.4ms/step | 19.3 samples/s +Step 47950/50462 | Loss: 2.6430 | LR: 3.14e-06 | 3122.1ms/step | 19.2 samples/s +Step 47960/50462 | Loss: 2.6095 | LR: 3.12e-06 | 3116.7ms/step | 19.3 samples/s +Step 47970/50462 | Loss: 2.6455 | LR: 3.09e-06 | 3114.3ms/step | 19.3 samples/s +Step 47980/50462 | Loss: 2.6451 | LR: 3.07e-06 | 3121.4ms/step | 19.2 samples/s +Step 47990/50462 | Loss: 2.6265 | LR: 3.04e-06 | 3113.8ms/step | 19.3 samples/s +Step 48000/50462 | Loss: 2.6342 | LR: 3.02e-06 | 3117.0ms/step | 19.2 samples/s +Validation at step 48000: Loss = 2.6523, Perplexity = 14.19 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-48000 +Step 48010/50462 | Loss: 2.6480 | LR: 3.00e-06 | 5645.9ms/step | 10.6 samples/s +Step 48020/50462 | Loss: 2.6806 | LR: 2.97e-06 | 3114.5ms/step | 19.3 samples/s +Step 48030/50462 | Loss: 2.6194 | LR: 2.95e-06 | 3118.7ms/step | 19.2 samples/s +Step 48040/50462 | Loss: 2.6310 | LR: 2.92e-06 | 3114.1ms/step | 19.3 samples/s +Step 48050/50462 | Loss: 2.6274 | LR: 2.90e-06 | 3122.7ms/step | 19.2 samples/s +Step 48060/50462 | Loss: 2.6418 | LR: 2.87e-06 | 3128.3ms/step | 19.2 samples/s +Step 48070/50462 | Loss: 2.6465 | LR: 2.85e-06 | 3119.0ms/step | 19.2 samples/s +Step 48080/50462 | Loss: 2.6679 | LR: 2.83e-06 | 3120.1ms/step | 19.2 samples/s +Step 48090/50462 | Loss: 2.6455 | LR: 2.80e-06 | 3123.8ms/step | 19.2 samples/s +Step 48100/50462 | Loss: 2.6506 | LR: 2.78e-06 | 3119.7ms/step | 19.2 samples/s +Step 48110/50462 | Loss: 2.6378 | LR: 2.76e-06 | 3121.4ms/step | 19.2 samples/s +Step 48120/50462 | Loss: 2.6355 | LR: 2.73e-06 | 3114.9ms/step | 19.3 samples/s +Step 48130/50462 | Loss: 2.6366 | LR: 2.71e-06 | 3122.4ms/step | 19.2 samples/s +Step 48140/50462 | Loss: 2.6167 | LR: 2.69e-06 | 3121.0ms/step | 19.2 samples/s +Step 48150/50462 | Loss: 2.6306 | LR: 2.66e-06 | 3135.3ms/step | 19.1 samples/s +Step 48160/50462 | Loss: 2.6534 | LR: 2.64e-06 | 3179.5ms/step | 18.9 samples/s +Step 48170/50462 | Loss: 2.6327 | LR: 2.62e-06 | 3120.6ms/step | 19.2 samples/s +Step 48180/50462 | Loss: 2.6445 | LR: 2.60e-06 | 3136.1ms/step | 19.1 samples/s +Step 48190/50462 | Loss: 2.6398 | LR: 2.57e-06 | 3132.0ms/step | 19.2 samples/s +Step 48200/50462 | Loss: 2.6519 | LR: 2.55e-06 | 3117.3ms/step | 19.2 samples/s +Step 48210/50462 | Loss: 2.6254 | LR: 2.53e-06 | 3111.4ms/step | 19.3 samples/s +Step 48220/50462 | Loss: 2.6330 | LR: 2.51e-06 | 3114.7ms/step | 19.3 samples/s +Step 48230/50462 | Loss: 2.6414 | LR: 2.48e-06 | 3118.9ms/step | 19.2 samples/s +Step 48240/50462 | Loss: 2.6382 | LR: 2.46e-06 | 3232.7ms/step | 18.6 samples/s +Step 48250/50462 | Loss: 2.6743 | LR: 2.44e-06 | 3123.5ms/step | 19.2 samples/s +Validation at step 48250: Loss = 2.6523, Perplexity = 14.19 +Step 48260/50462 | Loss: 2.6108 | LR: 2.42e-06 | 4787.2ms/step | 12.5 samples/s +Step 48270/50462 | Loss: 2.6390 | LR: 2.39e-06 | 3122.3ms/step | 19.2 samples/s +Step 48280/50462 | Loss: 2.6491 | LR: 2.37e-06 | 3119.7ms/step | 19.2 samples/s +Step 48290/50462 | Loss: 2.6340 | LR: 2.35e-06 | 3119.4ms/step | 19.2 samples/s +Step 48300/50462 | Loss: 2.6260 | LR: 2.33e-06 | 3117.7ms/step | 19.2 samples/s +Step 48310/50462 | Loss: 2.6507 | LR: 2.31e-06 | 3121.7ms/step | 19.2 samples/s +Step 48320/50462 | Loss: 2.6454 | LR: 2.29e-06 | 3122.4ms/step | 19.2 samples/s +Step 48330/50462 | Loss: 2.6343 | LR: 2.27e-06 | 3126.8ms/step | 19.2 samples/s +Step 48340/50462 | Loss: 2.6522 | LR: 2.24e-06 | 3118.9ms/step | 19.2 samples/s +Step 48350/50462 | Loss: 2.6664 | LR: 2.22e-06 | 3130.5ms/step | 19.2 samples/s +Step 48360/50462 | Loss: 2.6257 | LR: 2.20e-06 | 3127.1ms/step | 19.2 samples/s +Step 48370/50462 | Loss: 2.6615 | LR: 2.18e-06 | 3129.2ms/step | 19.2 samples/s +Step 48380/50462 | Loss: 2.6281 | LR: 2.16e-06 | 3129.0ms/step | 19.2 samples/s +Step 48390/50462 | Loss: 2.6724 | LR: 2.14e-06 | 3131.4ms/step | 19.2 samples/s +Step 48400/50462 | Loss: 2.6312 | LR: 2.12e-06 | 3129.3ms/step | 19.2 samples/s +Step 48410/50462 | Loss: 2.6582 | LR: 2.10e-06 | 3124.5ms/step | 19.2 samples/s +Step 48420/50462 | Loss: 2.6338 | LR: 2.08e-06 | 3127.1ms/step | 19.2 samples/s +Step 48430/50462 | Loss: 2.6318 | LR: 2.06e-06 | 3132.0ms/step | 19.2 samples/s +Step 48440/50462 | Loss: 2.6503 | LR: 2.04e-06 | 3123.5ms/step | 19.2 samples/s +Step 48450/50462 | Loss: 2.6135 | LR: 2.02e-06 | 3122.5ms/step | 19.2 samples/s +Step 48460/50462 | Loss: 2.6638 | LR: 2.00e-06 | 3130.0ms/step | 19.2 samples/s +Step 48470/50462 | Loss: 2.6523 | LR: 1.98e-06 | 3127.1ms/step | 19.2 samples/s +Step 48480/50462 | Loss: 2.6337 | LR: 1.96e-06 | 3187.0ms/step | 18.8 samples/s +Step 48490/50462 | Loss: 2.6403 | LR: 1.94e-06 | 3125.0ms/step | 19.2 samples/s +Step 48500/50462 | Loss: 2.6083 | LR: 1.92e-06 | 3129.4ms/step | 19.2 samples/s +Validation at step 48500: Loss = 2.6523, Perplexity = 14.19 +Step 48510/50462 | Loss: 2.6432 | LR: 1.90e-06 | 4784.6ms/step | 12.5 samples/s +Step 48520/50462 | Loss: 2.6429 | LR: 1.88e-06 | 3127.9ms/step | 19.2 samples/s +Step 48530/50462 | Loss: 2.6614 | LR: 1.86e-06 | 3131.5ms/step | 19.2 samples/s +Step 48540/50462 | Loss: 2.6170 | LR: 1.84e-06 | 3132.8ms/step | 19.2 samples/s +Step 48550/50462 | Loss: 2.6260 | LR: 1.82e-06 | 3127.2ms/step | 19.2 samples/s +Step 48560/50462 | Loss: 2.6367 | LR: 1.80e-06 | 3240.2ms/step | 18.5 samples/s +Step 48570/50462 | Loss: 2.6389 | LR: 1.78e-06 | 3141.1ms/step | 19.1 samples/s +Step 48580/50462 | Loss: 2.6311 | LR: 1.77e-06 | 3146.8ms/step | 19.1 samples/s +Step 48590/50462 | Loss: 2.6570 | LR: 1.75e-06 | 3126.7ms/step | 19.2 samples/s +Step 48600/50462 | Loss: 2.6448 | LR: 1.73e-06 | 3129.2ms/step | 19.2 samples/s +Step 48610/50462 | Loss: 2.6306 | LR: 1.71e-06 | 3124.5ms/step | 19.2 samples/s +Step 48620/50462 | Loss: 2.6389 | LR: 1.69e-06 | 3120.5ms/step | 19.2 samples/s +Step 48630/50462 | Loss: 2.6608 | LR: 1.67e-06 | 3124.2ms/step | 19.2 samples/s +Step 48640/50462 | Loss: 2.6422 | LR: 1.66e-06 | 3127.5ms/step | 19.2 samples/s +Step 48650/50462 | Loss: 2.6574 | LR: 1.64e-06 | 3118.2ms/step | 19.2 samples/s +Step 48660/50462 | Loss: 2.6542 | LR: 1.62e-06 | 3117.6ms/step | 19.2 samples/s +Step 48670/50462 | Loss: 2.6545 | LR: 1.60e-06 | 3122.9ms/step | 19.2 samples/s +Step 48680/50462 | Loss: 2.6374 | LR: 1.58e-06 | 3115.2ms/step | 19.3 samples/s +Step 48690/50462 | Loss: 2.6510 | LR: 1.57e-06 | 3115.2ms/step | 19.3 samples/s +Step 48700/50462 | Loss: 2.6147 | LR: 1.55e-06 | 3118.5ms/step | 19.2 samples/s +Step 48710/50462 | Loss: 2.6366 | LR: 1.53e-06 | 3117.5ms/step | 19.2 samples/s +Step 48720/50462 | Loss: 2.6569 | LR: 1.51e-06 | 3114.4ms/step | 19.3 samples/s +Step 48730/50462 | Loss: 2.6399 | LR: 1.50e-06 | 3116.9ms/step | 19.2 samples/s +Step 48740/50462 | Loss: 2.6190 | LR: 1.48e-06 | 3115.2ms/step | 19.3 samples/s +Step 48750/50462 | Loss: 2.6620 | LR: 1.46e-06 | 3129.1ms/step | 19.2 samples/s +Validation at step 48750: Loss = 2.6523, Perplexity = 14.19 +Step 48760/50462 | Loss: 2.6731 | LR: 1.44e-06 | 4803.6ms/step | 12.5 samples/s +Step 48770/50462 | Loss: 2.6131 | LR: 1.43e-06 | 3135.4ms/step | 19.1 samples/s +Step 48780/50462 | Loss: 2.6367 | LR: 1.41e-06 | 3128.4ms/step | 19.2 samples/s +Step 48790/50462 | Loss: 2.6470 | LR: 1.39e-06 | 3132.9ms/step | 19.2 samples/s +Step 48800/50462 | Loss: 2.6238 | LR: 1.38e-06 | 3186.9ms/step | 18.8 samples/s +Step 48810/50462 | Loss: 2.6301 | LR: 1.36e-06 | 3143.4ms/step | 19.1 samples/s +Step 48820/50462 | Loss: 2.6411 | LR: 1.34e-06 | 3133.1ms/step | 19.2 samples/s +Step 48830/50462 | Loss: 2.6525 | LR: 1.33e-06 | 3140.2ms/step | 19.1 samples/s +Step 48840/50462 | Loss: 2.6681 | LR: 1.31e-06 | 3131.4ms/step | 19.2 samples/s +Step 48850/50462 | Loss: 2.6377 | LR: 1.30e-06 | 3144.9ms/step | 19.1 samples/s +Step 48860/50462 | Loss: 2.6064 | LR: 1.28e-06 | 3133.4ms/step | 19.1 samples/s +Step 48870/50462 | Loss: 2.6508 | LR: 1.26e-06 | 3139.7ms/step | 19.1 samples/s +Step 48880/50462 | Loss: 2.6319 | LR: 1.25e-06 | 3235.1ms/step | 18.5 samples/s +Step 48890/50462 | Loss: 2.6605 | LR: 1.23e-06 | 3128.4ms/step | 19.2 samples/s +Step 48900/50462 | Loss: 2.6412 | LR: 1.22e-06 | 3128.0ms/step | 19.2 samples/s +Step 48910/50462 | Loss: 2.6548 | LR: 1.20e-06 | 3139.1ms/step | 19.1 samples/s +Step 48920/50462 | Loss: 2.6401 | LR: 1.19e-06 | 3134.9ms/step | 19.1 samples/s +Step 48930/50462 | Loss: 2.6366 | LR: 1.17e-06 | 3136.3ms/step | 19.1 samples/s +Step 48940/50462 | Loss: 2.6168 | LR: 1.16e-06 | 3138.8ms/step | 19.1 samples/s +Step 48950/50462 | Loss: 2.6181 | LR: 1.14e-06 | 3128.2ms/step | 19.2 samples/s +Step 48960/50462 | Loss: 2.6492 | LR: 1.13e-06 | 3135.7ms/step | 19.1 samples/s +Step 48970/50462 | Loss: 2.6148 | LR: 1.11e-06 | 3137.5ms/step | 19.1 samples/s +Step 48980/50462 | Loss: 2.6797 | LR: 1.10e-06 | 3132.5ms/step | 19.2 samples/s +Step 48990/50462 | Loss: 2.6659 | LR: 1.08e-06 | 3140.4ms/step | 19.1 samples/s +Step 49000/50462 | Loss: 2.6508 | LR: 1.07e-06 | 3124.7ms/step | 19.2 samples/s +Validation at step 49000: Loss = 2.6523, Perplexity = 14.19 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-49000 +Step 49010/50462 | Loss: 2.6364 | LR: 1.05e-06 | 5665.9ms/step | 10.6 samples/s +Step 49020/50462 | Loss: 2.6454 | LR: 1.04e-06 | 3123.0ms/step | 19.2 samples/s +Step 49030/50462 | Loss: 2.6483 | LR: 1.02e-06 | 3131.6ms/step | 19.2 samples/s +Step 49040/50462 | Loss: 2.6438 | LR: 1.01e-06 | 3135.7ms/step | 19.1 samples/s +Step 49050/50462 | Loss: 2.6468 | LR: 9.95e-07 | 3136.0ms/step | 19.1 samples/s +Step 49060/50462 | Loss: 2.6646 | LR: 9.81e-07 | 3137.6ms/step | 19.1 samples/s +Step 49070/50462 | Loss: 2.6621 | LR: 9.67e-07 | 3133.6ms/step | 19.1 samples/s +Step 49080/50462 | Loss: 2.6672 | LR: 9.53e-07 | 3132.1ms/step | 19.2 samples/s +Step 49090/50462 | Loss: 2.6163 | LR: 9.39e-07 | 3130.9ms/step | 19.2 samples/s +Step 49100/50462 | Loss: 2.6666 | LR: 9.25e-07 | 3133.5ms/step | 19.1 samples/s +Step 49110/50462 | Loss: 2.6509 | LR: 9.12e-07 | 3135.7ms/step | 19.1 samples/s +Step 49120/50462 | Loss: 2.6218 | LR: 8.99e-07 | 3190.8ms/step | 18.8 samples/s +Step 49130/50462 | Loss: 2.6496 | LR: 8.85e-07 | 3120.6ms/step | 19.2 samples/s +Step 49140/50462 | Loss: 2.6220 | LR: 8.72e-07 | 3124.7ms/step | 19.2 samples/s +Step 49150/50462 | Loss: 2.6525 | LR: 8.59e-07 | 3130.8ms/step | 19.2 samples/s +Step 49160/50462 | Loss: 2.6446 | LR: 8.46e-07 | 3139.6ms/step | 19.1 samples/s +Step 49170/50462 | Loss: 2.6399 | LR: 8.33e-07 | 3133.1ms/step | 19.2 samples/s +Step 49180/50462 | Loss: 2.6310 | LR: 8.20e-07 | 3135.4ms/step | 19.1 samples/s +Step 49190/50462 | Loss: 2.6192 | LR: 8.07e-07 | 3131.4ms/step | 19.2 samples/s +Step 49200/50462 | Loss: 2.6753 | LR: 7.95e-07 | 3246.2ms/step | 18.5 samples/s +Step 49210/50462 | Loss: 2.6427 | LR: 7.82e-07 | 3128.1ms/step | 19.2 samples/s +Step 49220/50462 | Loss: 2.6532 | LR: 7.70e-07 | 3142.5ms/step | 19.1 samples/s +Step 49230/50462 | Loss: 2.6531 | LR: 7.57e-07 | 3142.5ms/step | 19.1 samples/s +Step 49240/50462 | Loss: 2.6469 | LR: 7.45e-07 | 3141.4ms/step | 19.1 samples/s +Step 49250/50462 | Loss: 2.6385 | LR: 7.33e-07 | 3138.3ms/step | 19.1 samples/s +Validation at step 49250: Loss = 2.6523, Perplexity = 14.19 +Step 49260/50462 | Loss: 2.6136 | LR: 7.21e-07 | 4808.9ms/step | 12.5 samples/s +Step 49270/50462 | Loss: 2.6324 | LR: 7.09e-07 | 3139.8ms/step | 19.1 samples/s +Step 49280/50462 | Loss: 2.6143 | LR: 6.97e-07 | 3117.8ms/step | 19.2 samples/s +Step 49290/50462 | Loss: 2.6369 | LR: 6.85e-07 | 3127.9ms/step | 19.2 samples/s +Step 49300/50462 | Loss: 2.6367 | LR: 6.74e-07 | 3126.8ms/step | 19.2 samples/s +Step 49310/50462 | Loss: 2.6406 | LR: 6.62e-07 | 3133.9ms/step | 19.1 samples/s +Step 49320/50462 | Loss: 2.6290 | LR: 6.51e-07 | 3130.7ms/step | 19.2 samples/s +Step 49330/50462 | Loss: 2.6392 | LR: 6.39e-07 | 3134.2ms/step | 19.1 samples/s +Step 49340/50462 | Loss: 2.6482 | LR: 6.28e-07 | 3132.1ms/step | 19.2 samples/s +Step 49350/50462 | Loss: 2.6516 | LR: 6.17e-07 | 3130.4ms/step | 19.2 samples/s +Step 49360/50462 | Loss: 2.6623 | LR: 6.06e-07 | 3123.2ms/step | 19.2 samples/s +Step 49370/50462 | Loss: 2.6421 | LR: 5.95e-07 | 3129.0ms/step | 19.2 samples/s +Step 49380/50462 | Loss: 2.6582 | LR: 5.84e-07 | 3132.6ms/step | 19.2 samples/s +Step 49390/50462 | Loss: 2.6546 | LR: 5.73e-07 | 3135.6ms/step | 19.1 samples/s +Step 49400/50462 | Loss: 2.6524 | LR: 5.63e-07 | 3140.0ms/step | 19.1 samples/s +Step 49410/50462 | Loss: 2.6462 | LR: 5.52e-07 | 3132.6ms/step | 19.2 samples/s +Step 49420/50462 | Loss: 2.6431 | LR: 5.42e-07 | 3134.7ms/step | 19.1 samples/s +Step 49430/50462 | Loss: 2.6236 | LR: 5.31e-07 | 3126.1ms/step | 19.2 samples/s +Step 49440/50462 | Loss: 2.6314 | LR: 5.21e-07 | 3191.9ms/step | 18.8 samples/s +Step 49450/50462 | Loss: 2.6196 | LR: 5.11e-07 | 3124.8ms/step | 19.2 samples/s +Step 49460/50462 | Loss: 2.6511 | LR: 5.01e-07 | 3121.1ms/step | 19.2 samples/s +Step 49470/50462 | Loss: 2.6542 | LR: 4.91e-07 | 3123.7ms/step | 19.2 samples/s +Step 49480/50462 | Loss: 2.6438 | LR: 4.81e-07 | 3133.1ms/step | 19.2 samples/s +Step 49490/50462 | Loss: 2.6287 | LR: 4.72e-07 | 3124.8ms/step | 19.2 samples/s +Step 49500/50462 | Loss: 2.6497 | LR: 4.62e-07 | 3128.2ms/step | 19.2 samples/s +Validation at step 49500: Loss = 2.6523, Perplexity = 14.19 +Step 49510/50462 | Loss: 2.6558 | LR: 4.52e-07 | 4809.3ms/step | 12.5 samples/s +Step 49520/50462 | Loss: 2.6382 | LR: 4.43e-07 | 3193.1ms/step | 18.8 samples/s +Step 49530/50462 | Loss: 2.6433 | LR: 4.34e-07 | 3184.2ms/step | 18.8 samples/s +Step 49540/50462 | Loss: 2.6334 | LR: 4.24e-07 | 3138.4ms/step | 19.1 samples/s +Step 49550/50462 | Loss: 2.6429 | LR: 4.15e-07 | 3145.6ms/step | 19.1 samples/s +Step 49560/50462 | Loss: 2.6582 | LR: 4.06e-07 | 3142.4ms/step | 19.1 samples/s +Step 49570/50462 | Loss: 2.6565 | LR: 3.97e-07 | 3119.7ms/step | 19.2 samples/s +Step 49580/50462 | Loss: 2.6552 | LR: 3.88e-07 | 3130.3ms/step | 19.2 samples/s +Step 49590/50462 | Loss: 2.6288 | LR: 3.79e-07 | 3133.1ms/step | 19.2 samples/s +Step 49600/50462 | Loss: 2.6643 | LR: 3.71e-07 | 3138.8ms/step | 19.1 samples/s +Step 49610/50462 | Loss: 2.6421 | LR: 3.62e-07 | 3139.4ms/step | 19.1 samples/s +Step 49620/50462 | Loss: 2.6415 | LR: 3.54e-07 | 3122.1ms/step | 19.2 samples/s +Step 49630/50462 | Loss: 2.6422 | LR: 3.45e-07 | 3128.4ms/step | 19.2 samples/s +Step 49640/50462 | Loss: 2.6431 | LR: 3.37e-07 | 3129.2ms/step | 19.2 samples/s +Step 49650/50462 | Loss: 2.6487 | LR: 3.29e-07 | 3140.3ms/step | 19.1 samples/s +Step 49660/50462 | Loss: 2.6300 | LR: 3.21e-07 | 3132.0ms/step | 19.2 samples/s +Step 49670/50462 | Loss: 2.6176 | LR: 3.13e-07 | 3129.7ms/step | 19.2 samples/s +Step 49680/50462 | Loss: 2.6613 | LR: 3.05e-07 | 3133.9ms/step | 19.1 samples/s +Step 49690/50462 | Loss: 2.6764 | LR: 2.97e-07 | 3136.0ms/step | 19.1 samples/s +Step 49700/50462 | Loss: 2.6506 | LR: 2.90e-07 | 3137.4ms/step | 19.1 samples/s +Step 49710/50462 | Loss: 2.6473 | LR: 2.82e-07 | 3128.4ms/step | 19.2 samples/s +Step 49720/50462 | Loss: 2.6337 | LR: 2.75e-07 | 3117.8ms/step | 19.2 samples/s +Step 49730/50462 | Loss: 2.6492 | LR: 2.67e-07 | 3131.5ms/step | 19.2 samples/s +Step 49740/50462 | Loss: 2.6723 | LR: 2.60e-07 | 3136.6ms/step | 19.1 samples/s +Step 49750/50462 | Loss: 2.6420 | LR: 2.53e-07 | 3130.2ms/step | 19.2 samples/s +Validation at step 49750: Loss = 2.6523, Perplexity = 14.19 +Step 49760/50462 | Loss: 2.6594 | LR: 2.46e-07 | 4862.1ms/step | 12.3 samples/s +Step 49770/50462 | Loss: 2.6490 | LR: 2.39e-07 | 3126.2ms/step | 19.2 samples/s +Step 49780/50462 | Loss: 2.6339 | LR: 2.32e-07 | 3132.9ms/step | 19.2 samples/s +Step 49790/50462 | Loss: 2.6268 | LR: 2.25e-07 | 3140.6ms/step | 19.1 samples/s +Step 49800/50462 | Loss: 2.6680 | LR: 2.19e-07 | 3140.4ms/step | 19.1 samples/s +Step 49810/50462 | Loss: 2.6596 | LR: 2.12e-07 | 3128.4ms/step | 19.2 samples/s +Step 49820/50462 | Loss: 2.6532 | LR: 2.06e-07 | 3121.5ms/step | 19.2 samples/s +Step 49830/50462 | Loss: 2.6438 | LR: 1.99e-07 | 3128.7ms/step | 19.2 samples/s +Step 49840/50462 | Loss: 2.6568 | LR: 1.93e-07 | 3189.1ms/step | 18.8 samples/s +Step 49850/50462 | Loss: 2.6366 | LR: 1.87e-07 | 3191.5ms/step | 18.8 samples/s +Step 49860/50462 | Loss: 2.6358 | LR: 1.81e-07 | 3132.2ms/step | 19.2 samples/s +Step 49870/50462 | Loss: 2.6514 | LR: 1.75e-07 | 3140.8ms/step | 19.1 samples/s +Step 49880/50462 | Loss: 2.6378 | LR: 1.69e-07 | 3125.5ms/step | 19.2 samples/s +Step 49890/50462 | Loss: 2.6452 | LR: 1.63e-07 | 3126.7ms/step | 19.2 samples/s +Step 49900/50462 | Loss: 2.6582 | LR: 1.58e-07 | 3128.8ms/step | 19.2 samples/s +Step 49910/50462 | Loss: 2.6448 | LR: 1.52e-07 | 3122.8ms/step | 19.2 samples/s +Step 49920/50462 | Loss: 2.6564 | LR: 1.47e-07 | 3118.7ms/step | 19.2 samples/s +Step 49930/50462 | Loss: 2.6286 | LR: 1.41e-07 | 3132.0ms/step | 19.2 samples/s +Step 49940/50462 | Loss: 2.6390 | LR: 1.36e-07 | 3133.6ms/step | 19.1 samples/s +Step 49950/50462 | Loss: 2.6016 | LR: 1.31e-07 | 3132.1ms/step | 19.2 samples/s +Step 49960/50462 | Loss: 2.6337 | LR: 1.26e-07 | 3137.0ms/step | 19.1 samples/s +Step 49970/50462 | Loss: 2.6243 | LR: 1.21e-07 | 3133.5ms/step | 19.1 samples/s +Step 49980/50462 | Loss: 2.6393 | LR: 1.16e-07 | 3128.7ms/step | 19.2 samples/s +Step 49990/50462 | Loss: 2.6154 | LR: 1.11e-07 | 3134.8ms/step | 19.1 samples/s +Step 50000/50462 | Loss: 2.6229 | LR: 1.07e-07 | 3131.3ms/step | 19.2 samples/s +Validation at step 50000: Loss = 2.6523, Perplexity = 14.19 +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-50000 +Step 50010/50462 | Loss: 2.6196 | LR: 1.02e-07 | 5684.8ms/step | 10.6 samples/s +Step 50020/50462 | Loss: 2.6302 | LR: 9.75e-08 | 3133.6ms/step | 19.1 samples/s +Step 50030/50462 | Loss: 2.6589 | LR: 9.32e-08 | 3130.3ms/step | 19.2 samples/s +Step 50040/50462 | Loss: 2.6888 | LR: 8.89e-08 | 3136.8ms/step | 19.1 samples/s +Step 50050/50462 | Loss: 2.6419 | LR: 8.47e-08 | 3133.8ms/step | 19.1 samples/s +Step 50060/50462 | Loss: 2.6373 | LR: 8.07e-08 | 3127.6ms/step | 19.2 samples/s +Step 50070/50462 | Loss: 2.6557 | LR: 7.67e-08 | 3122.9ms/step | 19.2 samples/s +Step 50080/50462 | Loss: 2.6276 | LR: 7.28e-08 | 3186.8ms/step | 18.8 samples/s +Step 50090/50462 | Loss: 2.6211 | LR: 6.91e-08 | 3138.0ms/step | 19.1 samples/s +Step 50100/50462 | Loss: 2.6282 | LR: 6.54e-08 | 3124.0ms/step | 19.2 samples/s +Step 50110/50462 | Loss: 2.6667 | LR: 6.19e-08 | 3130.8ms/step | 19.2 samples/s +Step 50120/50462 | Loss: 2.6510 | LR: 5.84e-08 | 3138.7ms/step | 19.1 samples/s +Step 50130/50462 | Loss: 2.6420 | LR: 5.50e-08 | 3131.9ms/step | 19.2 samples/s +Step 50140/50462 | Loss: 2.6642 | LR: 5.18e-08 | 3132.6ms/step | 19.2 samples/s +Step 50150/50462 | Loss: 2.6204 | LR: 4.86e-08 | 3134.7ms/step | 19.1 samples/s +Step 50160/50462 | Loss: 2.6376 | LR: 4.55e-08 | 3195.8ms/step | 18.8 samples/s +Step 50170/50462 | Loss: 2.6048 | LR: 4.26e-08 | 3179.8ms/step | 18.9 samples/s +Step 50180/50462 | Loss: 2.6089 | LR: 3.97e-08 | 3126.2ms/step | 19.2 samples/s +Step 50190/50462 | Loss: 2.6590 | LR: 3.69e-08 | 3126.9ms/step | 19.2 samples/s +Step 50200/50462 | Loss: 2.6269 | LR: 3.43e-08 | 3137.4ms/step | 19.1 samples/s +Step 50210/50462 | Loss: 2.6562 | LR: 3.17e-08 | 3148.9ms/step | 19.1 samples/s +Step 50220/50462 | Loss: 2.6462 | LR: 2.92e-08 | 3140.9ms/step | 19.1 samples/s +Step 50230/50462 | Loss: 2.6657 | LR: 2.69e-08 | 3140.3ms/step | 19.1 samples/s +Step 50240/50462 | Loss: 2.6311 | LR: 2.46e-08 | 3129.2ms/step | 19.2 samples/s +Step 50250/50462 | Loss: 2.6219 | LR: 2.24e-08 | 3126.1ms/step | 19.2 samples/s +Validation at step 50250: Loss = 2.6523, Perplexity = 14.19 +Step 50260/50462 | Loss: 2.6716 | LR: 2.04e-08 | 4799.8ms/step | 12.5 samples/s +Step 50270/50462 | Loss: 2.6808 | LR: 1.84e-08 | 3129.7ms/step | 19.2 samples/s +Step 50280/50462 | Loss: 2.6529 | LR: 1.65e-08 | 3142.4ms/step | 19.1 samples/s +Step 50290/50462 | Loss: 2.6686 | LR: 1.48e-08 | 3138.6ms/step | 19.1 samples/s +Step 50300/50462 | Loss: 2.6209 | LR: 1.31e-08 | 3142.0ms/step | 19.1 samples/s +Step 50310/50462 | Loss: 2.6156 | LR: 1.15e-08 | 3129.9ms/step | 19.2 samples/s +Step 50320/50462 | Loss: 2.6395 | LR: 1.01e-08 | 3130.9ms/step | 19.2 samples/s +Step 50330/50462 | Loss: 2.6627 | LR: 8.70e-09 | 3132.4ms/step | 19.2 samples/s +Step 50340/50462 | Loss: 2.6483 | LR: 7.43e-09 | 3140.9ms/step | 19.1 samples/s +Step 50350/50462 | Loss: 2.6276 | LR: 6.26e-09 | 3136.2ms/step | 19.1 samples/s +Step 50360/50462 | Loss: 2.6498 | LR: 5.19e-09 | 3129.6ms/step | 19.2 samples/s +Step 50370/50462 | Loss: 2.6263 | LR: 4.23e-09 | 3133.8ms/step | 19.1 samples/s +Step 50380/50462 | Loss: 2.6210 | LR: 3.36e-09 | 3125.8ms/step | 19.2 samples/s +Step 50390/50462 | Loss: 2.6584 | LR: 2.59e-09 | 3136.7ms/step | 19.1 samples/s +Step 50400/50462 | Loss: 2.6425 | LR: 1.92e-09 | 3197.8ms/step | 18.8 samples/s +Step 50410/50462 | Loss: 2.6343 | LR: 1.35e-09 | 3127.8ms/step | 19.2 samples/s +Step 50420/50462 | Loss: 2.6569 | LR: 8.81e-10 | 3129.9ms/step | 19.2 samples/s +Step 50430/50462 | Loss: 2.6486 | LR: 5.11e-10 | 3132.9ms/step | 19.2 samples/s +Step 50440/50462 | Loss: 2.6173 | LR: 2.42e-10 | 3138.6ms/step | 19.1 samples/s +Step 50450/50462 | Loss: 2.6578 | LR: 7.19e-11 | 3141.1ms/step | 19.1 samples/s +Step 50460/50462 | Loss: 2.6329 | LR: 2.00e-12 | 3130.1ms/step | 19.2 samples/s +Saved checkpoint to checkpoints/run_20260616_011729/checkpoint-50462 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading history steps 1046-1046, summary, console lines 1100-1100 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▂▂▂▃▂▂▃▂▂▂▃▂▃▂█▂▂▂▃▃▂▂▁▁▃▂▂▂▃▂▂▂▂▃▅▃▄▂▄▄ +wandb: speed/samples_per_sec ████████▁███████████████████████████████ +wandb: speed/steps_per_sec ▇▇▇▇▇████▇██▇▇█▇▁▆▇▇▇▇▇▇████▇█▆▁▆▆▇▇▁▇▂▇ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.65233 +wandb: eval/perplexity 14.18704 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 3130.08585 +wandb: speed/samples_per_sec 19.1688 +wandb: speed/steps_per_sec 0.31948 +wandb: +4 ... +wandb: +wandb: 🚀 View run sparse 1mask at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/apvt4901 +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260616_012002-apvt4901/logs +Training complete! diff --git a/epfl_final/log_files/sparse_pretrain_64%.log b/epfl_final/log_files/sparse_pretrain_64%.log new file mode 100644 index 0000000000000000000000000000000000000000..e93c7eb6a3228b1810abc8d323bbdb3a46c40418 --- /dev/null +++ b/epfl_final/log_files/sparse_pretrain_64%.log @@ -0,0 +1,1442 @@ +nohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +Resuming: Using existing output directory checkpoints/run_20260609_063834 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "checkpoints/run_20260609_063834", + "resume_from_checkpoint": "checkpoints/run_20260609_063834/checkpoint-39000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "sparse 64%", + "wandb_tags": [ + "qwen3", + "1.2b", + "sparse" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 22, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "mask_cache_23", + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Resuming from checkpoint: checkpoints/run_20260609_063834/checkpoint-39000 +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 22 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Using pre-tokenized dataset (no runtime tokenization) + +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +Loading checkpoint from checkpoints/run_20260609_063834/checkpoint-39000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 39000 (epoch 0, micro_step 390000) +Resuming training from step 39000 +Resuming wandb run jh4dfz9a +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run jh4dfz9a +wandb: Tracking run with wandb version 0.27.2 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260612_172414-jh4dfz9a +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run sparse 64% +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/jh4dfz9a +Starting training... +Validation at step 39000: Loss = 2.5565, Perplexity = 12.89 +wandb: WARNING Tried to log to step 39000 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 39001: sparse_stochastic (configured sparse attention) +Step 39010/50462 | Loss: 2.5751 | LR: 6.27e-05 | 69227.7ms/step | 0.9 samples/s +wandb: WARNING Tried to log to step 39010 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39020/50462 | Loss: 2.5254 | LR: 6.26e-05 | 2990.5ms/step | 20.1 samples/s +wandb: WARNING Tried to log to step 39020 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39030/50462 | Loss: 2.5528 | LR: 6.25e-05 | 2990.4ms/step | 20.1 samples/s +wandb: WARNING Tried to log to step 39030 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39040/50462 | Loss: 2.5419 | LR: 6.23e-05 | 2989.6ms/step | 20.1 samples/s +wandb: WARNING Tried to log to step 39040 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39050/50462 | Loss: 2.5181 | LR: 6.22e-05 | 2989.8ms/step | 20.1 samples/s +wandb: WARNING Tried to log to step 39050 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39060/50462 | Loss: 2.5302 | LR: 6.21e-05 | 2995.8ms/step | 20.0 samples/s +wandb: WARNING Tried to log to step 39060 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39070/50462 | Loss: 2.5524 | LR: 6.20e-05 | 3000.0ms/step | 20.0 samples/s +wandb: WARNING Tried to log to step 39070 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39080/50462 | Loss: 2.5686 | LR: 6.19e-05 | 2996.9ms/step | 20.0 samples/s +wandb: WARNING Tried to log to step 39080 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39090/50462 | Loss: 2.5252 | LR: 6.18e-05 | 2993.5ms/step | 20.0 samples/s +wandb: WARNING Tried to log to step 39090 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39100/50462 | Loss: 2.5487 | LR: 6.17e-05 | 2997.6ms/step | 20.0 samples/s +wandb: WARNING Tried to log to step 39100 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39110/50462 | Loss: 2.5217 | LR: 6.16e-05 | 2996.5ms/step | 20.0 samples/s +wandb: WARNING Tried to log to step 39110 that is less than the current step 39111. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 39120/50462 | Loss: 2.5284 | LR: 6.15e-05 | 2996.0ms/step | 20.0 samples/s +Step 39130/50462 | Loss: 2.5663 | LR: 6.14e-05 | 2995.9ms/step | 20.0 samples/s +Step 39140/50462 | Loss: 2.5391 | LR: 6.13e-05 | 2995.2ms/step | 20.0 samples/s +Step 39150/50462 | Loss: 2.5429 | LR: 6.12e-05 | 2995.5ms/step | 20.0 samples/s +Step 39160/50462 | Loss: 2.5303 | LR: 6.11e-05 | 2993.4ms/step | 20.0 samples/s +Step 39170/50462 | Loss: 2.5573 | LR: 6.10e-05 | 2995.8ms/step | 20.0 samples/s +Step 39180/50462 | Loss: 2.5674 | LR: 6.09e-05 | 2994.4ms/step | 20.0 samples/s +Step 39190/50462 | Loss: 2.5520 | LR: 6.08e-05 | 2997.5ms/step | 20.0 samples/s +Step 39200/50462 | Loss: 2.5202 | LR: 6.07e-05 | 2996.4ms/step | 20.0 samples/s +Step 39210/50462 | Loss: 2.5425 | LR: 6.06e-05 | 2995.9ms/step | 20.0 samples/s +Step 39220/50462 | Loss: 2.5161 | LR: 6.05e-05 | 2998.4ms/step | 20.0 samples/s +Step 39230/50462 | Loss: 2.5613 | LR: 6.04e-05 | 2996.1ms/step | 20.0 samples/s +Step 39240/50462 | Loss: 2.5403 | LR: 6.03e-05 | 2993.5ms/step | 20.0 samples/s +Step 39250/50462 | Loss: 2.5608 | LR: 6.02e-05 | 2996.1ms/step | 20.0 samples/s +Validation at step 39250: Loss = 2.5562, Perplexity = 12.89 +Step 39260/50462 | Loss: 2.5445 | LR: 6.01e-05 | 4592.2ms/step | 13.1 samples/s +Step 39270/50462 | Loss: 2.5458 | LR: 6.00e-05 | 2992.1ms/step | 20.1 samples/s +Step 39280/50462 | Loss: 2.5598 | LR: 5.99e-05 | 2992.0ms/step | 20.1 samples/s +Step 39290/50462 | Loss: 2.5394 | LR: 5.98e-05 | 2995.8ms/step | 20.0 samples/s +Step 39300/50462 | Loss: 2.5580 | LR: 5.97e-05 | 2993.1ms/step | 20.0 samples/s +Step 39310/50462 | Loss: 2.5464 | LR: 5.96e-05 | 2992.9ms/step | 20.0 samples/s +Step 39320/50462 | Loss: 2.4999 | LR: 5.95e-05 | 2992.6ms/step | 20.0 samples/s +Step 39330/50462 | Loss: 2.5461 | LR: 5.94e-05 | 2999.1ms/step | 20.0 samples/s +Step 39340/50462 | Loss: 2.5502 | LR: 5.93e-05 | 2993.8ms/step | 20.0 samples/s +Step 39350/50462 | Loss: 2.5446 | LR: 5.91e-05 | 2994.3ms/step | 20.0 samples/s +Step 39360/50462 | Loss: 2.5185 | LR: 5.90e-05 | 2996.5ms/step | 20.0 samples/s +Step 39370/50462 | Loss: 2.5470 | LR: 5.89e-05 | 2995.1ms/step | 20.0 samples/s +Step 39380/50462 | Loss: 2.5346 | LR: 5.88e-05 | 2997.2ms/step | 20.0 samples/s +Step 39390/50462 | Loss: 2.5485 | LR: 5.87e-05 | 2995.0ms/step | 20.0 samples/s +Step 39400/50462 | Loss: 2.5578 | LR: 5.86e-05 | 2992.6ms/step | 20.0 samples/s +Step 39410/50462 | Loss: 2.5073 | LR: 5.85e-05 | 2996.2ms/step | 20.0 samples/s +Step 39420/50462 | Loss: 2.5264 | LR: 5.84e-05 | 2998.1ms/step | 20.0 samples/s +Step 39430/50462 | Loss: 2.5231 | LR: 5.83e-05 | 2997.9ms/step | 20.0 samples/s +Step 39440/50462 | Loss: 2.5430 | LR: 5.82e-05 | 2994.9ms/step | 20.0 samples/s +Step 39450/50462 | Loss: 2.5328 | LR: 5.81e-05 | 2995.6ms/step | 20.0 samples/s +Step 39460/50462 | Loss: 2.5467 | LR: 5.80e-05 | 2997.4ms/step | 20.0 samples/s +Step 39470/50462 | Loss: 2.5815 | LR: 5.79e-05 | 2999.9ms/step | 20.0 samples/s +Step 39480/50462 | Loss: 2.5664 | LR: 5.78e-05 | 2992.3ms/step | 20.1 samples/s +Step 39490/50462 | Loss: 2.5363 | LR: 5.77e-05 | 2995.6ms/step | 20.0 samples/s +Step 39500/50462 | Loss: 2.5502 | LR: 5.76e-05 | 2994.6ms/step | 20.0 samples/s +Validation at step 39500: Loss = 2.5553, Perplexity = 12.88 +Step 39510/50462 | Loss: 2.5479 | LR: 5.75e-05 | 4595.5ms/step | 13.1 samples/s +Step 39520/50462 | Loss: 2.5591 | LR: 5.74e-05 | 2994.6ms/step | 20.0 samples/s +Step 39530/50462 | Loss: 2.5463 | LR: 5.73e-05 | 2995.8ms/step | 20.0 samples/s +Step 39540/50462 | Loss: 2.5464 | LR: 5.72e-05 | 2996.0ms/step | 20.0 samples/s +Step 39550/50462 | Loss: 2.5498 | LR: 5.71e-05 | 2995.7ms/step | 20.0 samples/s +Step 39560/50462 | Loss: 2.5390 | LR: 5.70e-05 | 2999.5ms/step | 20.0 samples/s +Step 39570/50462 | Loss: 2.5503 | LR: 5.69e-05 | 2997.2ms/step | 20.0 samples/s +Step 39580/50462 | Loss: 2.5316 | LR: 5.68e-05 | 2997.6ms/step | 20.0 samples/s +Step 39590/50462 | Loss: 2.5555 | LR: 5.67e-05 | 2998.1ms/step | 20.0 samples/s +Step 39600/50462 | Loss: 2.5391 | LR: 5.66e-05 | 3583.8ms/step | 16.7 samples/s +Step 39610/50462 | Loss: 2.5316 | LR: 5.65e-05 | 3654.3ms/step | 16.4 samples/s +Step 39620/50462 | Loss: 2.5260 | LR: 5.64e-05 | 2991.7ms/step | 20.1 samples/s +Step 39630/50462 | Loss: 2.5168 | LR: 5.63e-05 | 2994.8ms/step | 20.0 samples/s +Step 39640/50462 | Loss: 2.5715 | LR: 5.62e-05 | 2999.7ms/step | 20.0 samples/s +Step 39650/50462 | Loss: 2.5465 | LR: 5.61e-05 | 2998.2ms/step | 20.0 samples/s +Step 39660/50462 | Loss: 2.5464 | LR: 5.60e-05 | 2995.8ms/step | 20.0 samples/s +Step 39670/50462 | Loss: 2.5459 | LR: 5.59e-05 | 2994.7ms/step | 20.0 samples/s +Step 39680/50462 | Loss: 2.5548 | LR: 5.58e-05 | 2994.2ms/step | 20.0 samples/s +Step 39690/50462 | Loss: 2.5408 | LR: 5.57e-05 | 3584.2ms/step | 16.7 samples/s +Step 39700/50462 | Loss: 2.5370 | LR: 5.56e-05 | 2993.5ms/step | 20.0 samples/s +Step 39710/50462 | Loss: 2.5269 | LR: 5.55e-05 | 2999.7ms/step | 20.0 samples/s +Step 39720/50462 | Loss: 2.5184 | LR: 5.54e-05 | 2995.7ms/step | 20.0 samples/s +Step 39730/50462 | Loss: 2.5206 | LR: 5.53e-05 | 2998.5ms/step | 20.0 samples/s +Step 39740/50462 | Loss: 2.5410 | LR: 5.52e-05 | 3000.0ms/step | 20.0 samples/s +Step 39750/50462 | Loss: 2.5507 | LR: 5.51e-05 | 3000.2ms/step | 20.0 samples/s +Validation at step 39750: Loss = 2.5551, Perplexity = 12.87 +Step 39760/50462 | Loss: 2.5401 | LR: 5.50e-05 | 4595.5ms/step | 13.1 samples/s +Step 39770/50462 | Loss: 2.5626 | LR: 5.49e-05 | 2996.6ms/step | 20.0 samples/s +Step 39780/50462 | Loss: 2.5626 | LR: 5.48e-05 | 2999.7ms/step | 20.0 samples/s +Step 39790/50462 | Loss: 2.5398 | LR: 5.47e-05 | 3000.2ms/step | 20.0 samples/s +Step 39800/50462 | Loss: 2.5328 | LR: 5.46e-05 | 2999.9ms/step | 20.0 samples/s +Step 39810/50462 | Loss: 2.5465 | LR: 5.45e-05 | 2999.8ms/step | 20.0 samples/s +Step 39820/50462 | Loss: 2.5667 | LR: 5.44e-05 | 3000.3ms/step | 20.0 samples/s +Step 39830/50462 | Loss: 2.5386 | LR: 5.43e-05 | 2999.9ms/step | 20.0 samples/s +Step 39840/50462 | Loss: 2.5491 | LR: 5.42e-05 | 3000.0ms/step | 20.0 samples/s +Step 39850/50462 | Loss: 2.5559 | LR: 5.41e-05 | 3000.6ms/step | 20.0 samples/s +Step 39860/50462 | Loss: 2.5189 | LR: 5.40e-05 | 2995.3ms/step | 20.0 samples/s +Step 39870/50462 | Loss: 2.5444 | LR: 5.39e-05 | 3000.3ms/step | 20.0 samples/s +Step 39880/50462 | Loss: 2.5443 | LR: 5.38e-05 | 2996.4ms/step | 20.0 samples/s +Step 39890/50462 | Loss: 2.5536 | LR: 5.38e-05 | 2997.5ms/step | 20.0 samples/s +Step 39900/50462 | Loss: 2.5359 | LR: 5.37e-05 | 2998.2ms/step | 20.0 samples/s +Step 39910/50462 | Loss: 2.5423 | LR: 5.36e-05 | 2998.2ms/step | 20.0 samples/s +Step 39920/50462 | Loss: 2.5422 | LR: 5.35e-05 | 2997.5ms/step | 20.0 samples/s +Step 39930/50462 | Loss: 2.5463 | LR: 5.34e-05 | 3000.1ms/step | 20.0 samples/s +Step 39940/50462 | Loss: 2.5366 | LR: 5.33e-05 | 3000.5ms/step | 20.0 samples/s +Step 39950/50462 | Loss: 2.5269 | LR: 5.32e-05 | 2999.8ms/step | 20.0 samples/s +Step 39960/50462 | Loss: 2.5446 | LR: 5.31e-05 | 3000.2ms/step | 20.0 samples/s +Step 39970/50462 | Loss: 2.5673 | LR: 5.30e-05 | 2995.9ms/step | 20.0 samples/s +Step 39980/50462 | Loss: 2.5446 | LR: 5.29e-05 | 3000.4ms/step | 20.0 samples/s +Step 39990/50462 | Loss: 2.5791 | LR: 5.28e-05 | 2999.5ms/step | 20.0 samples/s +Step 40000/50462 | Loss: 2.5167 | LR: 5.27e-05 | 3000.3ms/step | 20.0 samples/s +Validation at step 40000: Loss = 2.5550, Perplexity = 12.87 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-40000 +Step 40010/50462 | Loss: 2.5602 | LR: 5.26e-05 | 5615.9ms/step | 10.7 samples/s +Step 40020/50462 | Loss: 2.5821 | LR: 5.25e-05 | 2995.9ms/step | 20.0 samples/s +Step 40030/50462 | Loss: 2.5278 | LR: 5.24e-05 | 2998.5ms/step | 20.0 samples/s +Step 40040/50462 | Loss: 2.5587 | LR: 5.23e-05 | 2999.0ms/step | 20.0 samples/s +Step 40050/50462 | Loss: 2.5460 | LR: 5.22e-05 | 3000.7ms/step | 20.0 samples/s +Step 40060/50462 | Loss: 2.5399 | LR: 5.21e-05 | 2999.4ms/step | 20.0 samples/s +Step 40070/50462 | Loss: 2.5506 | LR: 5.20e-05 | 3000.4ms/step | 20.0 samples/s +Step 40080/50462 | Loss: 2.5255 | LR: 5.19e-05 | 2998.5ms/step | 20.0 samples/s +Step 40090/50462 | Loss: 2.5585 | LR: 5.18e-05 | 2999.7ms/step | 20.0 samples/s +Step 40100/50462 | Loss: 2.5554 | LR: 5.17e-05 | 2995.0ms/step | 20.0 samples/s +Step 40110/50462 | Loss: 2.5421 | LR: 5.16e-05 | 2994.8ms/step | 20.0 samples/s +Step 40120/50462 | Loss: 2.5564 | LR: 5.15e-05 | 3000.1ms/step | 20.0 samples/s +Step 40130/50462 | Loss: 2.5390 | LR: 5.14e-05 | 3000.3ms/step | 20.0 samples/s +Step 40140/50462 | Loss: 2.5353 | LR: 5.13e-05 | 2999.4ms/step | 20.0 samples/s +Step 40150/50462 | Loss: 2.5135 | LR: 5.12e-05 | 3000.8ms/step | 20.0 samples/s +Step 40160/50462 | Loss: 2.5109 | LR: 5.11e-05 | 3001.8ms/step | 20.0 samples/s +Step 40170/50462 | Loss: 2.5495 | LR: 5.10e-05 | 2998.1ms/step | 20.0 samples/s +Step 40180/50462 | Loss: 2.5517 | LR: 5.09e-05 | 2997.9ms/step | 20.0 samples/s +Step 40190/50462 | Loss: 2.5482 | LR: 5.09e-05 | 2998.1ms/step | 20.0 samples/s +Step 40200/50462 | Loss: 2.5573 | LR: 5.08e-05 | 2999.7ms/step | 20.0 samples/s +Step 40210/50462 | Loss: 2.5622 | LR: 5.07e-05 | 2996.6ms/step | 20.0 samples/s +Step 40220/50462 | Loss: 2.5620 | LR: 5.06e-05 | 2999.7ms/step | 20.0 samples/s +Step 40230/50462 | Loss: 2.5539 | LR: 5.05e-05 | 2999.9ms/step | 20.0 samples/s +Step 40240/50462 | Loss: 2.5841 | LR: 5.04e-05 | 3000.2ms/step | 20.0 samples/s +Step 40250/50462 | Loss: 2.5438 | LR: 5.03e-05 | 3000.7ms/step | 20.0 samples/s +Validation at step 40250: Loss = 2.5549, Perplexity = 12.87 +Step 40260/50462 | Loss: 2.5495 | LR: 5.02e-05 | 4597.6ms/step | 13.1 samples/s +Step 40270/50462 | Loss: 2.5244 | LR: 5.01e-05 | 2999.9ms/step | 20.0 samples/s +Step 40280/50462 | Loss: 2.5483 | LR: 5.00e-05 | 2999.8ms/step | 20.0 samples/s +Step 40290/50462 | Loss: 2.5457 | LR: 4.99e-05 | 2999.9ms/step | 20.0 samples/s +Step 40300/50462 | Loss: 2.5417 | LR: 4.98e-05 | 2995.4ms/step | 20.0 samples/s +Step 40310/50462 | Loss: 2.5278 | LR: 4.97e-05 | 2994.6ms/step | 20.0 samples/s +Step 40320/50462 | Loss: 2.5463 | LR: 4.96e-05 | 2999.7ms/step | 20.0 samples/s +Step 40330/50462 | Loss: 2.5686 | LR: 4.95e-05 | 2998.1ms/step | 20.0 samples/s +Step 40340/50462 | Loss: 2.5288 | LR: 4.94e-05 | 2998.6ms/step | 20.0 samples/s +Step 40350/50462 | Loss: 2.5199 | LR: 4.93e-05 | 2999.3ms/step | 20.0 samples/s +Step 40360/50462 | Loss: 2.5492 | LR: 4.92e-05 | 2996.5ms/step | 20.0 samples/s +Step 40370/50462 | Loss: 2.5620 | LR: 4.91e-05 | 2998.2ms/step | 20.0 samples/s +Step 40380/50462 | Loss: 2.5479 | LR: 4.90e-05 | 3000.3ms/step | 20.0 samples/s +Step 40390/50462 | Loss: 2.5433 | LR: 4.90e-05 | 2999.4ms/step | 20.0 samples/s +Step 40400/50462 | Loss: 2.5567 | LR: 4.89e-05 | 2998.3ms/step | 20.0 samples/s +Step 40410/50462 | Loss: 2.5687 | LR: 4.88e-05 | 2999.6ms/step | 20.0 samples/s +Step 40420/50462 | Loss: 2.5685 | LR: 4.87e-05 | 3000.6ms/step | 20.0 samples/s +Step 40430/50462 | Loss: 2.5358 | LR: 4.86e-05 | 3000.0ms/step | 20.0 samples/s +Step 40440/50462 | Loss: 2.5580 | LR: 4.85e-05 | 3000.1ms/step | 20.0 samples/s +Step 40450/50462 | Loss: 2.5438 | LR: 4.84e-05 | 3000.0ms/step | 20.0 samples/s +Step 40460/50462 | Loss: 2.5764 | LR: 4.83e-05 | 3000.2ms/step | 20.0 samples/s +Step 40470/50462 | Loss: 2.5297 | LR: 4.82e-05 | 2999.6ms/step | 20.0 samples/s +Step 40480/50462 | Loss: 2.5420 | LR: 4.81e-05 | 2998.3ms/step | 20.0 samples/s +Step 40490/50462 | Loss: 2.5576 | LR: 4.80e-05 | 3001.9ms/step | 20.0 samples/s +Step 40500/50462 | Loss: 2.5478 | LR: 4.79e-05 | 2999.9ms/step | 20.0 samples/s +Validation at step 40500: Loss = 2.5547, Perplexity = 12.87 +Step 40510/50462 | Loss: 2.5462 | LR: 4.78e-05 | 4600.3ms/step | 13.0 samples/s +Step 40520/50462 | Loss: 2.5319 | LR: 4.77e-05 | 2998.1ms/step | 20.0 samples/s +Step 40530/50462 | Loss: 2.5367 | LR: 4.76e-05 | 2999.3ms/step | 20.0 samples/s +Step 40540/50462 | Loss: 2.5097 | LR: 4.76e-05 | 2998.7ms/step | 20.0 samples/s +Step 40550/50462 | Loss: 2.5639 | LR: 4.75e-05 | 3000.0ms/step | 20.0 samples/s +Step 40560/50462 | Loss: 2.5548 | LR: 4.74e-05 | 3000.0ms/step | 20.0 samples/s +Step 40570/50462 | Loss: 2.5304 | LR: 4.73e-05 | 2999.5ms/step | 20.0 samples/s +Step 40580/50462 | Loss: 2.5583 | LR: 4.72e-05 | 3000.7ms/step | 20.0 samples/s +Step 40590/50462 | Loss: 2.5418 | LR: 4.71e-05 | 3000.1ms/step | 20.0 samples/s +Step 40600/50462 | Loss: 2.5425 | LR: 4.70e-05 | 2999.7ms/step | 20.0 samples/s +Step 40610/50462 | Loss: 2.5619 | LR: 4.69e-05 | 3000.1ms/step | 20.0 samples/s +Step 40620/50462 | Loss: 2.5186 | LR: 4.68e-05 | 3004.1ms/step | 20.0 samples/s +Step 40630/50462 | Loss: 2.5394 | LR: 4.67e-05 | 2999.7ms/step | 20.0 samples/s +Step 40640/50462 | Loss: 2.5496 | LR: 4.66e-05 | 3000.4ms/step | 20.0 samples/s +Step 40650/50462 | Loss: 2.5498 | LR: 4.65e-05 | 2999.9ms/step | 20.0 samples/s +Step 40660/50462 | Loss: 2.5629 | LR: 4.64e-05 | 3000.2ms/step | 20.0 samples/s +Step 40670/50462 | Loss: 2.5379 | LR: 4.64e-05 | 2999.8ms/step | 20.0 samples/s +Step 40680/50462 | Loss: 2.5434 | LR: 4.63e-05 | 3000.3ms/step | 20.0 samples/s +Step 40690/50462 | Loss: 2.5404 | LR: 4.62e-05 | 2998.0ms/step | 20.0 samples/s +Step 40700/50462 | Loss: 2.5121 | LR: 4.61e-05 | 3000.0ms/step | 20.0 samples/s +Step 40710/50462 | Loss: 2.5525 | LR: 4.60e-05 | 3000.0ms/step | 20.0 samples/s +Step 40720/50462 | Loss: 2.5181 | LR: 4.59e-05 | 3000.5ms/step | 20.0 samples/s +Step 40730/50462 | Loss: 2.5299 | LR: 4.58e-05 | 2997.4ms/step | 20.0 samples/s +Step 40740/50462 | Loss: 2.5257 | LR: 4.57e-05 | 2998.1ms/step | 20.0 samples/s +Step 40750/50462 | Loss: 2.5355 | LR: 4.56e-05 | 2998.1ms/step | 20.0 samples/s +Validation at step 40750: Loss = 2.5548, Perplexity = 12.87 +Step 40760/50462 | Loss: 2.5616 | LR: 4.55e-05 | 4593.3ms/step | 13.1 samples/s +Step 40770/50462 | Loss: 2.5420 | LR: 4.54e-05 | 3002.7ms/step | 20.0 samples/s +Step 40780/50462 | Loss: 2.5568 | LR: 4.54e-05 | 2999.2ms/step | 20.0 samples/s +Step 40790/50462 | Loss: 2.5417 | LR: 4.53e-05 | 2996.9ms/step | 20.0 samples/s +Step 40800/50462 | Loss: 2.5508 | LR: 4.52e-05 | 3000.1ms/step | 20.0 samples/s +Step 40810/50462 | Loss: 2.5080 | LR: 4.51e-05 | 3000.0ms/step | 20.0 samples/s +Step 40820/50462 | Loss: 2.5822 | LR: 4.50e-05 | 2999.8ms/step | 20.0 samples/s +Step 40830/50462 | Loss: 2.4968 | LR: 4.49e-05 | 3000.0ms/step | 20.0 samples/s +Step 40840/50462 | Loss: 2.5220 | LR: 4.48e-05 | 2998.3ms/step | 20.0 samples/s +Step 40850/50462 | Loss: 2.5653 | LR: 4.47e-05 | 2995.6ms/step | 20.0 samples/s +Step 40860/50462 | Loss: 2.5104 | LR: 4.46e-05 | 2998.3ms/step | 20.0 samples/s +Step 40870/50462 | Loss: 2.5655 | LR: 4.45e-05 | 2998.1ms/step | 20.0 samples/s +Step 40880/50462 | Loss: 2.5715 | LR: 4.45e-05 | 2999.5ms/step | 20.0 samples/s +Step 40890/50462 | Loss: 2.5041 | LR: 4.44e-05 | 3000.4ms/step | 20.0 samples/s +Step 40900/50462 | Loss: 2.5293 | LR: 4.43e-05 | 3000.2ms/step | 20.0 samples/s +Step 40910/50462 | Loss: 2.5504 | LR: 4.42e-05 | 2999.5ms/step | 20.0 samples/s +Step 40920/50462 | Loss: 2.5588 | LR: 4.41e-05 | 2999.4ms/step | 20.0 samples/s +Step 40930/50462 | Loss: 2.5494 | LR: 4.40e-05 | 2995.0ms/step | 20.0 samples/s +Step 40940/50462 | Loss: 2.5672 | LR: 4.39e-05 | 2998.8ms/step | 20.0 samples/s +Step 40950/50462 | Loss: 2.5218 | LR: 4.38e-05 | 2996.5ms/step | 20.0 samples/s +Step 40960/50462 | Loss: 2.5510 | LR: 4.37e-05 | 2995.0ms/step | 20.0 samples/s +Step 40970/50462 | Loss: 2.5414 | LR: 4.36e-05 | 2999.5ms/step | 20.0 samples/s +Step 40980/50462 | Loss: 2.5417 | LR: 4.36e-05 | 2993.9ms/step | 20.0 samples/s +Step 40990/50462 | Loss: 2.5098 | LR: 4.35e-05 | 3001.8ms/step | 20.0 samples/s +Step 41000/50462 | Loss: 2.5550 | LR: 4.34e-05 | 3000.6ms/step | 20.0 samples/s +Validation at step 41000: Loss = 2.5545, Perplexity = 12.87 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-41000 +Step 41010/50462 | Loss: 2.5619 | LR: 4.33e-05 | 5559.2ms/step | 10.8 samples/s +Step 41020/50462 | Loss: 2.5572 | LR: 4.32e-05 | 2994.2ms/step | 20.0 samples/s +Step 41030/50462 | Loss: 2.5593 | LR: 4.31e-05 | 2998.2ms/step | 20.0 samples/s +Step 41040/50462 | Loss: 2.5228 | LR: 4.30e-05 | 2998.6ms/step | 20.0 samples/s +Step 41050/50462 | Loss: 2.5531 | LR: 4.29e-05 | 2999.9ms/step | 20.0 samples/s +Step 41060/50462 | Loss: 2.5593 | LR: 4.28e-05 | 2990.7ms/step | 20.1 samples/s +Step 41070/50462 | Loss: 2.5374 | LR: 4.28e-05 | 2995.2ms/step | 20.0 samples/s +Step 41080/50462 | Loss: 2.5303 | LR: 4.27e-05 | 2993.6ms/step | 20.0 samples/s +Step 41090/50462 | Loss: 2.5088 | LR: 4.26e-05 | 2994.5ms/step | 20.0 samples/s +Step 41100/50462 | Loss: 2.5429 | LR: 4.25e-05 | 2994.4ms/step | 20.0 samples/s +Step 41110/50462 | Loss: 2.5471 | LR: 4.24e-05 | 3001.7ms/step | 20.0 samples/s +Step 41120/50462 | Loss: 2.5380 | LR: 4.23e-05 | 2995.1ms/step | 20.0 samples/s +Step 41130/50462 | Loss: 2.5600 | LR: 4.22e-05 | 2995.1ms/step | 20.0 samples/s +Step 41140/50462 | Loss: 2.5338 | LR: 4.21e-05 | 2998.1ms/step | 20.0 samples/s +Step 41150/50462 | Loss: 2.5480 | LR: 4.21e-05 | 2999.7ms/step | 20.0 samples/s +Step 41160/50462 | Loss: 2.5456 | LR: 4.20e-05 | 2993.9ms/step | 20.0 samples/s +Step 41170/50462 | Loss: 2.5502 | LR: 4.19e-05 | 2998.3ms/step | 20.0 samples/s +Step 41180/50462 | Loss: 2.5354 | LR: 4.18e-05 | 3000.1ms/step | 20.0 samples/s +Step 41190/50462 | Loss: 2.4946 | LR: 4.17e-05 | 2997.5ms/step | 20.0 samples/s +Step 41200/50462 | Loss: 2.5320 | LR: 4.16e-05 | 2995.5ms/step | 20.0 samples/s +Step 41210/50462 | Loss: 2.5083 | LR: 4.15e-05 | 2997.1ms/step | 20.0 samples/s +Step 41220/50462 | Loss: 2.5242 | LR: 4.14e-05 | 2999.9ms/step | 20.0 samples/s +Step 41230/50462 | Loss: 2.5482 | LR: 4.14e-05 | 3000.0ms/step | 20.0 samples/s +Step 41240/50462 | Loss: 2.5366 | LR: 4.13e-05 | 2998.0ms/step | 20.0 samples/s +Step 41250/50462 | Loss: 2.5500 | LR: 4.12e-05 | 2999.5ms/step | 20.0 samples/s +Validation at step 41250: Loss = 2.5543, Perplexity = 12.86 +Step 41260/50462 | Loss: 2.5464 | LR: 4.11e-05 | 4592.4ms/step | 13.1 samples/s +Step 41270/50462 | Loss: 2.5208 | LR: 4.10e-05 | 2997.9ms/step | 20.0 samples/s +Step 41280/50462 | Loss: 2.4995 | LR: 4.09e-05 | 3000.1ms/step | 20.0 samples/s +Step 41290/50462 | Loss: 2.5605 | LR: 4.08e-05 | 3000.1ms/step | 20.0 samples/s +Step 41300/50462 | Loss: 2.5517 | LR: 4.07e-05 | 2998.8ms/step | 20.0 samples/s +Step 41310/50462 | Loss: 2.5129 | LR: 4.07e-05 | 2997.2ms/step | 20.0 samples/s +Step 41320/50462 | Loss: 2.5948 | LR: 4.06e-05 | 2996.0ms/step | 20.0 samples/s +Step 41330/50462 | Loss: 2.5310 | LR: 4.05e-05 | 2998.1ms/step | 20.0 samples/s +Step 41340/50462 | Loss: 2.5241 | LR: 4.04e-05 | 2999.9ms/step | 20.0 samples/s +Step 41350/50462 | Loss: 2.5461 | LR: 4.03e-05 | 2997.9ms/step | 20.0 samples/s +Step 41360/50462 | Loss: 2.5722 | LR: 4.02e-05 | 2995.8ms/step | 20.0 samples/s +Step 41370/50462 | Loss: 2.5433 | LR: 4.01e-05 | 2998.2ms/step | 20.0 samples/s +Step 41380/50462 | Loss: 2.5363 | LR: 4.01e-05 | 2998.3ms/step | 20.0 samples/s +Step 41390/50462 | Loss: 2.5118 | LR: 4.00e-05 | 3001.9ms/step | 20.0 samples/s +Step 41400/50462 | Loss: 2.5525 | LR: 3.99e-05 | 3000.4ms/step | 20.0 samples/s +Step 41410/50462 | Loss: 2.5579 | LR: 3.98e-05 | 2999.7ms/step | 20.0 samples/s +Step 41420/50462 | Loss: 2.5210 | LR: 3.97e-05 | 2996.0ms/step | 20.0 samples/s +Step 41430/50462 | Loss: 2.5532 | LR: 3.96e-05 | 2997.8ms/step | 20.0 samples/s +Step 41440/50462 | Loss: 2.5218 | LR: 3.95e-05 | 2994.5ms/step | 20.0 samples/s +Step 41450/50462 | Loss: 2.5417 | LR: 3.95e-05 | 2997.3ms/step | 20.0 samples/s +Step 41460/50462 | Loss: 2.5240 | LR: 3.94e-05 | 2996.9ms/step | 20.0 samples/s +Step 41470/50462 | Loss: 2.5026 | LR: 3.93e-05 | 2991.4ms/step | 20.1 samples/s +Step 41480/50462 | Loss: 2.5617 | LR: 3.92e-05 | 2996.3ms/step | 20.0 samples/s +Step 41490/50462 | Loss: 2.5149 | LR: 3.91e-05 | 2997.8ms/step | 20.0 samples/s +Step 41500/50462 | Loss: 2.5579 | LR: 3.90e-05 | 2996.0ms/step | 20.0 samples/s +Validation at step 41500: Loss = 2.5538, Perplexity = 12.86 +Step 41510/50462 | Loss: 2.5263 | LR: 3.90e-05 | 4593.9ms/step | 13.1 samples/s +Step 41520/50462 | Loss: 2.5418 | LR: 3.89e-05 | 2995.5ms/step | 20.0 samples/s +Step 41530/50462 | Loss: 2.5530 | LR: 3.88e-05 | 2996.9ms/step | 20.0 samples/s +Step 41540/50462 | Loss: 2.5225 | LR: 3.87e-05 | 2995.8ms/step | 20.0 samples/s +Step 41550/50462 | Loss: 2.5480 | LR: 3.86e-05 | 2995.8ms/step | 20.0 samples/s +Step 41560/50462 | Loss: 2.5601 | LR: 3.85e-05 | 2998.0ms/step | 20.0 samples/s +Step 41570/50462 | Loss: 2.5475 | LR: 3.84e-05 | 3000.4ms/step | 20.0 samples/s +Step 41580/50462 | Loss: 2.5522 | LR: 3.84e-05 | 2998.2ms/step | 20.0 samples/s +Step 41590/50462 | Loss: 2.5349 | LR: 3.83e-05 | 2999.8ms/step | 20.0 samples/s +Step 41600/50462 | Loss: 2.5490 | LR: 3.82e-05 | 3000.1ms/step | 20.0 samples/s +Step 41610/50462 | Loss: 2.5319 | LR: 3.81e-05 | 2993.7ms/step | 20.0 samples/s +Step 41620/50462 | Loss: 2.5373 | LR: 3.80e-05 | 2996.3ms/step | 20.0 samples/s +Step 41630/50462 | Loss: 2.5188 | LR: 3.79e-05 | 2994.1ms/step | 20.0 samples/s +Step 41640/50462 | Loss: 2.5348 | LR: 3.79e-05 | 2998.0ms/step | 20.0 samples/s +Step 41650/50462 | Loss: 2.5211 | LR: 3.78e-05 | 2995.1ms/step | 20.0 samples/s +Step 41660/50462 | Loss: 2.5500 | LR: 3.77e-05 | 2998.5ms/step | 20.0 samples/s +Step 41670/50462 | Loss: 2.5426 | LR: 3.76e-05 | 2996.9ms/step | 20.0 samples/s +Step 41680/50462 | Loss: 2.5274 | LR: 3.75e-05 | 2995.4ms/step | 20.0 samples/s +Step 41690/50462 | Loss: 2.5239 | LR: 3.74e-05 | 2993.7ms/step | 20.0 samples/s +Step 41700/50462 | Loss: 2.5368 | LR: 3.74e-05 | 2993.8ms/step | 20.0 samples/s +Step 41710/50462 | Loss: 2.5272 | LR: 3.73e-05 | 2996.4ms/step | 20.0 samples/s +Step 41720/50462 | Loss: 2.5244 | LR: 3.72e-05 | 2998.1ms/step | 20.0 samples/s +Step 41730/50462 | Loss: 2.5304 | LR: 3.71e-05 | 3540.3ms/step | 16.9 samples/s +Step 41740/50462 | Loss: 2.5238 | LR: 3.70e-05 | 3626.7ms/step | 16.5 samples/s +Step 41750/50462 | Loss: 2.5298 | LR: 3.69e-05 | 2993.4ms/step | 20.0 samples/s +Validation at step 41750: Loss = 2.5538, Perplexity = 12.86 +Step 41760/50462 | Loss: 2.5423 | LR: 3.69e-05 | 4591.8ms/step | 13.1 samples/s +Step 41770/50462 | Loss: 2.5493 | LR: 3.68e-05 | 2992.0ms/step | 20.1 samples/s +Step 41780/50462 | Loss: 2.5367 | LR: 3.67e-05 | 2998.4ms/step | 20.0 samples/s +Step 41790/50462 | Loss: 2.5220 | LR: 3.66e-05 | 3544.7ms/step | 16.9 samples/s +Step 41800/50462 | Loss: 2.5256 | LR: 3.65e-05 | 2987.0ms/step | 20.1 samples/s +Step 41810/50462 | Loss: 2.5598 | LR: 3.64e-05 | 2997.8ms/step | 20.0 samples/s +Step 41820/50462 | Loss: 2.4852 | LR: 3.64e-05 | 3000.4ms/step | 20.0 samples/s +Step 41830/50462 | Loss: 2.5327 | LR: 3.63e-05 | 2995.1ms/step | 20.0 samples/s +Step 41840/50462 | Loss: 2.5086 | LR: 3.62e-05 | 2993.6ms/step | 20.0 samples/s +Step 41850/50462 | Loss: 2.5496 | LR: 3.61e-05 | 2992.8ms/step | 20.0 samples/s +Step 41860/50462 | Loss: 2.5356 | LR: 3.60e-05 | 2998.5ms/step | 20.0 samples/s +Step 41870/50462 | Loss: 2.5318 | LR: 3.60e-05 | 2999.8ms/step | 20.0 samples/s +Step 41880/50462 | Loss: 2.5467 | LR: 3.59e-05 | 2997.1ms/step | 20.0 samples/s +Step 41890/50462 | Loss: 2.5382 | LR: 3.58e-05 | 2993.1ms/step | 20.0 samples/s +Step 41900/50462 | Loss: 2.5632 | LR: 3.57e-05 | 2995.1ms/step | 20.0 samples/s +Step 41910/50462 | Loss: 2.5533 | LR: 3.56e-05 | 2992.4ms/step | 20.1 samples/s +Step 41920/50462 | Loss: 2.5455 | LR: 3.55e-05 | 2994.0ms/step | 20.0 samples/s +Step 41930/50462 | Loss: 2.5254 | LR: 3.55e-05 | 2996.7ms/step | 20.0 samples/s +Step 41940/50462 | Loss: 2.5534 | LR: 3.54e-05 | 2998.1ms/step | 20.0 samples/s +Step 41950/50462 | Loss: 2.5237 | LR: 3.53e-05 | 2993.4ms/step | 20.0 samples/s +Step 41960/50462 | Loss: 2.5502 | LR: 3.52e-05 | 3000.7ms/step | 20.0 samples/s +Step 41970/50462 | Loss: 2.5384 | LR: 3.51e-05 | 2997.6ms/step | 20.0 samples/s +Step 41980/50462 | Loss: 2.5363 | LR: 3.51e-05 | 2996.2ms/step | 20.0 samples/s +Step 41990/50462 | Loss: 2.5680 | LR: 3.50e-05 | 2996.3ms/step | 20.0 samples/s +Step 42000/50462 | Loss: 2.5463 | LR: 3.49e-05 | 3000.1ms/step | 20.0 samples/s +Validation at step 42000: Loss = 2.5538, Perplexity = 12.86 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-42000 +Step 42010/50462 | Loss: 2.5147 | LR: 3.48e-05 | 5600.3ms/step | 10.7 samples/s +Step 42020/50462 | Loss: 2.5582 | LR: 3.47e-05 | 2999.5ms/step | 20.0 samples/s +Step 42030/50462 | Loss: 2.5766 | LR: 3.47e-05 | 2998.3ms/step | 20.0 samples/s +Step 42040/50462 | Loss: 2.5409 | LR: 3.46e-05 | 3002.2ms/step | 20.0 samples/s +Step 42050/50462 | Loss: 2.5428 | LR: 3.45e-05 | 2999.8ms/step | 20.0 samples/s +Step 42060/50462 | Loss: 2.5762 | LR: 3.44e-05 | 3000.1ms/step | 20.0 samples/s +Step 42070/50462 | Loss: 2.5580 | LR: 3.43e-05 | 3000.0ms/step | 20.0 samples/s +Step 42080/50462 | Loss: 2.5293 | LR: 3.43e-05 | 3000.0ms/step | 20.0 samples/s +Step 42090/50462 | Loss: 2.5601 | LR: 3.42e-05 | 2999.8ms/step | 20.0 samples/s +Step 42100/50462 | Loss: 2.5423 | LR: 3.41e-05 | 3000.4ms/step | 20.0 samples/s +Step 42110/50462 | Loss: 2.5098 | LR: 3.40e-05 | 2998.1ms/step | 20.0 samples/s +Step 42120/50462 | Loss: 2.5445 | LR: 3.39e-05 | 2995.1ms/step | 20.0 samples/s +Step 42130/50462 | Loss: 2.5560 | LR: 3.39e-05 | 2992.7ms/step | 20.0 samples/s +Step 42140/50462 | Loss: 2.5530 | LR: 3.38e-05 | 2997.4ms/step | 20.0 samples/s +Step 42150/50462 | Loss: 2.5491 | LR: 3.37e-05 | 2989.8ms/step | 20.1 samples/s +Step 42160/50462 | Loss: 2.5075 | LR: 3.36e-05 | 2990.1ms/step | 20.1 samples/s +Step 42170/50462 | Loss: 2.5342 | LR: 3.35e-05 | 2994.1ms/step | 20.0 samples/s +Step 42180/50462 | Loss: 2.5413 | LR: 3.35e-05 | 2991.9ms/step | 20.1 samples/s +Step 42190/50462 | Loss: 2.5709 | LR: 3.34e-05 | 2988.1ms/step | 20.1 samples/s +Step 42200/50462 | Loss: 2.5271 | LR: 3.33e-05 | 2990.8ms/step | 20.1 samples/s +Step 42210/50462 | Loss: 2.5320 | LR: 3.32e-05 | 2993.5ms/step | 20.0 samples/s +Step 42220/50462 | Loss: 2.5589 | LR: 3.32e-05 | 2994.6ms/step | 20.0 samples/s +Step 42230/50462 | Loss: 2.5457 | LR: 3.31e-05 | 2994.2ms/step | 20.0 samples/s +Step 42240/50462 | Loss: 2.5002 | LR: 3.30e-05 | 2998.4ms/step | 20.0 samples/s +Step 42250/50462 | Loss: 2.5513 | LR: 3.29e-05 | 2990.8ms/step | 20.1 samples/s +Validation at step 42250: Loss = 2.5537, Perplexity = 12.85 +Step 42260/50462 | Loss: 2.5581 | LR: 3.28e-05 | 4591.9ms/step | 13.1 samples/s +Step 42270/50462 | Loss: 2.5699 | LR: 3.28e-05 | 2996.1ms/step | 20.0 samples/s +Step 42280/50462 | Loss: 2.5385 | LR: 3.27e-05 | 2996.4ms/step | 20.0 samples/s +Step 42290/50462 | Loss: 2.5534 | LR: 3.26e-05 | 2994.5ms/step | 20.0 samples/s +Step 42300/50462 | Loss: 2.5384 | LR: 3.25e-05 | 2995.9ms/step | 20.0 samples/s +Step 42310/50462 | Loss: 2.5313 | LR: 3.24e-05 | 2991.5ms/step | 20.1 samples/s +Step 42320/50462 | Loss: 2.5135 | LR: 3.24e-05 | 2990.6ms/step | 20.1 samples/s +Step 42330/50462 | Loss: 2.5611 | LR: 3.23e-05 | 2993.4ms/step | 20.0 samples/s +Step 42340/50462 | Loss: 2.5680 | LR: 3.22e-05 | 2992.1ms/step | 20.1 samples/s +Step 42350/50462 | Loss: 2.5153 | LR: 3.21e-05 | 2992.0ms/step | 20.1 samples/s +Step 42360/50462 | Loss: 2.5260 | LR: 3.21e-05 | 2995.1ms/step | 20.0 samples/s +Step 42370/50462 | Loss: 2.5389 | LR: 3.20e-05 | 2999.6ms/step | 20.0 samples/s +Step 42380/50462 | Loss: 2.5257 | LR: 3.19e-05 | 2995.7ms/step | 20.0 samples/s +Step 42390/50462 | Loss: 2.5533 | LR: 3.18e-05 | 2993.3ms/step | 20.0 samples/s +Step 42400/50462 | Loss: 2.5604 | LR: 3.18e-05 | 2994.9ms/step | 20.0 samples/s +Step 42410/50462 | Loss: 2.5544 | LR: 3.17e-05 | 2991.6ms/step | 20.1 samples/s +Step 42420/50462 | Loss: 2.5461 | LR: 3.16e-05 | 2991.9ms/step | 20.1 samples/s +Step 42430/50462 | Loss: 2.5349 | LR: 3.15e-05 | 2994.8ms/step | 20.0 samples/s +Step 42440/50462 | Loss: 2.5349 | LR: 3.14e-05 | 2993.9ms/step | 20.0 samples/s +Step 42450/50462 | Loss: 2.5526 | LR: 3.14e-05 | 2993.3ms/step | 20.0 samples/s +Step 42460/50462 | Loss: 2.5327 | LR: 3.13e-05 | 2989.0ms/step | 20.1 samples/s +Step 42470/50462 | Loss: 2.5431 | LR: 3.12e-05 | 2987.3ms/step | 20.1 samples/s +Step 42480/50462 | Loss: 2.5021 | LR: 3.11e-05 | 2992.1ms/step | 20.1 samples/s +Step 42490/50462 | Loss: 2.5475 | LR: 3.11e-05 | 2990.0ms/step | 20.1 samples/s +Step 42500/50462 | Loss: 2.5569 | LR: 3.10e-05 | 2992.6ms/step | 20.0 samples/s +Validation at step 42500: Loss = 2.5537, Perplexity = 12.85 +Step 42510/50462 | Loss: 2.5537 | LR: 3.09e-05 | 4591.3ms/step | 13.1 samples/s +Step 42520/50462 | Loss: 2.5379 | LR: 3.08e-05 | 2993.2ms/step | 20.0 samples/s +Step 42530/50462 | Loss: 2.5310 | LR: 3.08e-05 | 2997.4ms/step | 20.0 samples/s +Step 42540/50462 | Loss: 2.5359 | LR: 3.07e-05 | 2994.1ms/step | 20.0 samples/s +Step 42550/50462 | Loss: 2.5619 | LR: 3.06e-05 | 2997.4ms/step | 20.0 samples/s +Step 42560/50462 | Loss: 2.6203 | LR: 3.05e-05 | 2998.2ms/step | 20.0 samples/s +Step 42570/50462 | Loss: 2.5497 | LR: 3.05e-05 | 2995.3ms/step | 20.0 samples/s +Step 42580/50462 | Loss: 2.5433 | LR: 3.04e-05 | 2995.2ms/step | 20.0 samples/s +Step 42590/50462 | Loss: 2.5372 | LR: 3.03e-05 | 2998.0ms/step | 20.0 samples/s +Step 42600/50462 | Loss: 2.5851 | LR: 3.02e-05 | 2994.2ms/step | 20.0 samples/s +Step 42610/50462 | Loss: 2.5338 | LR: 3.02e-05 | 2993.0ms/step | 20.0 samples/s +Step 42620/50462 | Loss: 2.5281 | LR: 3.01e-05 | 2994.6ms/step | 20.0 samples/s +Step 42630/50462 | Loss: 2.5588 | LR: 3.00e-05 | 2994.5ms/step | 20.0 samples/s +Step 42640/50462 | Loss: 2.5847 | LR: 2.99e-05 | 2992.1ms/step | 20.1 samples/s +Step 42650/50462 | Loss: 2.5542 | LR: 2.99e-05 | 2994.4ms/step | 20.0 samples/s +Step 42660/50462 | Loss: 2.5569 | LR: 2.98e-05 | 2989.7ms/step | 20.1 samples/s +Step 42670/50462 | Loss: 2.5274 | LR: 2.97e-05 | 2994.0ms/step | 20.0 samples/s +Step 42680/50462 | Loss: 2.5273 | LR: 2.96e-05 | 2991.2ms/step | 20.1 samples/s +Step 42690/50462 | Loss: 2.5459 | LR: 2.96e-05 | 2989.3ms/step | 20.1 samples/s +Step 42700/50462 | Loss: 2.5544 | LR: 2.95e-05 | 2992.1ms/step | 20.1 samples/s +Step 42710/50462 | Loss: 2.5354 | LR: 2.94e-05 | 2988.8ms/step | 20.1 samples/s +Step 42720/50462 | Loss: 2.5372 | LR: 2.93e-05 | 2994.5ms/step | 20.0 samples/s +Step 42730/50462 | Loss: 2.5370 | LR: 2.93e-05 | 2990.2ms/step | 20.1 samples/s +Step 42740/50462 | Loss: 2.5414 | LR: 2.92e-05 | 2993.5ms/step | 20.0 samples/s +Step 42750/50462 | Loss: 2.5382 | LR: 2.91e-05 | 2994.4ms/step | 20.0 samples/s +Validation at step 42750: Loss = 2.5534, Perplexity = 12.85 +Step 42760/50462 | Loss: 2.5523 | LR: 2.90e-05 | 4592.3ms/step | 13.1 samples/s +Step 42770/50462 | Loss: 2.5429 | LR: 2.90e-05 | 2995.7ms/step | 20.0 samples/s +Step 42780/50462 | Loss: 2.5408 | LR: 2.89e-05 | 2993.7ms/step | 20.0 samples/s +Step 42790/50462 | Loss: 2.5581 | LR: 2.88e-05 | 2998.3ms/step | 20.0 samples/s +Step 42800/50462 | Loss: 2.5586 | LR: 2.87e-05 | 2996.1ms/step | 20.0 samples/s +Step 42810/50462 | Loss: 2.5649 | LR: 2.87e-05 | 2993.5ms/step | 20.0 samples/s +Step 42820/50462 | Loss: 2.5339 | LR: 2.86e-05 | 2992.3ms/step | 20.1 samples/s +Step 42830/50462 | Loss: 2.5262 | LR: 2.85e-05 | 2998.1ms/step | 20.0 samples/s +Step 42840/50462 | Loss: 2.5544 | LR: 2.84e-05 | 2993.6ms/step | 20.0 samples/s +Step 42850/50462 | Loss: 2.5451 | LR: 2.84e-05 | 2996.7ms/step | 20.0 samples/s +Step 42860/50462 | Loss: 2.5717 | LR: 2.83e-05 | 2991.7ms/step | 20.1 samples/s +Step 42870/50462 | Loss: 2.5560 | LR: 2.82e-05 | 2996.3ms/step | 20.0 samples/s +Step 42880/50462 | Loss: 2.5502 | LR: 2.82e-05 | 2994.4ms/step | 20.0 samples/s +Step 42890/50462 | Loss: 2.5350 | LR: 2.81e-05 | 2993.8ms/step | 20.0 samples/s +Step 42900/50462 | Loss: 2.5435 | LR: 2.80e-05 | 2991.7ms/step | 20.1 samples/s +Step 42910/50462 | Loss: 2.5733 | LR: 2.79e-05 | 2993.9ms/step | 20.0 samples/s +Step 42920/50462 | Loss: 2.5323 | LR: 2.79e-05 | 2997.7ms/step | 20.0 samples/s +Step 42930/50462 | Loss: 2.5315 | LR: 2.78e-05 | 2992.3ms/step | 20.1 samples/s +Step 42940/50462 | Loss: 2.5211 | LR: 2.77e-05 | 2997.9ms/step | 20.0 samples/s +Step 42950/50462 | Loss: 2.5713 | LR: 2.76e-05 | 2997.2ms/step | 20.0 samples/s +Step 42960/50462 | Loss: 2.5535 | LR: 2.76e-05 | 2995.0ms/step | 20.0 samples/s +Step 42970/50462 | Loss: 2.5262 | LR: 2.75e-05 | 2995.5ms/step | 20.0 samples/s +Step 42980/50462 | Loss: 2.5393 | LR: 2.74e-05 | 2994.7ms/step | 20.0 samples/s +Step 42990/50462 | Loss: 2.5465 | LR: 2.74e-05 | 2997.6ms/step | 20.0 samples/s +Step 43000/50462 | Loss: 2.5248 | LR: 2.73e-05 | 2991.7ms/step | 20.1 samples/s +Validation at step 43000: Loss = 2.5537, Perplexity = 12.85 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-43000 +Step 43010/50462 | Loss: 2.5654 | LR: 2.72e-05 | 5640.5ms/step | 10.6 samples/s +Step 43020/50462 | Loss: 2.5385 | LR: 2.71e-05 | 2997.8ms/step | 20.0 samples/s +Step 43030/50462 | Loss: 2.5444 | LR: 2.71e-05 | 3000.4ms/step | 20.0 samples/s +Step 43040/50462 | Loss: 2.5310 | LR: 2.70e-05 | 2999.3ms/step | 20.0 samples/s +Step 43050/50462 | Loss: 2.5520 | LR: 2.69e-05 | 2996.6ms/step | 20.0 samples/s +Step 43060/50462 | Loss: 2.5268 | LR: 2.69e-05 | 2997.7ms/step | 20.0 samples/s +Step 43070/50462 | Loss: 2.5447 | LR: 2.68e-05 | 2998.4ms/step | 20.0 samples/s +Step 43080/50462 | Loss: 2.5228 | LR: 2.67e-05 | 3000.2ms/step | 20.0 samples/s +Step 43090/50462 | Loss: 2.5203 | LR: 2.66e-05 | 2993.3ms/step | 20.0 samples/s +Step 43100/50462 | Loss: 2.5185 | LR: 2.66e-05 | 2997.7ms/step | 20.0 samples/s +Step 43110/50462 | Loss: 2.5276 | LR: 2.65e-05 | 2999.0ms/step | 20.0 samples/s +Step 43120/50462 | Loss: 2.5422 | LR: 2.64e-05 | 3000.0ms/step | 20.0 samples/s +Step 43130/50462 | Loss: 2.5531 | LR: 2.64e-05 | 2998.6ms/step | 20.0 samples/s +Step 43140/50462 | Loss: 2.5231 | LR: 2.63e-05 | 2999.7ms/step | 20.0 samples/s +Step 43150/50462 | Loss: 2.5241 | LR: 2.62e-05 | 2999.7ms/step | 20.0 samples/s +Step 43160/50462 | Loss: 2.5633 | LR: 2.61e-05 | 3000.4ms/step | 20.0 samples/s +Step 43170/50462 | Loss: 2.5547 | LR: 2.61e-05 | 2997.2ms/step | 20.0 samples/s +Step 43180/50462 | Loss: 2.5083 | LR: 2.60e-05 | 2998.7ms/step | 20.0 samples/s +Step 43190/50462 | Loss: 2.5639 | LR: 2.59e-05 | 2996.8ms/step | 20.0 samples/s +Step 43200/50462 | Loss: 2.5419 | LR: 2.59e-05 | 2993.2ms/step | 20.0 samples/s +Step 43210/50462 | Loss: 2.5471 | LR: 2.58e-05 | 2998.2ms/step | 20.0 samples/s +Step 43220/50462 | Loss: 2.5553 | LR: 2.57e-05 | 2997.9ms/step | 20.0 samples/s +Step 43230/50462 | Loss: 2.5404 | LR: 2.57e-05 | 2997.9ms/step | 20.0 samples/s +Step 43240/50462 | Loss: 2.5614 | LR: 2.56e-05 | 2993.6ms/step | 20.0 samples/s +Step 43250/50462 | Loss: 2.5121 | LR: 2.55e-05 | 2996.5ms/step | 20.0 samples/s +Validation at step 43250: Loss = 2.5537, Perplexity = 12.85 +Step 43260/50462 | Loss: 2.5154 | LR: 2.54e-05 | 4587.7ms/step | 13.1 samples/s +Step 43270/50462 | Loss: 2.5253 | LR: 2.54e-05 | 2996.7ms/step | 20.0 samples/s +Step 43280/50462 | Loss: 2.5483 | LR: 2.53e-05 | 2995.0ms/step | 20.0 samples/s +Step 43290/50462 | Loss: 2.5261 | LR: 2.52e-05 | 2990.9ms/step | 20.1 samples/s +Step 43300/50462 | Loss: 2.5703 | LR: 2.52e-05 | 2995.1ms/step | 20.0 samples/s +Step 43310/50462 | Loss: 2.5350 | LR: 2.51e-05 | 2997.0ms/step | 20.0 samples/s +Step 43320/50462 | Loss: 2.5389 | LR: 2.50e-05 | 2997.8ms/step | 20.0 samples/s +Step 43330/50462 | Loss: 2.5244 | LR: 2.50e-05 | 2991.8ms/step | 20.1 samples/s +Step 43340/50462 | Loss: 2.5358 | LR: 2.49e-05 | 2994.1ms/step | 20.0 samples/s +Step 43350/50462 | Loss: 2.5216 | LR: 2.48e-05 | 2996.1ms/step | 20.0 samples/s +Step 43360/50462 | Loss: 2.5544 | LR: 2.48e-05 | 2993.9ms/step | 20.0 samples/s +Step 43370/50462 | Loss: 2.5460 | LR: 2.47e-05 | 2994.0ms/step | 20.0 samples/s +Step 43380/50462 | Loss: 2.5246 | LR: 2.46e-05 | 2997.6ms/step | 20.0 samples/s +Step 43390/50462 | Loss: 2.5542 | LR: 2.46e-05 | 2992.2ms/step | 20.1 samples/s +Step 43400/50462 | Loss: 2.5366 | LR: 2.45e-05 | 2994.8ms/step | 20.0 samples/s +Step 43410/50462 | Loss: 2.5572 | LR: 2.44e-05 | 2996.0ms/step | 20.0 samples/s +Step 43420/50462 | Loss: 2.5350 | LR: 2.44e-05 | 2995.3ms/step | 20.0 samples/s +Step 43430/50462 | Loss: 2.5412 | LR: 2.43e-05 | 2996.6ms/step | 20.0 samples/s +Step 43440/50462 | Loss: 2.5370 | LR: 2.42e-05 | 2992.6ms/step | 20.0 samples/s +Step 43450/50462 | Loss: 2.5618 | LR: 2.41e-05 | 2990.5ms/step | 20.1 samples/s +Step 43460/50462 | Loss: 2.5505 | LR: 2.41e-05 | 2996.8ms/step | 20.0 samples/s +Step 43470/50462 | Loss: 2.5640 | LR: 2.40e-05 | 2993.1ms/step | 20.0 samples/s +Step 43480/50462 | Loss: 2.5648 | LR: 2.39e-05 | 2992.2ms/step | 20.1 samples/s +Step 43490/50462 | Loss: 2.5325 | LR: 2.39e-05 | 2995.4ms/step | 20.0 samples/s +Step 43500/50462 | Loss: 2.5496 | LR: 2.38e-05 | 2996.9ms/step | 20.0 samples/s +Validation at step 43500: Loss = 2.5534, Perplexity = 12.85 +Step 43510/50462 | Loss: 2.5537 | LR: 2.37e-05 | 4587.8ms/step | 13.1 samples/s +Step 43520/50462 | Loss: 2.5151 | LR: 2.37e-05 | 2998.6ms/step | 20.0 samples/s +Step 43530/50462 | Loss: 2.5377 | LR: 2.36e-05 | 2998.1ms/step | 20.0 samples/s +Step 43540/50462 | Loss: 2.5345 | LR: 2.35e-05 | 2989.4ms/step | 20.1 samples/s +Step 43550/50462 | Loss: 2.5428 | LR: 2.35e-05 | 2990.9ms/step | 20.1 samples/s +Step 43560/50462 | Loss: 2.5227 | LR: 2.34e-05 | 2991.3ms/step | 20.1 samples/s +Step 43570/50462 | Loss: 2.5420 | LR: 2.33e-05 | 2994.7ms/step | 20.0 samples/s +Step 43580/50462 | Loss: 2.5701 | LR: 2.33e-05 | 2995.4ms/step | 20.0 samples/s +Step 43590/50462 | Loss: 2.5276 | LR: 2.32e-05 | 2994.2ms/step | 20.0 samples/s +Step 43600/50462 | Loss: 2.5622 | LR: 2.31e-05 | 2994.0ms/step | 20.0 samples/s +Step 43610/50462 | Loss: 2.5476 | LR: 2.31e-05 | 2994.6ms/step | 20.0 samples/s +Step 43620/50462 | Loss: 2.5303 | LR: 2.30e-05 | 2993.0ms/step | 20.0 samples/s +Step 43630/50462 | Loss: 2.5677 | LR: 2.29e-05 | 2993.9ms/step | 20.0 samples/s +Step 43640/50462 | Loss: 2.5312 | LR: 2.29e-05 | 2990.5ms/step | 20.1 samples/s +Step 43650/50462 | Loss: 2.5508 | LR: 2.28e-05 | 2992.5ms/step | 20.1 samples/s +Step 43660/50462 | Loss: 2.5628 | LR: 2.27e-05 | 2992.0ms/step | 20.1 samples/s +Step 43670/50462 | Loss: 2.5498 | LR: 2.27e-05 | 2993.0ms/step | 20.0 samples/s +Step 43680/50462 | Loss: 2.5554 | LR: 2.26e-05 | 2993.1ms/step | 20.0 samples/s +Step 43690/50462 | Loss: 2.5483 | LR: 2.25e-05 | 2993.7ms/step | 20.0 samples/s +Step 43700/50462 | Loss: 2.5639 | LR: 2.25e-05 | 2992.4ms/step | 20.1 samples/s +Step 43710/50462 | Loss: 2.5695 | LR: 2.24e-05 | 2993.1ms/step | 20.0 samples/s +Step 43720/50462 | Loss: 2.5254 | LR: 2.24e-05 | 2987.2ms/step | 20.1 samples/s +Step 43730/50462 | Loss: 2.5156 | LR: 2.23e-05 | 2995.6ms/step | 20.0 samples/s +Step 43740/50462 | Loss: 2.5391 | LR: 2.22e-05 | 2993.9ms/step | 20.0 samples/s +Step 43750/50462 | Loss: 2.5400 | LR: 2.22e-05 | 2999.0ms/step | 20.0 samples/s +Validation at step 43750: Loss = 2.5534, Perplexity = 12.85 +Step 43760/50462 | Loss: 2.5459 | LR: 2.21e-05 | 4585.1ms/step | 13.1 samples/s +Step 43770/50462 | Loss: 2.5300 | LR: 2.20e-05 | 2995.2ms/step | 20.0 samples/s +Step 43780/50462 | Loss: 2.5396 | LR: 2.20e-05 | 2990.1ms/step | 20.1 samples/s +Step 43790/50462 | Loss: 2.5449 | LR: 2.19e-05 | 2991.0ms/step | 20.1 samples/s +Step 43800/50462 | Loss: 2.5605 | LR: 2.18e-05 | 2997.0ms/step | 20.0 samples/s +Step 43810/50462 | Loss: 2.5039 | LR: 2.18e-05 | 2994.1ms/step | 20.0 samples/s +Step 43820/50462 | Loss: 2.5084 | LR: 2.17e-05 | 2995.8ms/step | 20.0 samples/s +Step 43830/50462 | Loss: 2.5514 | LR: 2.16e-05 | 2992.6ms/step | 20.0 samples/s +Step 43840/50462 | Loss: 2.5673 | LR: 2.16e-05 | 2992.0ms/step | 20.1 samples/s +Step 43850/50462 | Loss: 2.5622 | LR: 2.15e-05 | 2988.6ms/step | 20.1 samples/s +Step 43860/50462 | Loss: 2.5445 | LR: 2.14e-05 | 3540.5ms/step | 16.9 samples/s +Step 43870/50462 | Loss: 2.5338 | LR: 2.14e-05 | 2984.3ms/step | 20.1 samples/s +Step 43880/50462 | Loss: 2.5550 | LR: 2.13e-05 | 3612.0ms/step | 16.6 samples/s +Step 43890/50462 | Loss: 2.5451 | LR: 2.13e-05 | 3516.2ms/step | 17.1 samples/s +Step 43900/50462 | Loss: 2.5992 | LR: 2.12e-05 | 2986.7ms/step | 20.1 samples/s +Step 43910/50462 | Loss: 2.5639 | LR: 2.11e-05 | 2995.6ms/step | 20.0 samples/s +Step 43920/50462 | Loss: 2.5404 | LR: 2.11e-05 | 2993.3ms/step | 20.0 samples/s +Step 43930/50462 | Loss: 2.5662 | LR: 2.10e-05 | 2995.0ms/step | 20.0 samples/s +Step 43940/50462 | Loss: 2.5408 | LR: 2.09e-05 | 2995.8ms/step | 20.0 samples/s +Step 43950/50462 | Loss: 2.5527 | LR: 2.09e-05 | 2990.2ms/step | 20.1 samples/s +Step 43960/50462 | Loss: 2.5356 | LR: 2.08e-05 | 2988.7ms/step | 20.1 samples/s +Step 43970/50462 | Loss: 2.5304 | LR: 2.07e-05 | 2993.6ms/step | 20.0 samples/s +Step 43980/50462 | Loss: 2.5233 | LR: 2.07e-05 | 2988.2ms/step | 20.1 samples/s +Step 43990/50462 | Loss: 2.5595 | LR: 2.06e-05 | 2992.2ms/step | 20.1 samples/s +Step 44000/50462 | Loss: 2.5656 | LR: 2.06e-05 | 2997.8ms/step | 20.0 samples/s +Validation at step 44000: Loss = 2.5530, Perplexity = 12.85 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-44000 +Step 44010/50462 | Loss: 2.5497 | LR: 2.05e-05 | 5657.0ms/step | 10.6 samples/s +Step 44020/50462 | Loss: 2.5412 | LR: 2.04e-05 | 2993.3ms/step | 20.0 samples/s +Step 44030/50462 | Loss: 2.5381 | LR: 2.04e-05 | 2990.5ms/step | 20.1 samples/s +Step 44040/50462 | Loss: 2.5311 | LR: 2.03e-05 | 2989.3ms/step | 20.1 samples/s +Step 44050/50462 | Loss: 2.5665 | LR: 2.02e-05 | 2993.2ms/step | 20.0 samples/s +Step 44060/50462 | Loss: 2.5641 | LR: 2.02e-05 | 2991.0ms/step | 20.1 samples/s +Step 44070/50462 | Loss: 2.5642 | LR: 2.01e-05 | 2993.3ms/step | 20.0 samples/s +Step 44080/50462 | Loss: 2.5459 | LR: 2.01e-05 | 2990.4ms/step | 20.1 samples/s +Step 44090/50462 | Loss: 2.5145 | LR: 2.00e-05 | 2991.5ms/step | 20.1 samples/s +Step 44100/50462 | Loss: 2.5348 | LR: 1.99e-05 | 2990.9ms/step | 20.1 samples/s +Step 44110/50462 | Loss: 2.5363 | LR: 1.99e-05 | 2992.3ms/step | 20.1 samples/s +Step 44120/50462 | Loss: 2.5534 | LR: 1.98e-05 | 2991.2ms/step | 20.1 samples/s +Step 44130/50462 | Loss: 2.5578 | LR: 1.97e-05 | 2993.1ms/step | 20.0 samples/s +Step 44140/50462 | Loss: 2.5283 | LR: 1.97e-05 | 2991.4ms/step | 20.1 samples/s +Step 44150/50462 | Loss: 2.5232 | LR: 1.96e-05 | 2989.4ms/step | 20.1 samples/s +Step 44160/50462 | Loss: 2.5523 | LR: 1.96e-05 | 2992.4ms/step | 20.1 samples/s +Step 44170/50462 | Loss: 2.5580 | LR: 1.95e-05 | 2994.1ms/step | 20.0 samples/s +Step 44180/50462 | Loss: 2.5235 | LR: 1.94e-05 | 2994.1ms/step | 20.0 samples/s +Step 44190/50462 | Loss: 2.5295 | LR: 1.94e-05 | 2990.9ms/step | 20.1 samples/s +Step 44200/50462 | Loss: 2.5504 | LR: 1.93e-05 | 2989.1ms/step | 20.1 samples/s +Step 44210/50462 | Loss: 2.5395 | LR: 1.93e-05 | 2991.3ms/step | 20.1 samples/s +Step 44220/50462 | Loss: 2.5441 | LR: 1.92e-05 | 2992.8ms/step | 20.0 samples/s +Step 44230/50462 | Loss: 2.5571 | LR: 1.91e-05 | 2996.1ms/step | 20.0 samples/s +Step 44240/50462 | Loss: 2.5449 | LR: 1.91e-05 | 2997.3ms/step | 20.0 samples/s +Step 44250/50462 | Loss: 2.5583 | LR: 1.90e-05 | 2993.7ms/step | 20.0 samples/s +Validation at step 44250: Loss = 2.5532, Perplexity = 12.85 +Step 44260/50462 | Loss: 2.5507 | LR: 1.90e-05 | 4591.0ms/step | 13.1 samples/s +Step 44270/50462 | Loss: 2.5239 | LR: 1.89e-05 | 2999.9ms/step | 20.0 samples/s +Step 44280/50462 | Loss: 2.5457 | LR: 1.88e-05 | 3000.3ms/step | 20.0 samples/s +Step 44290/50462 | Loss: 2.5434 | LR: 1.88e-05 | 2999.5ms/step | 20.0 samples/s +Step 44300/50462 | Loss: 2.4946 | LR: 1.87e-05 | 2990.0ms/step | 20.1 samples/s +Step 44310/50462 | Loss: 2.5233 | LR: 1.87e-05 | 2994.2ms/step | 20.0 samples/s +Step 44320/50462 | Loss: 2.5563 | LR: 1.86e-05 | 2998.7ms/step | 20.0 samples/s +Step 44330/50462 | Loss: 2.5466 | LR: 1.85e-05 | 2995.9ms/step | 20.0 samples/s +Step 44340/50462 | Loss: 2.5418 | LR: 1.85e-05 | 2996.6ms/step | 20.0 samples/s +Step 44350/50462 | Loss: 2.5488 | LR: 1.84e-05 | 2994.9ms/step | 20.0 samples/s +Step 44360/50462 | Loss: 2.5206 | LR: 1.84e-05 | 3000.1ms/step | 20.0 samples/s +Step 44370/50462 | Loss: 2.5456 | LR: 1.83e-05 | 2999.8ms/step | 20.0 samples/s +Step 44380/50462 | Loss: 2.5369 | LR: 1.82e-05 | 2999.1ms/step | 20.0 samples/s +Step 44390/50462 | Loss: 2.5571 | LR: 1.82e-05 | 2992.4ms/step | 20.1 samples/s +Step 44400/50462 | Loss: 2.5235 | LR: 1.81e-05 | 2993.4ms/step | 20.0 samples/s +Step 44410/50462 | Loss: 2.5319 | LR: 1.81e-05 | 2993.6ms/step | 20.0 samples/s +Step 44420/50462 | Loss: 2.5423 | LR: 1.80e-05 | 2996.1ms/step | 20.0 samples/s +Step 44430/50462 | Loss: 2.5502 | LR: 1.79e-05 | 2997.9ms/step | 20.0 samples/s +Step 44440/50462 | Loss: 2.5395 | LR: 1.79e-05 | 2998.1ms/step | 20.0 samples/s +Step 44450/50462 | Loss: 2.5277 | LR: 1.78e-05 | 3000.4ms/step | 20.0 samples/s +Step 44460/50462 | Loss: 2.5435 | LR: 1.78e-05 | 2999.2ms/step | 20.0 samples/s +Step 44470/50462 | Loss: 2.5568 | LR: 1.77e-05 | 3000.5ms/step | 20.0 samples/s +Step 44480/50462 | Loss: 2.5706 | LR: 1.77e-05 | 3000.1ms/step | 20.0 samples/s +Step 44490/50462 | Loss: 2.5667 | LR: 1.76e-05 | 2999.4ms/step | 20.0 samples/s +Step 44500/50462 | Loss: 2.5282 | LR: 1.75e-05 | 2997.9ms/step | 20.0 samples/s +Validation at step 44500: Loss = 2.5530, Perplexity = 12.85 +Step 44510/50462 | Loss: 2.5178 | LR: 1.75e-05 | 4592.0ms/step | 13.1 samples/s +Step 44520/50462 | Loss: 2.5167 | LR: 1.74e-05 | 2998.5ms/step | 20.0 samples/s +Step 44530/50462 | Loss: 2.5421 | LR: 1.74e-05 | 3000.0ms/step | 20.0 samples/s +Step 44540/50462 | Loss: 2.5313 | LR: 1.73e-05 | 3000.1ms/step | 20.0 samples/s +Step 44550/50462 | Loss: 2.5458 | LR: 1.72e-05 | 2998.1ms/step | 20.0 samples/s +Step 44560/50462 | Loss: 2.5492 | LR: 1.72e-05 | 2999.6ms/step | 20.0 samples/s +Step 44570/50462 | Loss: 2.5376 | LR: 1.71e-05 | 2996.7ms/step | 20.0 samples/s +Step 44580/50462 | Loss: 2.5151 | LR: 1.71e-05 | 2999.9ms/step | 20.0 samples/s +Step 44590/50462 | Loss: 2.5339 | LR: 1.70e-05 | 2997.8ms/step | 20.0 samples/s +Step 44600/50462 | Loss: 2.5571 | LR: 1.70e-05 | 3000.4ms/step | 20.0 samples/s +Step 44610/50462 | Loss: 2.5386 | LR: 1.69e-05 | 2999.6ms/step | 20.0 samples/s +Step 44620/50462 | Loss: 2.5559 | LR: 1.68e-05 | 2996.0ms/step | 20.0 samples/s +Step 44630/50462 | Loss: 2.5402 | LR: 1.68e-05 | 2994.9ms/step | 20.0 samples/s +Step 44640/50462 | Loss: 2.5325 | LR: 1.67e-05 | 2999.4ms/step | 20.0 samples/s +Step 44650/50462 | Loss: 2.5213 | LR: 1.67e-05 | 2995.9ms/step | 20.0 samples/s +Step 44660/50462 | Loss: 2.5585 | LR: 1.66e-05 | 2994.5ms/step | 20.0 samples/s +Step 44670/50462 | Loss: 2.5678 | LR: 1.66e-05 | 2999.2ms/step | 20.0 samples/s +Step 44680/50462 | Loss: 2.5542 | LR: 1.65e-05 | 2998.1ms/step | 20.0 samples/s +Step 44690/50462 | Loss: 2.5502 | LR: 1.64e-05 | 2995.9ms/step | 20.0 samples/s +Step 44700/50462 | Loss: 2.5563 | LR: 1.64e-05 | 2996.5ms/step | 20.0 samples/s +Step 44710/50462 | Loss: 2.5395 | LR: 1.63e-05 | 2997.4ms/step | 20.0 samples/s +Step 44720/50462 | Loss: 2.5413 | LR: 1.63e-05 | 2992.8ms/step | 20.0 samples/s +Step 44730/50462 | Loss: 2.5297 | LR: 1.62e-05 | 2999.6ms/step | 20.0 samples/s +Step 44740/50462 | Loss: 2.5246 | LR: 1.62e-05 | 2998.0ms/step | 20.0 samples/s +Step 44750/50462 | Loss: 2.5495 | LR: 1.61e-05 | 2996.1ms/step | 20.0 samples/s +Validation at step 44750: Loss = 2.5528, Perplexity = 12.84 +Step 44760/50462 | Loss: 2.5655 | LR: 1.61e-05 | 4596.1ms/step | 13.1 samples/s +Step 44770/50462 | Loss: 2.5150 | LR: 1.60e-05 | 2996.2ms/step | 20.0 samples/s +Step 44780/50462 | Loss: 2.5435 | LR: 1.59e-05 | 2994.0ms/step | 20.0 samples/s +Step 44790/50462 | Loss: 2.5308 | LR: 1.59e-05 | 2997.8ms/step | 20.0 samples/s +Step 44800/50462 | Loss: 2.5324 | LR: 1.58e-05 | 3000.4ms/step | 20.0 samples/s +Step 44810/50462 | Loss: 2.5605 | LR: 1.58e-05 | 2995.9ms/step | 20.0 samples/s +Step 44820/50462 | Loss: 2.5486 | LR: 1.57e-05 | 2999.7ms/step | 20.0 samples/s +Step 44830/50462 | Loss: 2.5710 | LR: 1.57e-05 | 3000.5ms/step | 20.0 samples/s +Step 44840/50462 | Loss: 2.5270 | LR: 1.56e-05 | 2997.6ms/step | 20.0 samples/s +Step 44850/50462 | Loss: 2.5324 | LR: 1.56e-05 | 2998.7ms/step | 20.0 samples/s +Step 44860/50462 | Loss: 2.5307 | LR: 1.55e-05 | 2999.0ms/step | 20.0 samples/s +Step 44870/50462 | Loss: 2.5198 | LR: 1.54e-05 | 2993.6ms/step | 20.0 samples/s +Step 44880/50462 | Loss: 2.5439 | LR: 1.54e-05 | 2997.2ms/step | 20.0 samples/s +Step 44890/50462 | Loss: 2.5058 | LR: 1.53e-05 | 2997.0ms/step | 20.0 samples/s +Step 44900/50462 | Loss: 2.5062 | LR: 1.53e-05 | 2993.2ms/step | 20.0 samples/s +Step 44910/50462 | Loss: 2.5267 | LR: 1.52e-05 | 2997.9ms/step | 20.0 samples/s +Step 44920/50462 | Loss: 2.5305 | LR: 1.52e-05 | 2997.8ms/step | 20.0 samples/s +Step 44930/50462 | Loss: 2.5353 | LR: 1.51e-05 | 2999.9ms/step | 20.0 samples/s +Step 44940/50462 | Loss: 2.5364 | LR: 1.51e-05 | 2994.2ms/step | 20.0 samples/s +Step 44950/50462 | Loss: 2.4957 | LR: 1.50e-05 | 2998.4ms/step | 20.0 samples/s +Step 44960/50462 | Loss: 2.5741 | LR: 1.50e-05 | 2995.9ms/step | 20.0 samples/s +Step 44970/50462 | Loss: 2.5231 | LR: 1.49e-05 | 2995.8ms/step | 20.0 samples/s +Step 44980/50462 | Loss: 2.5369 | LR: 1.49e-05 | 2995.6ms/step | 20.0 samples/s +Step 44990/50462 | Loss: 2.5641 | LR: 1.48e-05 | 2996.2ms/step | 20.0 samples/s +Step 45000/50462 | Loss: 2.5555 | LR: 1.47e-05 | 2996.3ms/step | 20.0 samples/s +Validation at step 45000: Loss = 2.5529, Perplexity = 12.84 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-45000 +Step 45010/50462 | Loss: 2.5475 | LR: 1.47e-05 | 5703.4ms/step | 10.5 samples/s +Step 45020/50462 | Loss: 2.5312 | LR: 1.46e-05 | 2992.9ms/step | 20.0 samples/s +Step 45030/50462 | Loss: 2.5368 | LR: 1.46e-05 | 3000.2ms/step | 20.0 samples/s +Step 45040/50462 | Loss: 2.5519 | LR: 1.45e-05 | 2996.3ms/step | 20.0 samples/s +Step 45050/50462 | Loss: 2.5245 | LR: 1.45e-05 | 2999.7ms/step | 20.0 samples/s +Step 45060/50462 | Loss: 2.5491 | LR: 1.44e-05 | 2999.8ms/step | 20.0 samples/s +Step 45070/50462 | Loss: 2.5418 | LR: 1.44e-05 | 3000.0ms/step | 20.0 samples/s +Step 45080/50462 | Loss: 2.5336 | LR: 1.43e-05 | 2998.4ms/step | 20.0 samples/s +Step 45090/50462 | Loss: 2.5336 | LR: 1.43e-05 | 2999.6ms/step | 20.0 samples/s +Step 45100/50462 | Loss: 2.5676 | LR: 1.42e-05 | 2994.4ms/step | 20.0 samples/s +Step 45110/50462 | Loss: 2.5675 | LR: 1.42e-05 | 2993.3ms/step | 20.0 samples/s +Step 45120/50462 | Loss: 2.5503 | LR: 1.41e-05 | 2998.0ms/step | 20.0 samples/s +Step 45130/50462 | Loss: 2.5578 | LR: 1.41e-05 | 2998.4ms/step | 20.0 samples/s +Step 45140/50462 | Loss: 2.5278 | LR: 1.40e-05 | 2993.6ms/step | 20.0 samples/s +Step 45150/50462 | Loss: 2.5525 | LR: 1.40e-05 | 2995.0ms/step | 20.0 samples/s +Step 45160/50462 | Loss: 2.5588 | LR: 1.39e-05 | 2991.4ms/step | 20.1 samples/s +Step 45170/50462 | Loss: 2.5204 | LR: 1.39e-05 | 2994.8ms/step | 20.0 samples/s +Step 45180/50462 | Loss: 2.5483 | LR: 1.38e-05 | 2999.8ms/step | 20.0 samples/s +Step 45190/50462 | Loss: 2.5353 | LR: 1.37e-05 | 2995.7ms/step | 20.0 samples/s +Step 45200/50462 | Loss: 2.5601 | LR: 1.37e-05 | 2997.8ms/step | 20.0 samples/s +Step 45210/50462 | Loss: 2.5432 | LR: 1.36e-05 | 2998.3ms/step | 20.0 samples/s +Step 45220/50462 | Loss: 2.5284 | LR: 1.36e-05 | 2995.9ms/step | 20.0 samples/s +Step 45230/50462 | Loss: 2.5240 | LR: 1.35e-05 | 2994.2ms/step | 20.0 samples/s +Step 45240/50462 | Loss: 2.5578 | LR: 1.35e-05 | 2999.7ms/step | 20.0 samples/s +Step 45250/50462 | Loss: 2.5364 | LR: 1.34e-05 | 2999.6ms/step | 20.0 samples/s +Validation at step 45250: Loss = 2.5533, Perplexity = 12.85 +Step 45260/50462 | Loss: 2.5485 | LR: 1.34e-05 | 4592.9ms/step | 13.1 samples/s +Step 45270/50462 | Loss: 2.5565 | LR: 1.33e-05 | 2999.7ms/step | 20.0 samples/s +Step 45280/50462 | Loss: 2.5146 | LR: 1.33e-05 | 2996.0ms/step | 20.0 samples/s +Step 45290/50462 | Loss: 2.5407 | LR: 1.32e-05 | 2997.6ms/step | 20.0 samples/s +Step 45300/50462 | Loss: 2.5591 | LR: 1.32e-05 | 2992.2ms/step | 20.1 samples/s +Step 45310/50462 | Loss: 2.5287 | LR: 1.31e-05 | 2998.5ms/step | 20.0 samples/s +Step 45320/50462 | Loss: 2.5361 | LR: 1.31e-05 | 2999.8ms/step | 20.0 samples/s +Step 45330/50462 | Loss: 2.5511 | LR: 1.30e-05 | 2999.9ms/step | 20.0 samples/s +Step 45340/50462 | Loss: 2.5440 | LR: 1.30e-05 | 2997.5ms/step | 20.0 samples/s +Step 45350/50462 | Loss: 2.5421 | LR: 1.29e-05 | 2996.6ms/step | 20.0 samples/s +Step 45360/50462 | Loss: 2.5607 | LR: 1.29e-05 | 2995.7ms/step | 20.0 samples/s +Step 45370/50462 | Loss: 2.5329 | LR: 1.28e-05 | 2994.6ms/step | 20.0 samples/s +Step 45380/50462 | Loss: 2.5300 | LR: 1.28e-05 | 2999.5ms/step | 20.0 samples/s +Step 45390/50462 | Loss: 2.5362 | LR: 1.27e-05 | 3000.5ms/step | 20.0 samples/s +Step 45400/50462 | Loss: 2.5212 | LR: 1.27e-05 | 2999.9ms/step | 20.0 samples/s +Step 45410/50462 | Loss: 2.5656 | LR: 1.26e-05 | 2996.1ms/step | 20.0 samples/s +Step 45420/50462 | Loss: 2.5343 | LR: 1.26e-05 | 2998.9ms/step | 20.0 samples/s +Step 45430/50462 | Loss: 2.5102 | LR: 1.25e-05 | 2991.5ms/step | 20.1 samples/s +Step 45440/50462 | Loss: 2.5528 | LR: 1.25e-05 | 2999.7ms/step | 20.0 samples/s +Step 45450/50462 | Loss: 2.5437 | LR: 1.24e-05 | 3000.0ms/step | 20.0 samples/s +Step 45460/50462 | Loss: 2.5305 | LR: 1.24e-05 | 2998.6ms/step | 20.0 samples/s +Step 45470/50462 | Loss: 2.5413 | LR: 1.23e-05 | 2997.1ms/step | 20.0 samples/s +Step 45480/50462 | Loss: 2.5304 | LR: 1.23e-05 | 2998.3ms/step | 20.0 samples/s +Step 45490/50462 | Loss: 2.5516 | LR: 1.22e-05 | 3000.0ms/step | 20.0 samples/s +Step 45500/50462 | Loss: 2.5530 | LR: 1.22e-05 | 2993.7ms/step | 20.0 samples/s +Validation at step 45500: Loss = 2.5528, Perplexity = 12.84 +Step 45510/50462 | Loss: 2.5269 | LR: 1.21e-05 | 4606.7ms/step | 13.0 samples/s +Step 45520/50462 | Loss: 2.5724 | LR: 1.21e-05 | 2999.7ms/step | 20.0 samples/s +Step 45530/50462 | Loss: 2.5189 | LR: 1.20e-05 | 2999.6ms/step | 20.0 samples/s +Step 45540/50462 | Loss: 2.5676 | LR: 1.20e-05 | 3000.5ms/step | 20.0 samples/s +Step 45550/50462 | Loss: 2.5266 | LR: 1.19e-05 | 3005.3ms/step | 20.0 samples/s +Step 45560/50462 | Loss: 2.5507 | LR: 1.19e-05 | 3000.1ms/step | 20.0 samples/s +Step 45570/50462 | Loss: 2.5557 | LR: 1.19e-05 | 2998.4ms/step | 20.0 samples/s +Step 45580/50462 | Loss: 2.5332 | LR: 1.18e-05 | 2998.0ms/step | 20.0 samples/s +Step 45590/50462 | Loss: 2.5213 | LR: 1.18e-05 | 3004.3ms/step | 20.0 samples/s +Step 45600/50462 | Loss: 2.5295 | LR: 1.17e-05 | 3000.1ms/step | 20.0 samples/s +Step 45610/50462 | Loss: 2.5091 | LR: 1.17e-05 | 2999.1ms/step | 20.0 samples/s +Step 45620/50462 | Loss: 2.5407 | LR: 1.16e-05 | 2998.7ms/step | 20.0 samples/s +Step 45630/50462 | Loss: 2.5582 | LR: 1.16e-05 | 2999.6ms/step | 20.0 samples/s +Step 45640/50462 | Loss: 2.5359 | LR: 1.15e-05 | 3000.7ms/step | 20.0 samples/s +Step 45650/50462 | Loss: 2.5613 | LR: 1.15e-05 | 2999.6ms/step | 20.0 samples/s +Step 45660/50462 | Loss: 2.5454 | LR: 1.14e-05 | 2998.4ms/step | 20.0 samples/s +Step 45670/50462 | Loss: 2.5117 | LR: 1.14e-05 | 3000.0ms/step | 20.0 samples/s +Step 45680/50462 | Loss: 2.5318 | LR: 1.13e-05 | 3000.0ms/step | 20.0 samples/s +Step 45690/50462 | Loss: 2.5524 | LR: 1.13e-05 | 2998.0ms/step | 20.0 samples/s +Step 45700/50462 | Loss: 2.5457 | LR: 1.12e-05 | 2999.9ms/step | 20.0 samples/s +Step 45710/50462 | Loss: 2.5468 | LR: 1.12e-05 | 3002.3ms/step | 20.0 samples/s +Step 45720/50462 | Loss: 2.5380 | LR: 1.11e-05 | 2999.9ms/step | 20.0 samples/s +Step 45730/50462 | Loss: 2.5214 | LR: 1.11e-05 | 2999.9ms/step | 20.0 samples/s +Step 45740/50462 | Loss: 2.5346 | LR: 1.10e-05 | 3000.3ms/step | 20.0 samples/s +Step 45750/50462 | Loss: 2.5518 | LR: 1.10e-05 | 2999.7ms/step | 20.0 samples/s +Validation at step 45750: Loss = 2.5531, Perplexity = 12.85 +Step 45760/50462 | Loss: 2.5050 | LR: 1.10e-05 | 4592.1ms/step | 13.1 samples/s +Step 45770/50462 | Loss: 2.5390 | LR: 1.09e-05 | 3001.5ms/step | 20.0 samples/s +Step 45780/50462 | Loss: 2.5319 | LR: 1.09e-05 | 2998.2ms/step | 20.0 samples/s +Step 45790/50462 | Loss: 2.5682 | LR: 1.08e-05 | 3000.4ms/step | 20.0 samples/s +Step 45800/50462 | Loss: 2.4836 | LR: 1.08e-05 | 3000.2ms/step | 20.0 samples/s +Step 45810/50462 | Loss: 2.5423 | LR: 1.07e-05 | 2999.9ms/step | 20.0 samples/s +Step 45820/50462 | Loss: 2.5380 | LR: 1.07e-05 | 3000.5ms/step | 20.0 samples/s +Step 45830/50462 | Loss: 2.5458 | LR: 1.06e-05 | 3001.4ms/step | 20.0 samples/s +Step 45840/50462 | Loss: 2.5593 | LR: 1.06e-05 | 3000.4ms/step | 20.0 samples/s +Step 45850/50462 | Loss: 2.5420 | LR: 1.05e-05 | 3000.0ms/step | 20.0 samples/s +Step 45860/50462 | Loss: 2.5655 | LR: 1.05e-05 | 2999.9ms/step | 20.0 samples/s +Step 45870/50462 | Loss: 2.5511 | LR: 1.05e-05 | 3000.0ms/step | 20.0 samples/s +Step 45880/50462 | Loss: 2.5517 | LR: 1.04e-05 | 3000.1ms/step | 20.0 samples/s +Step 45890/50462 | Loss: 2.5149 | LR: 1.04e-05 | 3000.0ms/step | 20.0 samples/s +Step 45900/50462 | Loss: 2.5374 | LR: 1.03e-05 | 2999.1ms/step | 20.0 samples/s +Step 45910/50462 | Loss: 2.5672 | LR: 1.03e-05 | 2996.9ms/step | 20.0 samples/s +Step 45920/50462 | Loss: 2.5200 | LR: 1.02e-05 | 3000.0ms/step | 20.0 samples/s +Step 45930/50462 | Loss: 2.5328 | LR: 1.02e-05 | 3000.2ms/step | 20.0 samples/s +Step 45940/50462 | Loss: 2.5525 | LR: 1.01e-05 | 3000.0ms/step | 20.0 samples/s +Step 45950/50462 | Loss: 2.5378 | LR: 1.01e-05 | 2999.9ms/step | 20.0 samples/s +Step 45960/50462 | Loss: 2.5234 | LR: 1.01e-05 | 3001.9ms/step | 20.0 samples/s +Step 45970/50462 | Loss: 2.5311 | LR: 1.00e-05 | 2998.9ms/step | 20.0 samples/s +Step 45980/50462 | Loss: 2.5739 | LR: 9.96e-06 | 2999.4ms/step | 20.0 samples/s +Step 45990/50462 | Loss: 2.5269 | LR: 9.92e-06 | 4116.0ms/step | 14.6 samples/s +Step 46000/50462 | Loss: 2.5424 | LR: 9.87e-06 | 3617.7ms/step | 16.6 samples/s +Validation at step 46000: Loss = 2.5529, Perplexity = 12.84 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-46000 +Step 46010/50462 | Loss: 2.5447 | LR: 9.83e-06 | 5656.5ms/step | 10.6 samples/s +Step 46020/50462 | Loss: 2.5330 | LR: 9.79e-06 | 2991.6ms/step | 20.1 samples/s +Step 46030/50462 | Loss: 2.5550 | LR: 9.74e-06 | 2998.2ms/step | 20.0 samples/s +Step 46040/50462 | Loss: 2.5542 | LR: 9.70e-06 | 3000.2ms/step | 20.0 samples/s +Step 46050/50462 | Loss: 2.5372 | LR: 9.65e-06 | 3004.0ms/step | 20.0 samples/s +Step 46060/50462 | Loss: 2.5343 | LR: 9.61e-06 | 2998.0ms/step | 20.0 samples/s +Step 46070/50462 | Loss: 2.5518 | LR: 9.57e-06 | 2998.2ms/step | 20.0 samples/s +Step 46080/50462 | Loss: 2.5461 | LR: 9.52e-06 | 3000.1ms/step | 20.0 samples/s +Step 46090/50462 | Loss: 2.5158 | LR: 9.48e-06 | 2999.9ms/step | 20.0 samples/s +Step 46100/50462 | Loss: 2.5426 | LR: 9.44e-06 | 3000.1ms/step | 20.0 samples/s +Step 46110/50462 | Loss: 2.5383 | LR: 9.40e-06 | 3000.1ms/step | 20.0 samples/s +Step 46120/50462 | Loss: 2.5337 | LR: 9.35e-06 | 2999.9ms/step | 20.0 samples/s +Step 46130/50462 | Loss: 2.5357 | LR: 9.31e-06 | 3000.0ms/step | 20.0 samples/s +Step 46140/50462 | Loss: 2.5346 | LR: 9.27e-06 | 3000.1ms/step | 20.0 samples/s +Step 46150/50462 | Loss: 2.5320 | LR: 9.22e-06 | 2999.9ms/step | 20.0 samples/s +Step 46160/50462 | Loss: 2.5516 | LR: 9.18e-06 | 3000.4ms/step | 20.0 samples/s +Step 46170/50462 | Loss: 2.5511 | LR: 9.14e-06 | 2999.7ms/step | 20.0 samples/s +Step 46180/50462 | Loss: 2.5208 | LR: 9.10e-06 | 3000.0ms/step | 20.0 samples/s +Step 46190/50462 | Loss: 2.5222 | LR: 9.06e-06 | 3000.0ms/step | 20.0 samples/s +Step 46200/50462 | Loss: 2.5282 | LR: 9.01e-06 | 3000.3ms/step | 20.0 samples/s +Step 46210/50462 | Loss: 2.5471 | LR: 8.97e-06 | 2999.8ms/step | 20.0 samples/s +Step 46220/50462 | Loss: 2.5489 | LR: 8.93e-06 | 3000.1ms/step | 20.0 samples/s +Step 46230/50462 | Loss: 2.5539 | LR: 8.89e-06 | 2999.9ms/step | 20.0 samples/s +Step 46240/50462 | Loss: 2.5629 | LR: 8.85e-06 | 3000.2ms/step | 20.0 samples/s +Step 46250/50462 | Loss: 2.5604 | LR: 8.80e-06 | 2999.9ms/step | 20.0 samples/s +Validation at step 46250: Loss = 2.5529, Perplexity = 12.84 +Step 46260/50462 | Loss: 2.5615 | LR: 8.76e-06 | 4597.9ms/step | 13.0 samples/s +Step 46270/50462 | Loss: 2.5476 | LR: 8.72e-06 | 3000.1ms/step | 20.0 samples/s +Step 46280/50462 | Loss: 2.5794 | LR: 8.68e-06 | 2995.4ms/step | 20.0 samples/s +Step 46290/50462 | Loss: 2.5667 | LR: 8.64e-06 | 2996.6ms/step | 20.0 samples/s +Step 46300/50462 | Loss: 2.5358 | LR: 8.60e-06 | 2999.8ms/step | 20.0 samples/s +Step 46310/50462 | Loss: 2.5431 | LR: 8.56e-06 | 3000.5ms/step | 20.0 samples/s +Step 46320/50462 | Loss: 2.5693 | LR: 8.52e-06 | 2999.8ms/step | 20.0 samples/s +Step 46330/50462 | Loss: 2.5498 | LR: 8.47e-06 | 3000.0ms/step | 20.0 samples/s +Step 46340/50462 | Loss: 2.5494 | LR: 8.43e-06 | 3000.0ms/step | 20.0 samples/s +Step 46350/50462 | Loss: 2.5701 | LR: 8.39e-06 | 3000.1ms/step | 20.0 samples/s +Step 46360/50462 | Loss: 2.5333 | LR: 8.35e-06 | 2999.9ms/step | 20.0 samples/s +Step 46370/50462 | Loss: 2.5158 | LR: 8.31e-06 | 2998.5ms/step | 20.0 samples/s +Step 46380/50462 | Loss: 2.5433 | LR: 8.27e-06 | 2999.5ms/step | 20.0 samples/s +Step 46390/50462 | Loss: 2.5329 | LR: 8.23e-06 | 3000.2ms/step | 20.0 samples/s +Step 46400/50462 | Loss: 2.5248 | LR: 8.19e-06 | 3000.2ms/step | 20.0 samples/s +Step 46410/50462 | Loss: 2.5365 | LR: 8.15e-06 | 2994.8ms/step | 20.0 samples/s +Step 46420/50462 | Loss: 2.5351 | LR: 8.11e-06 | 2998.8ms/step | 20.0 samples/s +Step 46430/50462 | Loss: 2.5568 | LR: 8.07e-06 | 2998.3ms/step | 20.0 samples/s +Step 46440/50462 | Loss: 2.5391 | LR: 8.03e-06 | 2999.5ms/step | 20.0 samples/s +Step 46450/50462 | Loss: 2.5662 | LR: 7.99e-06 | 3000.6ms/step | 20.0 samples/s +Step 46460/50462 | Loss: 2.5436 | LR: 7.95e-06 | 2999.8ms/step | 20.0 samples/s +Step 46470/50462 | Loss: 2.5358 | LR: 7.91e-06 | 2999.9ms/step | 20.0 samples/s +Step 46480/50462 | Loss: 2.5296 | LR: 7.87e-06 | 3000.3ms/step | 20.0 samples/s +Step 46490/50462 | Loss: 2.5263 | LR: 7.83e-06 | 3000.1ms/step | 20.0 samples/s +Step 46500/50462 | Loss: 2.5459 | LR: 7.80e-06 | 2999.9ms/step | 20.0 samples/s +Validation at step 46500: Loss = 2.5526, Perplexity = 12.84 +Step 46510/50462 | Loss: 2.5368 | LR: 7.76e-06 | 4596.0ms/step | 13.1 samples/s +Step 46520/50462 | Loss: 2.5600 | LR: 7.72e-06 | 3000.0ms/step | 20.0 samples/s +Step 46530/50462 | Loss: 2.5407 | LR: 7.68e-06 | 2997.9ms/step | 20.0 samples/s +Step 46540/50462 | Loss: 2.5531 | LR: 7.64e-06 | 3000.3ms/step | 20.0 samples/s +Step 46550/50462 | Loss: 2.5163 | LR: 7.60e-06 | 2999.9ms/step | 20.0 samples/s +Step 46560/50462 | Loss: 2.5306 | LR: 7.56e-06 | 2999.9ms/step | 20.0 samples/s +Step 46570/50462 | Loss: 2.5429 | LR: 7.52e-06 | 3000.9ms/step | 20.0 samples/s +Step 46580/50462 | Loss: 2.5344 | LR: 7.49e-06 | 2999.2ms/step | 20.0 samples/s +Step 46590/50462 | Loss: 2.5267 | LR: 7.45e-06 | 3000.1ms/step | 20.0 samples/s +Step 46600/50462 | Loss: 2.5425 | LR: 7.41e-06 | 2999.3ms/step | 20.0 samples/s +Step 46610/50462 | Loss: 2.5408 | LR: 7.37e-06 | 3000.9ms/step | 20.0 samples/s +Step 46620/50462 | Loss: 2.5237 | LR: 7.33e-06 | 3001.6ms/step | 20.0 samples/s +Step 46630/50462 | Loss: 2.5373 | LR: 7.29e-06 | 2996.6ms/step | 20.0 samples/s +Step 46640/50462 | Loss: 2.5498 | LR: 7.26e-06 | 2997.6ms/step | 20.0 samples/s +Step 46650/50462 | Loss: 2.5430 | LR: 7.22e-06 | 2998.2ms/step | 20.0 samples/s +Step 46660/50462 | Loss: 2.5321 | LR: 7.18e-06 | 2995.8ms/step | 20.0 samples/s +Step 46670/50462 | Loss: 2.5469 | LR: 7.14e-06 | 2997.9ms/step | 20.0 samples/s +Step 46680/50462 | Loss: 2.5565 | LR: 7.11e-06 | 2996.0ms/step | 20.0 samples/s +Step 46690/50462 | Loss: 2.5722 | LR: 7.07e-06 | 3000.4ms/step | 20.0 samples/s +Step 46700/50462 | Loss: 2.5319 | LR: 7.03e-06 | 2997.5ms/step | 20.0 samples/s +Step 46710/50462 | Loss: 2.5666 | LR: 6.99e-06 | 2995.5ms/step | 20.0 samples/s +Step 46720/50462 | Loss: 2.5489 | LR: 6.96e-06 | 2998.7ms/step | 20.0 samples/s +Step 46730/50462 | Loss: 2.5287 | LR: 6.92e-06 | 2998.0ms/step | 20.0 samples/s +Step 46740/50462 | Loss: 2.5553 | LR: 6.88e-06 | 3000.5ms/step | 20.0 samples/s +Step 46750/50462 | Loss: 2.5384 | LR: 6.85e-06 | 2999.5ms/step | 20.0 samples/s +Validation at step 46750: Loss = 2.5527, Perplexity = 12.84 +Step 46760/50462 | Loss: 2.5527 | LR: 6.81e-06 | 4597.5ms/step | 13.1 samples/s +Step 46770/50462 | Loss: 2.5432 | LR: 6.77e-06 | 2999.0ms/step | 20.0 samples/s +Step 46780/50462 | Loss: 2.5668 | LR: 6.74e-06 | 2998.0ms/step | 20.0 samples/s +Step 46790/50462 | Loss: 2.5245 | LR: 6.70e-06 | 2995.9ms/step | 20.0 samples/s +Step 46800/50462 | Loss: 2.5629 | LR: 6.66e-06 | 2995.6ms/step | 20.0 samples/s +Step 46810/50462 | Loss: 2.5575 | LR: 6.63e-06 | 2996.4ms/step | 20.0 samples/s +Step 46820/50462 | Loss: 2.5227 | LR: 6.59e-06 | 2999.6ms/step | 20.0 samples/s +Step 46830/50462 | Loss: 2.5206 | LR: 6.56e-06 | 2992.3ms/step | 20.1 samples/s +Step 46840/50462 | Loss: 2.5464 | LR: 6.52e-06 | 2995.9ms/step | 20.0 samples/s +Step 46850/50462 | Loss: 2.5414 | LR: 6.48e-06 | 2996.1ms/step | 20.0 samples/s +Step 46860/50462 | Loss: 2.5420 | LR: 6.45e-06 | 2996.0ms/step | 20.0 samples/s +Step 46870/50462 | Loss: 2.5001 | LR: 6.41e-06 | 2996.3ms/step | 20.0 samples/s +Step 46880/50462 | Loss: 2.5365 | LR: 6.38e-06 | 2993.7ms/step | 20.0 samples/s +Step 46890/50462 | Loss: 2.5402 | LR: 6.34e-06 | 2993.7ms/step | 20.0 samples/s +Step 46900/50462 | Loss: 2.5342 | LR: 6.31e-06 | 2993.0ms/step | 20.0 samples/s +Step 46910/50462 | Loss: 2.5267 | LR: 6.27e-06 | 2991.5ms/step | 20.1 samples/s +Step 46920/50462 | Loss: 2.5648 | LR: 6.24e-06 | 2993.1ms/step | 20.0 samples/s +Step 46930/50462 | Loss: 2.5412 | LR: 6.20e-06 | 2993.6ms/step | 20.0 samples/s +Step 46940/50462 | Loss: 2.5432 | LR: 6.17e-06 | 2993.3ms/step | 20.0 samples/s +Step 46950/50462 | Loss: 2.5269 | LR: 6.13e-06 | 2994.0ms/step | 20.0 samples/s +Step 46960/50462 | Loss: 2.5509 | LR: 6.10e-06 | 2991.8ms/step | 20.1 samples/s +Step 46970/50462 | Loss: 2.5524 | LR: 6.06e-06 | 2992.3ms/step | 20.1 samples/s +Step 46980/50462 | Loss: 2.5511 | LR: 6.03e-06 | 2995.9ms/step | 20.0 samples/s +Step 46990/50462 | Loss: 2.5724 | LR: 5.99e-06 | 2995.8ms/step | 20.0 samples/s +Step 47000/50462 | Loss: 2.5515 | LR: 5.96e-06 | 2994.1ms/step | 20.0 samples/s +Validation at step 47000: Loss = 2.5526, Perplexity = 12.84 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-47000 +Step 47010/50462 | Loss: 2.5644 | LR: 5.93e-06 | 5653.6ms/step | 10.6 samples/s +Step 47020/50462 | Loss: 2.5409 | LR: 5.89e-06 | 2995.0ms/step | 20.0 samples/s +Step 47030/50462 | Loss: 2.5609 | LR: 5.86e-06 | 3000.1ms/step | 20.0 samples/s +Step 47040/50462 | Loss: 2.5054 | LR: 5.82e-06 | 2995.6ms/step | 20.0 samples/s +Step 47050/50462 | Loss: 2.5447 | LR: 5.79e-06 | 2993.8ms/step | 20.0 samples/s +Step 47060/50462 | Loss: 2.5414 | LR: 5.76e-06 | 2998.7ms/step | 20.0 samples/s +Step 47070/50462 | Loss: 2.5327 | LR: 5.72e-06 | 2994.8ms/step | 20.0 samples/s +Step 47080/50462 | Loss: 2.5425 | LR: 5.69e-06 | 2996.5ms/step | 20.0 samples/s +Step 47090/50462 | Loss: 2.5373 | LR: 5.65e-06 | 2995.6ms/step | 20.0 samples/s +Step 47100/50462 | Loss: 2.5407 | LR: 5.62e-06 | 2992.5ms/step | 20.1 samples/s +Step 47110/50462 | Loss: 2.5443 | LR: 5.59e-06 | 2994.7ms/step | 20.0 samples/s +Step 47120/50462 | Loss: 2.4941 | LR: 5.56e-06 | 2994.9ms/step | 20.0 samples/s +Step 47130/50462 | Loss: 2.5411 | LR: 5.52e-06 | 2996.5ms/step | 20.0 samples/s +Step 47140/50462 | Loss: 2.5541 | LR: 5.49e-06 | 2992.5ms/step | 20.0 samples/s +Step 47150/50462 | Loss: 2.5119 | LR: 5.46e-06 | 2993.4ms/step | 20.0 samples/s +Step 47160/50462 | Loss: 2.5537 | LR: 5.42e-06 | 2992.0ms/step | 20.1 samples/s +Step 47170/50462 | Loss: 2.5357 | LR: 5.39e-06 | 2996.6ms/step | 20.0 samples/s +Step 47180/50462 | Loss: 2.5458 | LR: 5.36e-06 | 2999.8ms/step | 20.0 samples/s +Step 47190/50462 | Loss: 2.5399 | LR: 5.33e-06 | 3000.5ms/step | 20.0 samples/s +Step 47200/50462 | Loss: 2.5367 | LR: 5.29e-06 | 2989.7ms/step | 20.1 samples/s +Step 47210/50462 | Loss: 2.5108 | LR: 5.26e-06 | 2994.3ms/step | 20.0 samples/s +Step 47220/50462 | Loss: 2.5522 | LR: 5.23e-06 | 2997.2ms/step | 20.0 samples/s +Step 47230/50462 | Loss: 2.5499 | LR: 5.20e-06 | 2993.7ms/step | 20.0 samples/s +Step 47240/50462 | Loss: 2.5176 | LR: 5.16e-06 | 2997.1ms/step | 20.0 samples/s +Step 47250/50462 | Loss: 2.5497 | LR: 5.13e-06 | 2997.8ms/step | 20.0 samples/s +Validation at step 47250: Loss = 2.5530, Perplexity = 12.85 +Step 47260/50462 | Loss: 2.5425 | LR: 5.10e-06 | 4589.3ms/step | 13.1 samples/s +Step 47270/50462 | Loss: 2.5549 | LR: 5.07e-06 | 2997.0ms/step | 20.0 samples/s +Step 47280/50462 | Loss: 2.5296 | LR: 5.04e-06 | 2995.7ms/step | 20.0 samples/s +Step 47290/50462 | Loss: 2.5568 | LR: 5.01e-06 | 2997.8ms/step | 20.0 samples/s +Step 47300/50462 | Loss: 2.5564 | LR: 4.97e-06 | 2994.5ms/step | 20.0 samples/s +Step 47310/50462 | Loss: 2.5173 | LR: 4.94e-06 | 2998.2ms/step | 20.0 samples/s +Step 47320/50462 | Loss: 2.5336 | LR: 4.91e-06 | 2993.5ms/step | 20.0 samples/s +Step 47330/50462 | Loss: 2.5369 | LR: 4.88e-06 | 2994.5ms/step | 20.0 samples/s +Step 47340/50462 | Loss: 2.5019 | LR: 4.85e-06 | 2997.6ms/step | 20.0 samples/s +Step 47350/50462 | Loss: 2.5510 | LR: 4.82e-06 | 2996.2ms/step | 20.0 samples/s +Step 47360/50462 | Loss: 2.5198 | LR: 4.79e-06 | 2997.9ms/step | 20.0 samples/s +Step 47370/50462 | Loss: 2.5587 | LR: 4.76e-06 | 2998.1ms/step | 20.0 samples/s +Step 47380/50462 | Loss: 2.5516 | LR: 4.73e-06 | 2989.2ms/step | 20.1 samples/s +Step 47390/50462 | Loss: 2.5346 | LR: 4.70e-06 | 2993.0ms/step | 20.0 samples/s +Step 47400/50462 | Loss: 2.5368 | LR: 4.67e-06 | 2992.6ms/step | 20.0 samples/s +Step 47410/50462 | Loss: 2.5496 | LR: 4.64e-06 | 2990.9ms/step | 20.1 samples/s +Step 47420/50462 | Loss: 2.4998 | LR: 4.61e-06 | 2991.6ms/step | 20.1 samples/s +Step 47430/50462 | Loss: 2.5276 | LR: 4.58e-06 | 2991.0ms/step | 20.1 samples/s +Step 47440/50462 | Loss: 2.5362 | LR: 4.55e-06 | 2993.9ms/step | 20.0 samples/s +Step 47450/50462 | Loss: 2.5105 | LR: 4.52e-06 | 2994.4ms/step | 20.0 samples/s +Step 47460/50462 | Loss: 2.5555 | LR: 4.49e-06 | 2993.8ms/step | 20.0 samples/s +Step 47470/50462 | Loss: 2.5543 | LR: 4.46e-06 | 2995.9ms/step | 20.0 samples/s +Step 47480/50462 | Loss: 2.5519 | LR: 4.43e-06 | 2993.3ms/step | 20.0 samples/s +Step 47490/50462 | Loss: 2.5456 | LR: 4.40e-06 | 2992.8ms/step | 20.0 samples/s +Step 47500/50462 | Loss: 2.5367 | LR: 4.37e-06 | 2994.2ms/step | 20.0 samples/s +Validation at step 47500: Loss = 2.5530, Perplexity = 12.85 +Step 47510/50462 | Loss: 2.5073 | LR: 4.34e-06 | 4594.1ms/step | 13.1 samples/s +Step 47520/50462 | Loss: 2.5557 | LR: 4.31e-06 | 2993.6ms/step | 20.0 samples/s +Step 47530/50462 | Loss: 2.5515 | LR: 4.28e-06 | 2994.6ms/step | 20.0 samples/s +Step 47540/50462 | Loss: 2.5191 | LR: 4.25e-06 | 2991.6ms/step | 20.1 samples/s +Step 47550/50462 | Loss: 2.5487 | LR: 4.22e-06 | 2989.3ms/step | 20.1 samples/s +Step 47560/50462 | Loss: 2.5331 | LR: 4.19e-06 | 2997.8ms/step | 20.0 samples/s +Step 47570/50462 | Loss: 2.5436 | LR: 4.16e-06 | 2995.3ms/step | 20.0 samples/s +Step 47580/50462 | Loss: 2.5794 | LR: 4.13e-06 | 2998.0ms/step | 20.0 samples/s +Step 47590/50462 | Loss: 2.5496 | LR: 4.11e-06 | 2993.3ms/step | 20.0 samples/s +Step 47600/50462 | Loss: 2.5406 | LR: 4.08e-06 | 2992.0ms/step | 20.1 samples/s +Step 47610/50462 | Loss: 2.5206 | LR: 4.05e-06 | 2994.8ms/step | 20.0 samples/s +Step 47620/50462 | Loss: 2.5327 | LR: 4.02e-06 | 2991.3ms/step | 20.1 samples/s +Step 47630/50462 | Loss: 2.5618 | LR: 3.99e-06 | 2992.1ms/step | 20.1 samples/s +Step 47640/50462 | Loss: 2.5168 | LR: 3.97e-06 | 2992.7ms/step | 20.0 samples/s +Step 47650/50462 | Loss: 2.5376 | LR: 3.94e-06 | 2990.5ms/step | 20.1 samples/s +Step 47660/50462 | Loss: 2.5346 | LR: 3.91e-06 | 2993.3ms/step | 20.0 samples/s +Step 47670/50462 | Loss: 2.5289 | LR: 3.88e-06 | 2994.9ms/step | 20.0 samples/s +Step 47680/50462 | Loss: 2.5456 | LR: 3.85e-06 | 2994.2ms/step | 20.0 samples/s +Step 47690/50462 | Loss: 2.5414 | LR: 3.83e-06 | 2994.3ms/step | 20.0 samples/s +Step 47700/50462 | Loss: 2.5503 | LR: 3.80e-06 | 2999.0ms/step | 20.0 samples/s +Step 47710/50462 | Loss: 2.5153 | LR: 3.77e-06 | 2999.7ms/step | 20.0 samples/s +Step 47720/50462 | Loss: 2.5434 | LR: 3.74e-06 | 2999.5ms/step | 20.0 samples/s +Step 47730/50462 | Loss: 2.5420 | LR: 3.72e-06 | 3000.7ms/step | 20.0 samples/s +Step 47740/50462 | Loss: 2.5276 | LR: 3.69e-06 | 2999.9ms/step | 20.0 samples/s +Step 47750/50462 | Loss: 2.5555 | LR: 3.66e-06 | 2998.0ms/step | 20.0 samples/s +Validation at step 47750: Loss = 2.5531, Perplexity = 12.85 +Step 47760/50462 | Loss: 2.5124 | LR: 3.64e-06 | 4591.5ms/step | 13.1 samples/s +Step 47770/50462 | Loss: 2.5744 | LR: 3.61e-06 | 2994.5ms/step | 20.0 samples/s +Step 47780/50462 | Loss: 2.5334 | LR: 3.58e-06 | 2994.0ms/step | 20.0 samples/s +Step 47790/50462 | Loss: 2.5273 | LR: 3.56e-06 | 2999.3ms/step | 20.0 samples/s +Step 47800/50462 | Loss: 2.5190 | LR: 3.53e-06 | 2992.7ms/step | 20.0 samples/s +Step 47810/50462 | Loss: 2.5507 | LR: 3.50e-06 | 2994.6ms/step | 20.0 samples/s +Step 47820/50462 | Loss: 2.5467 | LR: 3.48e-06 | 2991.8ms/step | 20.1 samples/s +Step 47830/50462 | Loss: 2.5385 | LR: 3.45e-06 | 2997.2ms/step | 20.0 samples/s +Step 47840/50462 | Loss: 2.5598 | LR: 3.42e-06 | 2996.7ms/step | 20.0 samples/s +Step 47850/50462 | Loss: 2.5511 | LR: 3.40e-06 | 3000.0ms/step | 20.0 samples/s +Step 47860/50462 | Loss: 2.5541 | LR: 3.37e-06 | 2995.9ms/step | 20.0 samples/s +Step 47870/50462 | Loss: 2.5472 | LR: 3.35e-06 | 2998.4ms/step | 20.0 samples/s +Step 47880/50462 | Loss: 2.5021 | LR: 3.32e-06 | 2997.9ms/step | 20.0 samples/s +Step 47890/50462 | Loss: 2.5604 | LR: 3.30e-06 | 2997.8ms/step | 20.0 samples/s +Step 47900/50462 | Loss: 2.5496 | LR: 3.27e-06 | 2998.4ms/step | 20.0 samples/s +Step 47910/50462 | Loss: 2.5477 | LR: 3.24e-06 | 2996.5ms/step | 20.0 samples/s +Step 47920/50462 | Loss: 2.5220 | LR: 3.22e-06 | 2995.5ms/step | 20.0 samples/s +Step 47930/50462 | Loss: 2.5603 | LR: 3.19e-06 | 2998.0ms/step | 20.0 samples/s +Step 47940/50462 | Loss: 2.5336 | LR: 3.17e-06 | 2999.0ms/step | 20.0 samples/s +Step 47950/50462 | Loss: 2.5416 | LR: 3.14e-06 | 2998.8ms/step | 20.0 samples/s +Step 47960/50462 | Loss: 2.5090 | LR: 3.12e-06 | 2993.8ms/step | 20.0 samples/s +Step 47970/50462 | Loss: 2.5442 | LR: 3.09e-06 | 2996.3ms/step | 20.0 samples/s +Step 47980/50462 | Loss: 2.5430 | LR: 3.07e-06 | 2994.1ms/step | 20.0 samples/s +Step 47990/50462 | Loss: 2.5258 | LR: 3.04e-06 | 2995.9ms/step | 20.0 samples/s +Step 48000/50462 | Loss: 2.5347 | LR: 3.02e-06 | 2998.2ms/step | 20.0 samples/s +Validation at step 48000: Loss = 2.5527, Perplexity = 12.84 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-48000 +Step 48010/50462 | Loss: 2.5475 | LR: 3.00e-06 | 5655.6ms/step | 10.6 samples/s +Step 48020/50462 | Loss: 2.5764 | LR: 2.97e-06 | 2992.7ms/step | 20.0 samples/s +Step 48030/50462 | Loss: 2.5186 | LR: 2.95e-06 | 2995.1ms/step | 20.0 samples/s +Step 48040/50462 | Loss: 2.5315 | LR: 2.92e-06 | 2992.7ms/step | 20.0 samples/s +Step 48050/50462 | Loss: 2.5256 | LR: 2.90e-06 | 2994.1ms/step | 20.0 samples/s +Step 48060/50462 | Loss: 2.5413 | LR: 2.87e-06 | 2994.0ms/step | 20.0 samples/s +Step 48070/50462 | Loss: 2.5442 | LR: 2.85e-06 | 2992.1ms/step | 20.1 samples/s +Step 48080/50462 | Loss: 2.5673 | LR: 2.83e-06 | 2993.7ms/step | 20.0 samples/s +Step 48090/50462 | Loss: 2.5450 | LR: 2.80e-06 | 2992.0ms/step | 20.1 samples/s +Step 48100/50462 | Loss: 2.5468 | LR: 2.78e-06 | 3546.7ms/step | 16.9 samples/s +Step 48110/50462 | Loss: 2.5371 | LR: 2.76e-06 | 3678.9ms/step | 16.3 samples/s +Step 48120/50462 | Loss: 2.5351 | LR: 2.73e-06 | 2984.5ms/step | 20.1 samples/s +Step 48130/50462 | Loss: 2.5356 | LR: 2.71e-06 | 3564.9ms/step | 16.8 samples/s +Step 48140/50462 | Loss: 2.5173 | LR: 2.69e-06 | 2993.3ms/step | 20.0 samples/s +Step 48150/50462 | Loss: 2.5315 | LR: 2.66e-06 | 2993.4ms/step | 20.0 samples/s +Step 48160/50462 | Loss: 2.5507 | LR: 2.64e-06 | 2992.7ms/step | 20.0 samples/s +Step 48170/50462 | Loss: 2.5325 | LR: 2.62e-06 | 2994.0ms/step | 20.0 samples/s +Step 48180/50462 | Loss: 2.5428 | LR: 2.60e-06 | 2995.9ms/step | 20.0 samples/s +Step 48190/50462 | Loss: 2.5374 | LR: 2.57e-06 | 2993.7ms/step | 20.0 samples/s +Step 48200/50462 | Loss: 2.5475 | LR: 2.55e-06 | 2996.3ms/step | 20.0 samples/s +Step 48210/50462 | Loss: 2.5245 | LR: 2.53e-06 | 2997.5ms/step | 20.0 samples/s +Step 48220/50462 | Loss: 2.5312 | LR: 2.51e-06 | 2999.8ms/step | 20.0 samples/s +Step 48230/50462 | Loss: 2.5384 | LR: 2.48e-06 | 2993.9ms/step | 20.0 samples/s +Step 48240/50462 | Loss: 2.5359 | LR: 2.46e-06 | 2996.5ms/step | 20.0 samples/s +Step 48250/50462 | Loss: 2.5748 | LR: 2.44e-06 | 2992.6ms/step | 20.0 samples/s +Validation at step 48250: Loss = 2.5529, Perplexity = 12.84 +Step 48260/50462 | Loss: 2.5127 | LR: 2.42e-06 | 4587.9ms/step | 13.1 samples/s +Step 48270/50462 | Loss: 2.5381 | LR: 2.39e-06 | 2992.3ms/step | 20.1 samples/s +Step 48280/50462 | Loss: 2.5472 | LR: 2.37e-06 | 2993.8ms/step | 20.0 samples/s +Step 48290/50462 | Loss: 2.5338 | LR: 2.35e-06 | 2998.2ms/step | 20.0 samples/s +Step 48300/50462 | Loss: 2.5265 | LR: 2.33e-06 | 2995.3ms/step | 20.0 samples/s +Step 48310/50462 | Loss: 2.5481 | LR: 2.31e-06 | 3000.4ms/step | 20.0 samples/s +Step 48320/50462 | Loss: 2.5457 | LR: 2.29e-06 | 2995.7ms/step | 20.0 samples/s +Step 48330/50462 | Loss: 2.5291 | LR: 2.27e-06 | 2996.2ms/step | 20.0 samples/s +Step 48340/50462 | Loss: 2.5522 | LR: 2.24e-06 | 3000.2ms/step | 20.0 samples/s +Step 48350/50462 | Loss: 2.5662 | LR: 2.22e-06 | 3000.3ms/step | 20.0 samples/s +Step 48360/50462 | Loss: 2.5224 | LR: 2.20e-06 | 2991.8ms/step | 20.1 samples/s +Step 48370/50462 | Loss: 2.5593 | LR: 2.18e-06 | 2993.9ms/step | 20.0 samples/s +Step 48380/50462 | Loss: 2.5268 | LR: 2.16e-06 | 2991.9ms/step | 20.1 samples/s +Step 48390/50462 | Loss: 2.5729 | LR: 2.14e-06 | 2992.6ms/step | 20.0 samples/s +Step 48400/50462 | Loss: 2.5292 | LR: 2.12e-06 | 2995.6ms/step | 20.0 samples/s +Step 48410/50462 | Loss: 2.5566 | LR: 2.10e-06 | 2996.2ms/step | 20.0 samples/s +Step 48420/50462 | Loss: 2.5346 | LR: 2.08e-06 | 2990.4ms/step | 20.1 samples/s +Step 48430/50462 | Loss: 2.5312 | LR: 2.06e-06 | 2991.4ms/step | 20.1 samples/s +Step 48440/50462 | Loss: 2.5481 | LR: 2.04e-06 | 2994.3ms/step | 20.0 samples/s +Step 48450/50462 | Loss: 2.5140 | LR: 2.02e-06 | 2995.4ms/step | 20.0 samples/s +Step 48460/50462 | Loss: 2.5620 | LR: 2.00e-06 | 2994.0ms/step | 20.0 samples/s +Step 48470/50462 | Loss: 2.5493 | LR: 1.98e-06 | 2994.3ms/step | 20.0 samples/s +Step 48480/50462 | Loss: 2.5315 | LR: 1.96e-06 | 2997.9ms/step | 20.0 samples/s +Step 48490/50462 | Loss: 2.5402 | LR: 1.94e-06 | 2998.3ms/step | 20.0 samples/s +Step 48500/50462 | Loss: 2.5087 | LR: 1.92e-06 | 2998.0ms/step | 20.0 samples/s +Validation at step 48500: Loss = 2.5532, Perplexity = 12.85 +Step 48510/50462 | Loss: 2.5428 | LR: 1.90e-06 | 4594.1ms/step | 13.1 samples/s +Step 48520/50462 | Loss: 2.5427 | LR: 1.88e-06 | 3000.0ms/step | 20.0 samples/s +Step 48530/50462 | Loss: 2.5598 | LR: 1.86e-06 | 2999.6ms/step | 20.0 samples/s +Step 48540/50462 | Loss: 2.5156 | LR: 1.84e-06 | 2997.2ms/step | 20.0 samples/s +Step 48550/50462 | Loss: 2.5251 | LR: 1.82e-06 | 2993.3ms/step | 20.0 samples/s +Step 48560/50462 | Loss: 2.5356 | LR: 1.80e-06 | 2994.0ms/step | 20.0 samples/s +Step 48570/50462 | Loss: 2.5371 | LR: 1.78e-06 | 2992.0ms/step | 20.1 samples/s +Step 48580/50462 | Loss: 2.5305 | LR: 1.77e-06 | 2990.7ms/step | 20.1 samples/s +Step 48590/50462 | Loss: 2.5549 | LR: 1.75e-06 | 2993.6ms/step | 20.0 samples/s +Step 48600/50462 | Loss: 2.5462 | LR: 1.73e-06 | 2991.6ms/step | 20.1 samples/s +Step 48610/50462 | Loss: 2.5323 | LR: 1.71e-06 | 2990.1ms/step | 20.1 samples/s +Step 48620/50462 | Loss: 2.5358 | LR: 1.69e-06 | 2993.6ms/step | 20.0 samples/s +Step 48630/50462 | Loss: 2.5588 | LR: 1.67e-06 | 2993.9ms/step | 20.0 samples/s +Step 48640/50462 | Loss: 2.5410 | LR: 1.66e-06 | 2990.3ms/step | 20.1 samples/s +Step 48650/50462 | Loss: 2.5593 | LR: 1.64e-06 | 2998.3ms/step | 20.0 samples/s +Step 48660/50462 | Loss: 2.5535 | LR: 1.62e-06 | 2995.5ms/step | 20.0 samples/s +Step 48670/50462 | Loss: 2.5553 | LR: 1.60e-06 | 2996.2ms/step | 20.0 samples/s +Step 48680/50462 | Loss: 2.5361 | LR: 1.58e-06 | 2993.9ms/step | 20.0 samples/s +Step 48690/50462 | Loss: 2.5508 | LR: 1.57e-06 | 2994.7ms/step | 20.0 samples/s +Step 48700/50462 | Loss: 2.5120 | LR: 1.55e-06 | 2993.7ms/step | 20.0 samples/s +Step 48710/50462 | Loss: 2.5348 | LR: 1.53e-06 | 2994.5ms/step | 20.0 samples/s +Step 48720/50462 | Loss: 2.5544 | LR: 1.51e-06 | 2993.4ms/step | 20.0 samples/s +Step 48730/50462 | Loss: 2.5401 | LR: 1.50e-06 | 2993.6ms/step | 20.0 samples/s +Step 48740/50462 | Loss: 2.5185 | LR: 1.48e-06 | 2997.0ms/step | 20.0 samples/s +Step 48750/50462 | Loss: 2.5616 | LR: 1.46e-06 | 2996.2ms/step | 20.0 samples/s +Validation at step 48750: Loss = 2.5530, Perplexity = 12.85 +Step 48760/50462 | Loss: 2.5722 | LR: 1.44e-06 | 4593.6ms/step | 13.1 samples/s +Step 48770/50462 | Loss: 2.5129 | LR: 1.43e-06 | 2998.6ms/step | 20.0 samples/s +Step 48780/50462 | Loss: 2.5353 | LR: 1.41e-06 | 2993.1ms/step | 20.0 samples/s +Step 48790/50462 | Loss: 2.5446 | LR: 1.39e-06 | 2998.8ms/step | 20.0 samples/s +Step 48800/50462 | Loss: 2.5227 | LR: 1.38e-06 | 2997.8ms/step | 20.0 samples/s +Step 48810/50462 | Loss: 2.5324 | LR: 1.36e-06 | 2995.7ms/step | 20.0 samples/s +Step 48820/50462 | Loss: 2.5372 | LR: 1.34e-06 | 2996.0ms/step | 20.0 samples/s +Step 48830/50462 | Loss: 2.5535 | LR: 1.33e-06 | 3000.5ms/step | 20.0 samples/s +Step 48840/50462 | Loss: 2.5666 | LR: 1.31e-06 | 2999.3ms/step | 20.0 samples/s +Step 48850/50462 | Loss: 2.5369 | LR: 1.30e-06 | 2994.3ms/step | 20.0 samples/s +Step 48860/50462 | Loss: 2.5052 | LR: 1.28e-06 | 2996.4ms/step | 20.0 samples/s +Step 48870/50462 | Loss: 2.5481 | LR: 1.26e-06 | 2991.9ms/step | 20.1 samples/s +Step 48880/50462 | Loss: 2.5299 | LR: 1.25e-06 | 2996.5ms/step | 20.0 samples/s +Step 48890/50462 | Loss: 2.5601 | LR: 1.23e-06 | 2995.6ms/step | 20.0 samples/s +Step 48900/50462 | Loss: 2.5377 | LR: 1.22e-06 | 2996.2ms/step | 20.0 samples/s +Step 48910/50462 | Loss: 2.5522 | LR: 1.20e-06 | 2999.8ms/step | 20.0 samples/s +Step 48920/50462 | Loss: 2.5396 | LR: 1.19e-06 | 2993.5ms/step | 20.0 samples/s +Step 48930/50462 | Loss: 2.5358 | LR: 1.17e-06 | 2996.7ms/step | 20.0 samples/s +Step 48940/50462 | Loss: 2.5156 | LR: 1.16e-06 | 2990.0ms/step | 20.1 samples/s +Step 48950/50462 | Loss: 2.5163 | LR: 1.14e-06 | 2999.9ms/step | 20.0 samples/s +Step 48960/50462 | Loss: 2.5490 | LR: 1.13e-06 | 2998.2ms/step | 20.0 samples/s +Step 48970/50462 | Loss: 2.5149 | LR: 1.11e-06 | 2996.9ms/step | 20.0 samples/s +Step 48980/50462 | Loss: 2.5826 | LR: 1.10e-06 | 2995.3ms/step | 20.0 samples/s +Step 48990/50462 | Loss: 2.5663 | LR: 1.08e-06 | 2995.7ms/step | 20.0 samples/s +Step 49000/50462 | Loss: 2.5499 | LR: 1.07e-06 | 2994.8ms/step | 20.0 samples/s +Validation at step 49000: Loss = 2.5530, Perplexity = 12.85 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-49000 +Step 49010/50462 | Loss: 2.5380 | LR: 1.05e-06 | 5646.7ms/step | 10.6 samples/s +Step 49020/50462 | Loss: 2.5451 | LR: 1.04e-06 | 2989.9ms/step | 20.1 samples/s +Step 49030/50462 | Loss: 2.5462 | LR: 1.02e-06 | 2992.8ms/step | 20.0 samples/s +Step 49040/50462 | Loss: 2.5429 | LR: 1.01e-06 | 3000.0ms/step | 20.0 samples/s +Step 49050/50462 | Loss: 2.5433 | LR: 9.95e-07 | 2995.7ms/step | 20.0 samples/s +Step 49060/50462 | Loss: 2.5632 | LR: 9.81e-07 | 3000.1ms/step | 20.0 samples/s +Step 49070/50462 | Loss: 2.5603 | LR: 9.67e-07 | 3000.8ms/step | 20.0 samples/s +Step 49080/50462 | Loss: 2.5632 | LR: 9.53e-07 | 2998.8ms/step | 20.0 samples/s +Step 49090/50462 | Loss: 2.5171 | LR: 9.39e-07 | 2992.8ms/step | 20.0 samples/s +Step 49100/50462 | Loss: 2.5658 | LR: 9.25e-07 | 2995.4ms/step | 20.0 samples/s +Step 49110/50462 | Loss: 2.5497 | LR: 9.12e-07 | 2994.4ms/step | 20.0 samples/s +Step 49120/50462 | Loss: 2.5200 | LR: 8.99e-07 | 2992.0ms/step | 20.1 samples/s +Step 49130/50462 | Loss: 2.5491 | LR: 8.85e-07 | 2993.7ms/step | 20.0 samples/s +Step 49140/50462 | Loss: 2.5206 | LR: 8.72e-07 | 2994.3ms/step | 20.0 samples/s +Step 49150/50462 | Loss: 2.5527 | LR: 8.59e-07 | 2990.3ms/step | 20.1 samples/s +Step 49160/50462 | Loss: 2.5422 | LR: 8.46e-07 | 2990.3ms/step | 20.1 samples/s +Step 49170/50462 | Loss: 2.5400 | LR: 8.33e-07 | 2992.9ms/step | 20.0 samples/s +Step 49180/50462 | Loss: 2.5303 | LR: 8.20e-07 | 2996.2ms/step | 20.0 samples/s +Step 49190/50462 | Loss: 2.5177 | LR: 8.07e-07 | 2996.1ms/step | 20.0 samples/s +Step 49200/50462 | Loss: 2.5737 | LR: 7.95e-07 | 2993.5ms/step | 20.0 samples/s +Step 49210/50462 | Loss: 2.5431 | LR: 7.82e-07 | 2993.0ms/step | 20.0 samples/s +Step 49220/50462 | Loss: 2.5526 | LR: 7.70e-07 | 2998.4ms/step | 20.0 samples/s +Step 49230/50462 | Loss: 2.5559 | LR: 7.57e-07 | 2995.7ms/step | 20.0 samples/s +Step 49240/50462 | Loss: 2.5452 | LR: 7.45e-07 | 2996.1ms/step | 20.0 samples/s +Step 49250/50462 | Loss: 2.5401 | LR: 7.33e-07 | 2994.2ms/step | 20.0 samples/s +Validation at step 49250: Loss = 2.5526, Perplexity = 12.84 +Step 49260/50462 | Loss: 2.5112 | LR: 7.21e-07 | 4589.0ms/step | 13.1 samples/s +Step 49270/50462 | Loss: 2.5319 | LR: 7.09e-07 | 2995.9ms/step | 20.0 samples/s +Step 49280/50462 | Loss: 2.5140 | LR: 6.97e-07 | 2996.6ms/step | 20.0 samples/s +Step 49290/50462 | Loss: 2.5371 | LR: 6.85e-07 | 2993.8ms/step | 20.0 samples/s +Step 49300/50462 | Loss: 2.5346 | LR: 6.74e-07 | 2996.4ms/step | 20.0 samples/s +Step 49310/50462 | Loss: 2.5407 | LR: 6.62e-07 | 2993.6ms/step | 20.0 samples/s +Step 49320/50462 | Loss: 2.5283 | LR: 6.51e-07 | 2995.0ms/step | 20.0 samples/s +Step 49330/50462 | Loss: 2.5375 | LR: 6.39e-07 | 2995.1ms/step | 20.0 samples/s +Step 49340/50462 | Loss: 2.5452 | LR: 6.28e-07 | 2992.4ms/step | 20.1 samples/s +Step 49350/50462 | Loss: 2.5504 | LR: 6.17e-07 | 2996.0ms/step | 20.0 samples/s +Step 49360/50462 | Loss: 2.5611 | LR: 6.06e-07 | 2993.8ms/step | 20.0 samples/s +Step 49370/50462 | Loss: 2.5435 | LR: 5.95e-07 | 2992.7ms/step | 20.0 samples/s +Step 49380/50462 | Loss: 2.5554 | LR: 5.84e-07 | 2990.9ms/step | 20.1 samples/s +Step 49390/50462 | Loss: 2.5545 | LR: 5.73e-07 | 2994.1ms/step | 20.0 samples/s +Step 49400/50462 | Loss: 2.5518 | LR: 5.63e-07 | 2993.5ms/step | 20.0 samples/s +Step 49410/50462 | Loss: 2.5457 | LR: 5.52e-07 | 2997.7ms/step | 20.0 samples/s +Step 49420/50462 | Loss: 2.5422 | LR: 5.42e-07 | 2995.2ms/step | 20.0 samples/s +Step 49430/50462 | Loss: 2.5220 | LR: 5.31e-07 | 2992.7ms/step | 20.0 samples/s +Step 49440/50462 | Loss: 2.5315 | LR: 5.21e-07 | 2995.3ms/step | 20.0 samples/s +Step 49450/50462 | Loss: 2.5178 | LR: 5.11e-07 | 2994.7ms/step | 20.0 samples/s +Step 49460/50462 | Loss: 2.5522 | LR: 5.01e-07 | 3000.0ms/step | 20.0 samples/s +Step 49470/50462 | Loss: 2.5517 | LR: 4.91e-07 | 2999.6ms/step | 20.0 samples/s +Step 49480/50462 | Loss: 2.5472 | LR: 4.81e-07 | 2996.1ms/step | 20.0 samples/s +Step 49490/50462 | Loss: 2.5285 | LR: 4.72e-07 | 2998.4ms/step | 20.0 samples/s +Step 49500/50462 | Loss: 2.5475 | LR: 4.62e-07 | 2997.8ms/step | 20.0 samples/s +Validation at step 49500: Loss = 2.5525, Perplexity = 12.84 +Step 49510/50462 | Loss: 2.5574 | LR: 4.52e-07 | 4594.2ms/step | 13.1 samples/s +Step 49520/50462 | Loss: 2.5367 | LR: 4.43e-07 | 3000.2ms/step | 20.0 samples/s +Step 49530/50462 | Loss: 2.5408 | LR: 4.34e-07 | 3001.9ms/step | 20.0 samples/s +Step 49540/50462 | Loss: 2.5319 | LR: 4.24e-07 | 3000.1ms/step | 20.0 samples/s +Step 49550/50462 | Loss: 2.5397 | LR: 4.15e-07 | 2999.8ms/step | 20.0 samples/s +Step 49560/50462 | Loss: 2.5559 | LR: 4.06e-07 | 3000.3ms/step | 20.0 samples/s +Step 49570/50462 | Loss: 2.5577 | LR: 3.97e-07 | 3000.3ms/step | 20.0 samples/s +Step 49580/50462 | Loss: 2.5555 | LR: 3.88e-07 | 2996.9ms/step | 20.0 samples/s +Step 49590/50462 | Loss: 2.5299 | LR: 3.79e-07 | 2997.0ms/step | 20.0 samples/s +Step 49600/50462 | Loss: 2.5611 | LR: 3.71e-07 | 2999.9ms/step | 20.0 samples/s +Step 49610/50462 | Loss: 2.5401 | LR: 3.62e-07 | 3000.1ms/step | 20.0 samples/s +Step 49620/50462 | Loss: 2.5411 | LR: 3.54e-07 | 2999.6ms/step | 20.0 samples/s +Step 49630/50462 | Loss: 2.5410 | LR: 3.45e-07 | 2998.5ms/step | 20.0 samples/s +Step 49640/50462 | Loss: 2.5428 | LR: 3.37e-07 | 3000.0ms/step | 20.0 samples/s +Step 49650/50462 | Loss: 2.5465 | LR: 3.29e-07 | 2999.9ms/step | 20.0 samples/s +Step 49660/50462 | Loss: 2.5283 | LR: 3.21e-07 | 3000.0ms/step | 20.0 samples/s +Step 49670/50462 | Loss: 2.5169 | LR: 3.13e-07 | 3000.2ms/step | 20.0 samples/s +Step 49680/50462 | Loss: 2.5601 | LR: 3.05e-07 | 2999.0ms/step | 20.0 samples/s +Step 49690/50462 | Loss: 2.5785 | LR: 2.97e-07 | 2998.7ms/step | 20.0 samples/s +Step 49700/50462 | Loss: 2.5494 | LR: 2.90e-07 | 2997.4ms/step | 20.0 samples/s +Step 49710/50462 | Loss: 2.5441 | LR: 2.82e-07 | 2997.0ms/step | 20.0 samples/s +Step 49720/50462 | Loss: 2.5345 | LR: 2.75e-07 | 2999.1ms/step | 20.0 samples/s +Step 49730/50462 | Loss: 2.5502 | LR: 2.67e-07 | 2998.8ms/step | 20.0 samples/s +Step 49740/50462 | Loss: 2.5710 | LR: 2.60e-07 | 3000.1ms/step | 20.0 samples/s +Step 49750/50462 | Loss: 2.5428 | LR: 2.53e-07 | 3001.0ms/step | 20.0 samples/s +Validation at step 49750: Loss = 2.5529, Perplexity = 12.84 +Step 49760/50462 | Loss: 2.5593 | LR: 2.46e-07 | 4598.8ms/step | 13.0 samples/s +Step 49770/50462 | Loss: 2.5487 | LR: 2.39e-07 | 3002.2ms/step | 20.0 samples/s +Step 49780/50462 | Loss: 2.5345 | LR: 2.32e-07 | 2999.9ms/step | 20.0 samples/s +Step 49790/50462 | Loss: 2.5260 | LR: 2.25e-07 | 2999.9ms/step | 20.0 samples/s +Step 49800/50462 | Loss: 2.5693 | LR: 2.19e-07 | 3000.1ms/step | 20.0 samples/s +Step 49810/50462 | Loss: 2.5576 | LR: 2.12e-07 | 3000.2ms/step | 20.0 samples/s +Step 49820/50462 | Loss: 2.5538 | LR: 2.06e-07 | 2999.7ms/step | 20.0 samples/s +Step 49830/50462 | Loss: 2.5443 | LR: 1.99e-07 | 3000.1ms/step | 20.0 samples/s +Step 49840/50462 | Loss: 2.5580 | LR: 1.93e-07 | 3001.8ms/step | 20.0 samples/s +Step 49850/50462 | Loss: 2.5344 | LR: 1.87e-07 | 2997.7ms/step | 20.0 samples/s +Step 49860/50462 | Loss: 2.5355 | LR: 1.81e-07 | 3000.1ms/step | 20.0 samples/s +Step 49870/50462 | Loss: 2.5512 | LR: 1.75e-07 | 3000.4ms/step | 20.0 samples/s +Step 49880/50462 | Loss: 2.5342 | LR: 1.69e-07 | 2998.7ms/step | 20.0 samples/s +Step 49890/50462 | Loss: 2.5424 | LR: 1.63e-07 | 2999.3ms/step | 20.0 samples/s +Step 49900/50462 | Loss: 2.5570 | LR: 1.58e-07 | 2996.9ms/step | 20.0 samples/s +Step 49910/50462 | Loss: 2.5419 | LR: 1.52e-07 | 2998.5ms/step | 20.0 samples/s +Step 49920/50462 | Loss: 2.5570 | LR: 1.47e-07 | 2997.5ms/step | 20.0 samples/s +Step 49930/50462 | Loss: 2.5267 | LR: 1.41e-07 | 2995.1ms/step | 20.0 samples/s +Step 49940/50462 | Loss: 2.5395 | LR: 1.36e-07 | 2998.4ms/step | 20.0 samples/s +Step 49950/50462 | Loss: 2.4992 | LR: 1.31e-07 | 2999.0ms/step | 20.0 samples/s +Step 49960/50462 | Loss: 2.5296 | LR: 1.26e-07 | 2997.0ms/step | 20.0 samples/s +Step 49970/50462 | Loss: 2.5220 | LR: 1.21e-07 | 3000.2ms/step | 20.0 samples/s +Step 49980/50462 | Loss: 2.5370 | LR: 1.16e-07 | 2999.8ms/step | 20.0 samples/s +Step 49990/50462 | Loss: 2.5137 | LR: 1.11e-07 | 3001.3ms/step | 20.0 samples/s +Step 50000/50462 | Loss: 2.5245 | LR: 1.07e-07 | 2998.3ms/step | 20.0 samples/s +Validation at step 50000: Loss = 2.5530, Perplexity = 12.85 +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-50000 +Step 50010/50462 | Loss: 2.5196 | LR: 1.02e-07 | 5661.7ms/step | 10.6 samples/s +Step 50020/50462 | Loss: 2.5292 | LR: 9.75e-08 | 2999.0ms/step | 20.0 samples/s +Step 50030/50462 | Loss: 2.5594 | LR: 9.32e-08 | 3000.5ms/step | 20.0 samples/s +Step 50040/50462 | Loss: 2.5876 | LR: 8.89e-08 | 2999.6ms/step | 20.0 samples/s +Step 50050/50462 | Loss: 2.5418 | LR: 8.47e-08 | 2999.6ms/step | 20.0 samples/s +Step 50060/50462 | Loss: 2.5354 | LR: 8.07e-08 | 3001.5ms/step | 20.0 samples/s +Step 50070/50462 | Loss: 2.5556 | LR: 7.67e-08 | 2999.1ms/step | 20.0 samples/s +Step 50080/50462 | Loss: 2.5278 | LR: 7.28e-08 | 2999.6ms/step | 20.0 samples/s +Step 50090/50462 | Loss: 2.5197 | LR: 6.91e-08 | 3006.8ms/step | 20.0 samples/s +Step 50100/50462 | Loss: 2.5263 | LR: 6.54e-08 | 2998.4ms/step | 20.0 samples/s +Step 50110/50462 | Loss: 2.5655 | LR: 6.19e-08 | 3003.1ms/step | 20.0 samples/s +Step 50120/50462 | Loss: 2.5473 | LR: 5.84e-08 | 3000.0ms/step | 20.0 samples/s +Step 50130/50462 | Loss: 2.5464 | LR: 5.50e-08 | 3000.1ms/step | 20.0 samples/s +Step 50140/50462 | Loss: 2.5645 | LR: 5.18e-08 | 3000.2ms/step | 20.0 samples/s +Step 50150/50462 | Loss: 2.5203 | LR: 4.86e-08 | 2998.8ms/step | 20.0 samples/s +Step 50160/50462 | Loss: 2.5365 | LR: 4.55e-08 | 2999.3ms/step | 20.0 samples/s +Step 50170/50462 | Loss: 2.5045 | LR: 4.26e-08 | 3002.1ms/step | 20.0 samples/s +Step 50180/50462 | Loss: 2.5092 | LR: 3.97e-08 | 3002.4ms/step | 20.0 samples/s +Step 50190/50462 | Loss: 2.5556 | LR: 3.69e-08 | 3001.2ms/step | 20.0 samples/s +Step 50200/50462 | Loss: 2.5263 | LR: 3.43e-08 | 3551.7ms/step | 16.9 samples/s +Step 50210/50462 | Loss: 2.5522 | LR: 3.17e-08 | 2993.8ms/step | 20.0 samples/s +Step 50220/50462 | Loss: 2.5455 | LR: 2.92e-08 | 2996.2ms/step | 20.0 samples/s +Step 50230/50462 | Loss: 2.5649 | LR: 2.69e-08 | 3001.2ms/step | 20.0 samples/s +Step 50240/50462 | Loss: 2.5284 | LR: 2.46e-08 | 4154.8ms/step | 14.4 samples/s +Step 50250/50462 | Loss: 2.5227 | LR: 2.24e-08 | 2994.0ms/step | 20.0 samples/s +Validation at step 50250: Loss = 2.5529, Perplexity = 12.84 +Step 50260/50462 | Loss: 2.5702 | LR: 2.04e-08 | 4596.9ms/step | 13.1 samples/s +Step 50270/50462 | Loss: 2.5814 | LR: 1.84e-08 | 2988.2ms/step | 20.1 samples/s +Step 50280/50462 | Loss: 2.5519 | LR: 1.65e-08 | 2987.1ms/step | 20.1 samples/s +Step 50290/50462 | Loss: 2.5665 | LR: 1.48e-08 | 2988.6ms/step | 20.1 samples/s +Step 50300/50462 | Loss: 2.5208 | LR: 1.31e-08 | 2988.3ms/step | 20.1 samples/s +Step 50310/50462 | Loss: 2.5170 | LR: 1.15e-08 | 2997.8ms/step | 20.0 samples/s +Step 50320/50462 | Loss: 2.5372 | LR: 1.01e-08 | 3000.0ms/step | 20.0 samples/s +Step 50330/50462 | Loss: 2.5651 | LR: 8.70e-09 | 2993.2ms/step | 20.0 samples/s +Step 50340/50462 | Loss: 2.5465 | LR: 7.43e-09 | 2989.4ms/step | 20.1 samples/s +Step 50350/50462 | Loss: 2.5261 | LR: 6.26e-09 | 2997.8ms/step | 20.0 samples/s +Step 50360/50462 | Loss: 2.5492 | LR: 5.19e-09 | 2989.9ms/step | 20.1 samples/s +Step 50370/50462 | Loss: 2.5247 | LR: 4.23e-09 | 2988.2ms/step | 20.1 samples/s +Step 50380/50462 | Loss: 2.5187 | LR: 3.36e-09 | 2989.6ms/step | 20.1 samples/s +Step 50390/50462 | Loss: 2.5567 | LR: 2.59e-09 | 2992.5ms/step | 20.0 samples/s +Step 50400/50462 | Loss: 2.5410 | LR: 1.92e-09 | 2990.7ms/step | 20.1 samples/s +Step 50410/50462 | Loss: 2.5311 | LR: 1.35e-09 | 2994.4ms/step | 20.0 samples/s +Step 50420/50462 | Loss: 2.5571 | LR: 8.81e-10 | 2992.9ms/step | 20.0 samples/s +Step 50430/50462 | Loss: 2.5479 | LR: 5.11e-10 | 2990.7ms/step | 20.1 samples/s +Step 50440/50462 | Loss: 2.5175 | LR: 2.42e-10 | 2992.0ms/step | 20.1 samples/s +Step 50450/50462 | Loss: 2.5583 | LR: 7.19e-11 | 2998.5ms/step | 20.0 samples/s +Step 50460/50462 | Loss: 2.5309 | LR: 2.00e-12 | 3000.6ms/step | 20.0 samples/s +Saved checkpoint to checkpoints/run_20260609_063834/checkpoint-50462 +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log; uploading config.yaml +wandb: uploading history steps 5045-5045, summary, console lines 1217-1217 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▆▆▆▆▅▅▅▃▃▃▃▃▃▃▃▃▂▂▂▁▂▂▁▂▂▁▁▁▂▂▂▂▂▂▂▁▁▂▂ +wandb: eval/perplexity █▆▆▆▆▅▅▅▃▃▃▃▃▃▃▃▃▂▂▂▁▂▂▁▂▂▁▁▁▂▂▂▂▂▂▂▁▁▂▂ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▁▁▁▁▁▁▅▁▁▃▁▁▁▁▁▁ +wandb: speed/samples_per_sec ██████████▁████████▁███████▁████████████ +wandb: speed/steps_per_sec █████████▁████████████▃█████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.55285 +wandb: eval/perplexity 12.84371 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 3000.55597 +wandb: speed/samples_per_sec 19.99629 +wandb: speed/steps_per_sec 0.33327 +wandb: +4 ... +wandb: +wandb: 🚀 View run sparse 64% at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/jh4dfz9a +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260612_172414-jh4dfz9a/logs +Training complete! diff --git a/epfl_final/log_files/sparse_pretrain_85%.log b/epfl_final/log_files/sparse_pretrain_85%.log new file mode 100644 index 0000000000000000000000000000000000000000..8e6fa0c5068225c3d6e6d3fd6b71ac2ef14f00a0 --- /dev/null +++ b/epfl_final/log_files/sparse_pretrain_85%.log @@ -0,0 +1,734 @@ +jnohup: ignoring input + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to ./checkpoints/run_20260613_222732 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "./checkpoints/run_20260613_222732", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "sparse 85%", + "wandb_tags": [ + "qwen3", + "1.2b", + "sparse" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 9, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "mask_cache_10", + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False +Checkpoint activations: False + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 9 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True +├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (G): 2.87 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 52437.19 +Params size (MB): 2871.45 +Estimated Total Size (MB): 55308.77 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 3 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 60 +Sequence length: 8192 +Tokens per optimizer step: 491,520 +Micro-batches per epoch (per rank): 504627 +Optimizer steps per epoch: 50462 +Total optimizer steps: 50462 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.27.2 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260613_222959-rp96qdel +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run sparse 85% +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rp96qdel +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/50462 | Loss: 12.2625 | LR: 6.67e-06 | 8182.9ms/step | 7.3 samples/s +Step 20/50462 | Loss: 12.0390 | LR: 1.33e-05 | 2751.4ms/step | 21.8 samples/s +Step 30/50462 | Loss: 11.2242 | LR: 2.00e-05 | 2756.9ms/step | 21.8 samples/s +Step 40/50462 | Loss: 10.2579 | LR: 2.67e-05 | 2751.4ms/step | 21.8 samples/s +Step 50/50462 | Loss: 9.8189 | LR: 3.33e-05 | 2750.1ms/step | 21.8 samples/s +Step 60/50462 | Loss: 9.3643 | LR: 4.00e-05 | 2749.4ms/step | 21.8 samples/s +Step 70/50462 | Loss: 8.8529 | LR: 4.67e-05 | 2749.5ms/step | 21.8 samples/s +Step 80/50462 | Loss: 8.3457 | LR: 5.33e-05 | 2750.8ms/step | 21.8 samples/s +Step 90/50462 | Loss: 7.9054 | LR: 6.00e-05 | 2753.4ms/step | 21.8 samples/s +Step 100/50462 | Loss: 7.5376 | LR: 6.67e-05 | 2751.6ms/step | 21.8 samples/s +Step 110/50462 | Loss: 7.2801 | LR: 7.33e-05 | 2754.5ms/step | 21.8 samples/s +Step 120/50462 | Loss: 7.1211 | LR: 8.00e-05 | 2758.6ms/step | 21.7 samples/s +Step 130/50462 | Loss: 7.0000 | LR: 8.67e-05 | 2761.6ms/step | 21.7 samples/s +Step 140/50462 | Loss: 6.8729 | LR: 9.33e-05 | 2761.3ms/step | 21.7 samples/s +Step 150/50462 | Loss: 6.7415 | LR: 1.00e-04 | 2760.7ms/step | 21.7 samples/s +Step 160/50462 | Loss: 6.6354 | LR: 1.07e-04 | 2761.9ms/step | 21.7 samples/s +Step 170/50462 | Loss: 6.5196 | LR: 1.13e-04 | 2759.5ms/step | 21.7 samples/s +Step 180/50462 | Loss: 6.4332 | LR: 1.20e-04 | 2759.1ms/step | 21.7 samples/s +Step 190/50462 | Loss: 6.3651 | LR: 1.27e-04 | 2762.1ms/step | 21.7 samples/s +Step 200/50462 | Loss: 6.2856 | LR: 1.33e-04 | 2765.8ms/step | 21.7 samples/s +Step 210/50462 | Loss: 6.2157 | LR: 1.40e-04 | 2763.2ms/step | 21.7 samples/s +Step 220/50462 | Loss: 6.1514 | LR: 1.47e-04 | 2764.4ms/step | 21.7 samples/s +Step 230/50462 | Loss: 6.1246 | LR: 1.53e-04 | 2764.8ms/step | 21.7 samples/s +Step 240/50462 | Loss: 6.0809 | LR: 1.60e-04 | 2765.7ms/step | 21.7 samples/s +Step 250/50462 | Loss: 6.0134 | LR: 1.67e-04 | 2766.9ms/step | 21.7 samples/s +Validation at step 250: Loss = 5.9597, Perplexity = 387.50 +Step 260/50462 | Loss: 5.9535 | LR: 1.73e-04 | 6912.4ms/step | 8.7 samples/s +Step 270/50462 | Loss: 5.9490 | LR: 1.80e-04 | 2761.4ms/step | 21.7 samples/s +Step 280/50462 | Loss: 5.8735 | LR: 1.87e-04 | 2759.1ms/step | 21.7 samples/s +Step 290/50462 | Loss: 5.8079 | LR: 1.93e-04 | 2761.0ms/step | 21.7 samples/s +Step 300/50462 | Loss: 5.7372 | LR: 2.00e-04 | 2762.7ms/step | 21.7 samples/s +Step 310/50462 | Loss: 5.6924 | LR: 2.07e-04 | 2761.0ms/step | 21.7 samples/s +Step 320/50462 | Loss: 5.6409 | LR: 2.13e-04 | 2760.7ms/step | 21.7 samples/s +Step 330/50462 | Loss: 5.6141 | LR: 2.20e-04 | 2760.6ms/step | 21.7 samples/s +Step 340/50462 | Loss: 5.5656 | LR: 2.27e-04 | 2760.9ms/step | 21.7 samples/s +Step 350/50462 | Loss: 5.5115 | LR: 2.33e-04 | 2759.4ms/step | 21.7 samples/s +Step 360/50462 | Loss: 5.4540 | LR: 2.40e-04 | 2760.6ms/step | 21.7 samples/s +Step 370/50462 | Loss: 5.3852 | LR: 2.47e-04 | 2760.4ms/step | 21.7 samples/s +Step 380/50462 | Loss: 5.3676 | LR: 2.53e-04 | 2761.4ms/step | 21.7 samples/s +Step 390/50462 | Loss: 5.3260 | LR: 2.60e-04 | 2759.9ms/step | 21.7 samples/s +Step 400/50462 | Loss: 5.2715 | LR: 2.67e-04 | 2760.3ms/step | 21.7 samples/s +Step 410/50462 | Loss: 5.2183 | LR: 2.73e-04 | 2759.8ms/step | 21.7 samples/s +Step 420/50462 | Loss: 5.1942 | LR: 2.80e-04 | 2761.0ms/step | 21.7 samples/s +Step 430/50462 | Loss: 5.1278 | LR: 2.87e-04 | 2761.1ms/step | 21.7 samples/s +Step 440/50462 | Loss: 5.1252 | LR: 2.93e-04 | 2759.0ms/step | 21.7 samples/s +Step 450/50462 | Loss: 5.0873 | LR: 3.00e-04 | 2758.1ms/step | 21.8 samples/s +Step 460/50462 | Loss: 5.0026 | LR: 3.07e-04 | 2756.5ms/step | 21.8 samples/s +Step 470/50462 | Loss: 5.0290 | LR: 3.13e-04 | 2759.6ms/step | 21.7 samples/s +Step 480/50462 | Loss: 5.0300 | LR: 3.20e-04 | 2760.6ms/step | 21.7 samples/s +Step 490/50462 | Loss: 4.9552 | LR: 3.27e-04 | 2760.2ms/step | 21.7 samples/s +Step 500/50462 | Loss: 4.9114 | LR: 3.33e-04 | 2759.2ms/step | 21.7 samples/s +Validation at step 500: Loss = 4.8836, Perplexity = 132.10 +Step 510/50462 | Loss: 4.9205 | LR: 3.40e-04 | 4253.1ms/step | 14.1 samples/s +Step 520/50462 | Loss: 4.8721 | LR: 3.47e-04 | 2759.4ms/step | 21.7 samples/s +Step 530/50462 | Loss: 4.8278 | LR: 3.53e-04 | 2758.7ms/step | 21.7 samples/s +Step 540/50462 | Loss: 4.7683 | LR: 3.60e-04 | 2756.9ms/step | 21.8 samples/s +Step 550/50462 | Loss: 4.8274 | LR: 3.67e-04 | 2758.3ms/step | 21.8 samples/s +Step 560/50462 | Loss: 4.7601 | LR: 3.73e-04 | 2758.6ms/step | 21.7 samples/s +Step 570/50462 | Loss: 4.6603 | LR: 3.80e-04 | 2757.7ms/step | 21.8 samples/s +Step 580/50462 | Loss: 4.6139 | LR: 3.87e-04 | 2758.0ms/step | 21.8 samples/s +Step 590/50462 | Loss: 4.7223 | LR: 3.93e-04 | 2757.1ms/step | 21.8 samples/s +Step 600/50462 | Loss: 4.7198 | LR: 4.00e-04 | 2757.4ms/step | 21.8 samples/s +Step 610/50462 | Loss: 4.5945 | LR: 4.07e-04 | 3350.6ms/step | 17.9 samples/s +Step 620/50462 | Loss: 4.5770 | LR: 4.13e-04 | 2751.4ms/step | 21.8 samples/s +Step 630/50462 | Loss: 4.5489 | LR: 4.20e-04 | 2755.7ms/step | 21.8 samples/s +Step 640/50462 | Loss: 4.5989 | LR: 4.27e-04 | 3351.4ms/step | 17.9 samples/s +Step 650/50462 | Loss: 4.5378 | LR: 4.33e-04 | 2757.0ms/step | 21.8 samples/s +Step 660/50462 | Loss: 4.4513 | LR: 4.40e-04 | 2759.7ms/step | 21.7 samples/s +Step 670/50462 | Loss: 4.4261 | LR: 4.47e-04 | 2759.6ms/step | 21.7 samples/s +Step 680/50462 | Loss: 4.4397 | LR: 4.53e-04 | 2759.4ms/step | 21.7 samples/s +Step 690/50462 | Loss: 4.4345 | LR: 4.60e-04 | 2758.6ms/step | 21.8 samples/s +Step 700/50462 | Loss: 4.4282 | LR: 4.67e-04 | 2756.7ms/step | 21.8 samples/s +Step 710/50462 | Loss: 4.3494 | LR: 4.73e-04 | 2759.1ms/step | 21.7 samples/s +Step 720/50462 | Loss: 4.3021 | LR: 4.80e-04 | 2758.6ms/step | 21.7 samples/s +Step 730/50462 | Loss: 4.3007 | LR: 4.87e-04 | 2761.2ms/step | 21.7 samples/s +Step 740/50462 | Loss: 4.2722 | LR: 4.93e-04 | 2761.0ms/step | 21.7 samples/s +Step 750/50462 | Loss: 4.2957 | LR: 5.00e-04 | 2760.2ms/step | 21.7 samples/s +Validation at step 750: Loss = 4.2539, Perplexity = 70.38 +Step 760/50462 | Loss: 4.2268 | LR: 5.00e-04 | 4247.4ms/step | 14.1 samples/s +Step 770/50462 | Loss: 4.2373 | LR: 5.00e-04 | 2755.4ms/step | 21.8 samples/s +Step 780/50462 | Loss: 4.1921 | LR: 5.00e-04 | 2756.7ms/step | 21.8 samples/s +Step 790/50462 | Loss: 4.1957 | LR: 5.00e-04 | 2755.7ms/step | 21.8 samples/s +Step 800/50462 | Loss: 4.1411 | LR: 5.00e-04 | 2759.5ms/step | 21.7 samples/s +Step 810/50462 | Loss: 4.1465 | LR: 5.00e-04 | 2758.6ms/step | 21.7 samples/s +Step 820/50462 | Loss: 4.1310 | LR: 5.00e-04 | 2757.7ms/step | 21.8 samples/s +Step 830/50462 | Loss: 4.1071 | LR: 5.00e-04 | 2753.5ms/step | 21.8 samples/s +Step 840/50462 | Loss: 4.0836 | LR: 5.00e-04 | 2758.5ms/step | 21.8 samples/s +Step 850/50462 | Loss: 4.0653 | LR: 5.00e-04 | 2759.2ms/step | 21.7 samples/s +Step 860/50462 | Loss: 4.0364 | LR: 5.00e-04 | 2757.6ms/step | 21.8 samples/s +Step 870/50462 | Loss: 4.0130 | LR: 5.00e-04 | 2759.9ms/step | 21.7 samples/s +Step 880/50462 | Loss: 3.9941 | LR: 5.00e-04 | 2756.1ms/step | 21.8 samples/s +Step 890/50462 | Loss: 3.9949 | LR: 5.00e-04 | 2758.7ms/step | 21.7 samples/s +Step 900/50462 | Loss: 3.9825 | LR: 5.00e-04 | 2759.2ms/step | 21.7 samples/s +Step 910/50462 | Loss: 3.9619 | LR: 5.00e-04 | 2759.3ms/step | 21.7 samples/s +Step 920/50462 | Loss: 3.9121 | LR: 5.00e-04 | 2758.2ms/step | 21.8 samples/s +Step 930/50462 | Loss: 3.9619 | LR: 5.00e-04 | 2760.1ms/step | 21.7 samples/s +Step 940/50462 | Loss: 3.9100 | LR: 5.00e-04 | 2756.4ms/step | 21.8 samples/s +Step 950/50462 | Loss: 3.8990 | LR: 5.00e-04 | 2757.9ms/step | 21.8 samples/s +Step 960/50462 | Loss: 3.8983 | LR: 5.00e-04 | 2756.9ms/step | 21.8 samples/s +Step 970/50462 | Loss: 3.8782 | LR: 5.00e-04 | 2759.8ms/step | 21.7 samples/s +Step 980/50462 | Loss: 3.8480 | LR: 5.00e-04 | 2759.5ms/step | 21.7 samples/s +Step 990/50462 | Loss: 3.8627 | LR: 5.00e-04 | 2759.6ms/step | 21.7 samples/s +Step 1000/50462 | Loss: 3.8628 | LR: 5.00e-04 | 2758.8ms/step | 21.7 samples/s +Validation at step 1000: Loss = 3.8567, Perplexity = 47.31 +Saved checkpoint to checkpoints/run_20260613_222732/checkpoint-1000 +Step 1010/50462 | Loss: 3.8364 | LR: 5.00e-04 | 5335.5ms/step | 11.2 samples/s +Step 1020/50462 | Loss: 3.8440 | LR: 5.00e-04 | 2758.6ms/step | 21.8 samples/s +Step 1030/50462 | Loss: 3.8335 | LR: 5.00e-04 | 2758.5ms/step | 21.8 samples/s +Step 1040/50462 | Loss: 3.8327 | LR: 5.00e-04 | 2757.2ms/step | 21.8 samples/s +Step 1050/50462 | Loss: 3.8008 | LR: 5.00e-04 | 2755.6ms/step | 21.8 samples/s +Step 1060/50462 | Loss: 3.7957 | LR: 5.00e-04 | 2757.5ms/step | 21.8 samples/s +Step 1070/50462 | Loss: 3.7822 | LR: 5.00e-04 | 2755.4ms/step | 21.8 samples/s +Step 1080/50462 | Loss: 3.7829 | LR: 5.00e-04 | 2755.4ms/step | 21.8 samples/s +Step 1090/50462 | Loss: 3.7583 | LR: 5.00e-04 | 2757.7ms/step | 21.8 samples/s +Step 1100/50462 | Loss: 3.7709 | LR: 5.00e-04 | 2754.8ms/step | 21.8 samples/s +Step 1110/50462 | Loss: 3.7790 | LR: 5.00e-04 | 2753.9ms/step | 21.8 samples/s +Step 1120/50462 | Loss: 3.7341 | LR: 5.00e-04 | 2755.7ms/step | 21.8 samples/s +Step 1130/50462 | Loss: 3.7454 | LR: 5.00e-04 | 2754.9ms/step | 21.8 samples/s +Step 1140/50462 | Loss: 3.7167 | LR: 5.00e-04 | 2754.6ms/step | 21.8 samples/s +Step 1150/50462 | Loss: 3.6859 | LR: 5.00e-04 | 2754.8ms/step | 21.8 samples/s +Step 1160/50462 | Loss: 3.7077 | LR: 5.00e-04 | 2755.9ms/step | 21.8 samples/s +Step 1170/50462 | Loss: 3.6990 | LR: 5.00e-04 | 2757.6ms/step | 21.8 samples/s +Step 1180/50462 | Loss: 3.6600 | LR: 5.00e-04 | 2754.0ms/step | 21.8 samples/s +Step 1190/50462 | Loss: 3.6831 | LR: 5.00e-04 | 2752.8ms/step | 21.8 samples/s +Step 1200/50462 | Loss: 3.6980 | LR: 5.00e-04 | 2756.9ms/step | 21.8 samples/s +Step 1210/50462 | Loss: 3.6939 | LR: 5.00e-04 | 2754.9ms/step | 21.8 samples/s +Step 1220/50462 | Loss: 3.6551 | LR: 5.00e-04 | 2753.8ms/step | 21.8 samples/s +Step 1230/50462 | Loss: 3.6261 | LR: 5.00e-04 | 2754.7ms/step | 21.8 samples/s +Step 1240/50462 | Loss: 3.6812 | LR: 5.00e-04 | 2754.7ms/step | 21.8 samples/s +Step 1250/50462 | Loss: 3.6441 | LR: 5.00e-04 | 2755.0ms/step | 21.8 samples/s +Validation at step 1250: Loss = 3.6542, Perplexity = 38.64 +Step 1260/50462 | Loss: 3.6657 | LR: 5.00e-04 | 4239.3ms/step | 14.2 samples/s +Step 1270/50462 | Loss: 3.6287 | LR: 5.00e-04 | 2756.0ms/step | 21.8 samples/s +Step 1280/50462 | Loss: 3.6062 | LR: 5.00e-04 | 2758.2ms/step | 21.8 samples/s +Step 1290/50462 | Loss: 3.6149 | LR: 5.00e-04 | 2756.7ms/step | 21.8 samples/s +Step 1300/50462 | Loss: 3.6040 | LR: 5.00e-04 | 2757.3ms/step | 21.8 samples/s +Step 1310/50462 | Loss: 3.6104 | LR: 5.00e-04 | 2758.2ms/step | 21.8 samples/s +Step 1320/50462 | Loss: 3.6035 | LR: 5.00e-04 | 2755.7ms/step | 21.8 samples/s +Step 1330/50462 | Loss: 3.5944 | LR: 5.00e-04 | 2757.9ms/step | 21.8 samples/s +Step 1340/50462 | Loss: 3.5838 | LR: 5.00e-04 | 2753.3ms/step | 21.8 samples/s +Step 1350/50462 | Loss: 3.5655 | LR: 5.00e-04 | 2752.6ms/step | 21.8 samples/s +Step 1360/50462 | Loss: 3.5793 | LR: 5.00e-04 | 2759.1ms/step | 21.7 samples/s +Step 1370/50462 | Loss: 3.5590 | LR: 5.00e-04 | 2760.2ms/step | 21.7 samples/s +Step 1380/50462 | Loss: 3.5983 | LR: 5.00e-04 | 2757.8ms/step | 21.8 samples/s +Step 1390/50462 | Loss: 3.5783 | LR: 5.00e-04 | 2750.2ms/step | 21.8 samples/s +Step 1400/50462 | Loss: 3.5413 | LR: 5.00e-04 | 2751.5ms/step | 21.8 samples/s +Step 1410/50462 | Loss: 3.5678 | LR: 5.00e-04 | 2750.0ms/step | 21.8 samples/s +Step 1420/50462 | Loss: 3.5473 | LR: 5.00e-04 | 2751.2ms/step | 21.8 samples/s +Step 1430/50462 | Loss: 3.5228 | LR: 5.00e-04 | 2751.3ms/step | 21.8 samples/s +Step 1440/50462 | Loss: 3.5522 | LR: 5.00e-04 | 2750.7ms/step | 21.8 samples/s +Step 1450/50462 | Loss: 3.5181 | LR: 5.00e-04 | 2750.4ms/step | 21.8 samples/s +Step 1460/50462 | Loss: 3.5244 | LR: 5.00e-04 | 2752.1ms/step | 21.8 samples/s +Step 1470/50462 | Loss: 3.4998 | LR: 5.00e-04 | 2749.7ms/step | 21.8 samples/s +Step 1480/50462 | Loss: 3.5392 | LR: 5.00e-04 | 2749.8ms/step | 21.8 samples/s +Step 1490/50462 | Loss: 3.5190 | LR: 5.00e-04 | 2754.1ms/step | 21.8 samples/s +Step 1500/50462 | Loss: 3.4885 | LR: 5.00e-04 | 2752.7ms/step | 21.8 samples/s +Validation at step 1500: Loss = 3.5274, Perplexity = 34.03 +Step 1510/50462 | Loss: 3.5219 | LR: 5.00e-04 | 4239.9ms/step | 14.2 samples/s +Step 1520/50462 | Loss: 3.4904 | LR: 5.00e-04 | 2756.5ms/step | 21.8 samples/s +Step 1530/50462 | Loss: 3.5169 | LR: 5.00e-04 | 2756.5ms/step | 21.8 samples/s +Step 1540/50462 | Loss: 3.4818 | LR: 5.00e-04 | 2756.4ms/step | 21.8 samples/s +Step 1550/50462 | Loss: 3.4889 | LR: 5.00e-04 | 2755.1ms/step | 21.8 samples/s +Step 1560/50462 | Loss: 3.4817 | LR: 5.00e-04 | 2754.7ms/step | 21.8 samples/s +Step 1570/50462 | Loss: 3.5134 | LR: 5.00e-04 | 2756.2ms/step | 21.8 samples/s +Step 1580/50462 | Loss: 3.4805 | LR: 5.00e-04 | 2757.0ms/step | 21.8 samples/s +Step 1590/50462 | Loss: 3.4989 | LR: 5.00e-04 | 2755.5ms/step | 21.8 samples/s +Step 1600/50462 | Loss: 3.4897 | LR: 5.00e-04 | 2756.5ms/step | 21.8 samples/s +Step 1610/50462 | Loss: 3.4518 | LR: 5.00e-04 | 2756.2ms/step | 21.8 samples/s +Step 1620/50462 | Loss: 3.4727 | LR: 5.00e-04 | 2756.0ms/step | 21.8 samples/s +Step 1630/50462 | Loss: 3.4882 | LR: 5.00e-04 | 2755.3ms/step | 21.8 samples/s +Step 1640/50462 | Loss: 3.4619 | LR: 5.00e-04 | 2755.1ms/step | 21.8 samples/s +Step 1650/50462 | Loss: 3.4196 | LR: 5.00e-04 | 2754.2ms/step | 21.8 samples/s +Step 1660/50462 | Loss: 3.4413 | LR: 5.00e-04 | 2757.2ms/step | 21.8 samples/s +Step 1670/50462 | Loss: 3.4653 | LR: 5.00e-04 | 2755.3ms/step | 21.8 samples/s +Step 1680/50462 | Loss: 3.4382 | LR: 5.00e-04 | 2757.2ms/step | 21.8 samples/s +Step 1690/50462 | Loss: 3.4365 | LR: 5.00e-04 | 2757.5ms/step | 21.8 samples/s +Step 1700/50462 | Loss: 3.4450 | LR: 5.00e-04 | 2754.2ms/step | 21.8 samples/s +Step 1710/50462 | Loss: 3.4607 | LR: 5.00e-04 | 2753.4ms/step | 21.8 samples/s +Step 1720/50462 | Loss: 3.4240 | LR: 5.00e-04 | 2753.0ms/step | 21.8 samples/s +Step 1730/50462 | Loss: 3.4246 | LR: 5.00e-04 | 2758.0ms/step | 21.8 samples/s +Step 1740/50462 | Loss: 3.4535 | LR: 5.00e-04 | 2757.1ms/step | 21.8 samples/s +Step 1750/50462 | Loss: 3.4654 | LR: 5.00e-04 | 2755.9ms/step | 21.8 samples/s +Validation at step 1750: Loss = 3.4314, Perplexity = 30.92 +Step 1760/50462 | Loss: 3.4416 | LR: 4.99e-04 | 4244.7ms/step | 14.1 samples/s +Step 1770/50462 | Loss: 3.4231 | LR: 4.99e-04 | 2757.2ms/step | 21.8 samples/s +Step 1780/50462 | Loss: 3.3943 | LR: 4.99e-04 | 2756.0ms/step | 21.8 samples/s +Step 1790/50462 | Loss: 3.3960 | LR: 4.99e-04 | 2753.5ms/step | 21.8 samples/s +Step 1800/50462 | Loss: 3.3913 | LR: 4.99e-04 | 2755.6ms/step | 21.8 samples/s +Step 1810/50462 | Loss: 3.4023 | LR: 4.99e-04 | 2754.3ms/step | 21.8 samples/s +Step 1820/50462 | Loss: 3.3828 | LR: 4.99e-04 | 2755.6ms/step | 21.8 samples/s +Step 1830/50462 | Loss: 3.3855 | LR: 4.99e-04 | 2755.8ms/step | 21.8 samples/s +Step 1840/50462 | Loss: 3.3854 | LR: 4.99e-04 | 2757.8ms/step | 21.8 samples/s +Step 1850/50462 | Loss: 3.3936 | LR: 4.99e-04 | 2758.2ms/step | 21.8 samples/s +Step 1860/50462 | Loss: 3.3723 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s +Step 1870/50462 | Loss: 3.4110 | LR: 4.99e-04 | 2752.6ms/step | 21.8 samples/s +Step 1880/50462 | Loss: 3.3946 | LR: 4.99e-04 | 2755.1ms/step | 21.8 samples/s +Step 1890/50462 | Loss: 3.3591 | LR: 4.99e-04 | 2754.6ms/step | 21.8 samples/s +Step 1900/50462 | Loss: 3.3619 | LR: 4.99e-04 | 2752.5ms/step | 21.8 samples/s +Step 1910/50462 | Loss: 3.3959 | LR: 4.99e-04 | 2753.5ms/step | 21.8 samples/s +Step 1920/50462 | Loss: 3.3929 | LR: 4.99e-04 | 2755.1ms/step | 21.8 samples/s +Step 1930/50462 | Loss: 3.3972 | LR: 4.99e-04 | 2748.4ms/step | 21.8 samples/s +Step 1940/50462 | Loss: 3.3899 | LR: 4.99e-04 | 2755.8ms/step | 21.8 samples/s +Step 1950/50462 | Loss: 3.3772 | LR: 4.99e-04 | 2753.1ms/step | 21.8 samples/s +Step 1960/50462 | Loss: 3.3742 | LR: 4.99e-04 | 2752.0ms/step | 21.8 samples/s +Step 1970/50462 | Loss: 3.3719 | LR: 4.99e-04 | 2753.3ms/step | 21.8 samples/s +Step 1980/50462 | Loss: 3.3642 | LR: 4.99e-04 | 2752.9ms/step | 21.8 samples/s +Step 1990/50462 | Loss: 3.3920 | LR: 4.99e-04 | 2751.7ms/step | 21.8 samples/s +Step 2000/50462 | Loss: 3.3910 | LR: 4.99e-04 | 2751.7ms/step | 21.8 samples/s +Validation at step 2000: Loss = 3.3635, Perplexity = 28.89 +Saved checkpoint to checkpoints/run_20260613_222732/checkpoint-2000 +Step 2010/50462 | Loss: 3.3737 | LR: 4.99e-04 | 5303.6ms/step | 11.3 samples/s +Step 2020/50462 | Loss: 3.3731 | LR: 4.99e-04 | 2749.0ms/step | 21.8 samples/s +Step 2030/50462 | Loss: 3.3423 | LR: 4.99e-04 | 2750.6ms/step | 21.8 samples/s +Step 2040/50462 | Loss: 3.3541 | LR: 4.99e-04 | 2754.8ms/step | 21.8 samples/s +Step 2050/50462 | Loss: 3.3236 | LR: 4.99e-04 | 2753.4ms/step | 21.8 samples/s +Step 2060/50462 | Loss: 3.3272 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s +Step 2070/50462 | Loss: 3.3468 | LR: 4.99e-04 | 2755.6ms/step | 21.8 samples/s +Step 2080/50462 | Loss: 3.3361 | LR: 4.99e-04 | 2752.7ms/step | 21.8 samples/s +Step 2090/50462 | Loss: 3.3368 | LR: 4.99e-04 | 2752.5ms/step | 21.8 samples/s +Step 2100/50462 | Loss: 3.3238 | LR: 4.99e-04 | 2754.0ms/step | 21.8 samples/s +Step 2110/50462 | Loss: 3.3676 | LR: 4.99e-04 | 2754.5ms/step | 21.8 samples/s +Step 2120/50462 | Loss: 3.3574 | LR: 4.99e-04 | 2751.1ms/step | 21.8 samples/s +Step 2130/50462 | Loss: 3.3197 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s +Step 2140/50462 | Loss: 3.3035 | LR: 4.99e-04 | 2754.4ms/step | 21.8 samples/s +Step 2150/50462 | Loss: 3.3093 | LR: 4.99e-04 | 2751.5ms/step | 21.8 samples/s +Step 2160/50462 | Loss: 3.3205 | LR: 4.99e-04 | 2752.5ms/step | 21.8 samples/s +Step 2170/50462 | Loss: 3.3091 | LR: 4.99e-04 | 2753.6ms/step | 21.8 samples/s +Step 2180/50462 | Loss: 3.3326 | LR: 4.99e-04 | 2754.3ms/step | 21.8 samples/s +Step 2190/50462 | Loss: 3.3087 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s +Step 2200/50462 | Loss: 3.3012 | LR: 4.99e-04 | 2752.7ms/step | 21.8 samples/s +Step 2210/50462 | Loss: 3.3139 | LR: 4.99e-04 | 2755.2ms/step | 21.8 samples/s +Step 2220/50462 | Loss: 3.2971 | LR: 4.99e-04 | 2753.7ms/step | 21.8 samples/s +Step 2230/50462 | Loss: 3.3110 | LR: 4.99e-04 | 2752.6ms/step | 21.8 samples/s +Step 2240/50462 | Loss: 3.3018 | LR: 4.99e-04 | 2756.0ms/step | 21.8 samples/s +Step 2250/50462 | Loss: 3.3460 | LR: 4.99e-04 | 2755.3ms/step | 21.8 samples/s +Validation at step 2250: Loss = 3.3093, Perplexity = 27.37 +Step 2260/50462 | Loss: 3.3055 | LR: 4.99e-04 | 4240.8ms/step | 14.1 samples/s +Step 2270/50462 | Loss: 3.3194 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s +Step 2280/50462 | Loss: 3.2712 | LR: 4.99e-04 | 2754.0ms/step | 21.8 samples/s +Step 2290/50462 | Loss: 3.2688 | LR: 4.99e-04 | 2753.0ms/step | 21.8 samples/s +Step 2300/50462 | Loss: 3.2727 | LR: 4.99e-04 | 2758.6ms/step | 21.8 samples/s +Step 2310/50462 | Loss: 3.2953 | LR: 4.99e-04 | 2757.3ms/step | 21.8 samples/s +Step 2320/50462 | Loss: 3.2905 | LR: 4.99e-04 | 2754.0ms/step | 21.8 samples/s +Step 2330/50462 | Loss: 3.2547 | LR: 4.99e-04 | 2754.7ms/step | 21.8 samples/s +Step 2340/50462 | Loss: 3.2857 | LR: 4.99e-04 | 2753.4ms/step | 21.8 samples/s +Step 2350/50462 | Loss: 3.2591 | LR: 4.99e-04 | 2753.4ms/step | 21.8 samples/s +Step 2360/50462 | Loss: 3.2575 | LR: 4.99e-04 | 2755.5ms/step | 21.8 samples/s +Step 2370/50462 | Loss: 3.2791 | LR: 4.99e-04 | 2752.6ms/step | 21.8 samples/s +Step 2380/50462 | Loss: 3.2312 | LR: 4.99e-04 | 2757.2ms/step | 21.8 samples/s +Step 2390/50462 | Loss: 3.2538 | LR: 4.99e-04 | 2752.6ms/step | 21.8 samples/s +Step 2400/50462 | Loss: 3.2761 | LR: 4.99e-04 | 2754.2ms/step | 21.8 samples/s +Step 2410/50462 | Loss: 3.2397 | LR: 4.99e-04 | 2753.6ms/step | 21.8 samples/s +Step 2420/50462 | Loss: 3.2826 | LR: 4.99e-04 | 2753.9ms/step | 21.8 samples/s +Step 2430/50462 | Loss: 3.2701 | LR: 4.99e-04 | 2753.9ms/step | 21.8 samples/s +Step 2440/50462 | Loss: 3.2825 | LR: 4.99e-04 | 2754.0ms/step | 21.8 samples/s +Step 2450/50462 | Loss: 3.3077 | LR: 4.99e-04 | 2754.3ms/step | 21.8 samples/s +Step 2460/50462 | Loss: 3.2552 | LR: 4.99e-04 | 2754.5ms/step | 21.8 samples/s +Step 2470/50462 | Loss: 3.2625 | LR: 4.99e-04 | 2750.3ms/step | 21.8 samples/s +Step 2480/50462 | Loss: 3.2772 | LR: 4.99e-04 | 2749.9ms/step | 21.8 samples/s +Step 2490/50462 | Loss: 3.2593 | LR: 4.98e-04 | 2750.2ms/step | 21.8 samples/s +Step 2500/50462 | Loss: 3.2519 | LR: 4.98e-04 | 2752.0ms/step | 21.8 samples/s +Validation at step 2500: Loss = 3.2614, Perplexity = 26.09 +Step 2510/50462 | Loss: 3.2464 | LR: 4.98e-04 | 4242.9ms/step | 14.1 samples/s +Step 2520/50462 | Loss: 3.2366 | LR: 4.98e-04 | 2755.4ms/step | 21.8 samples/s +Step 2530/50462 | Loss: 3.2379 | LR: 4.98e-04 | 2752.0ms/step | 21.8 samples/s +Step 2540/50462 | Loss: 3.2605 | LR: 4.98e-04 | 2752.4ms/step | 21.8 samples/s +Step 2550/50462 | Loss: 3.2680 | LR: 4.98e-04 | 2750.5ms/step | 21.8 samples/s +Step 2560/50462 | Loss: 3.2758 | LR: 4.98e-04 | 2752.0ms/step | 21.8 samples/s +Step 2570/50462 | Loss: 3.2169 | LR: 4.98e-04 | 2752.2ms/step | 21.8 samples/s +Step 2580/50462 | Loss: 3.2281 | LR: 4.98e-04 | 2751.8ms/step | 21.8 samples/s +Step 2590/50462 | Loss: 3.2566 | LR: 4.98e-04 | 2750.7ms/step | 21.8 samples/s +Step 2600/50462 | Loss: 3.2153 | LR: 4.98e-04 | 2752.6ms/step | 21.8 samples/s +Step 2610/50462 | Loss: 3.2294 | LR: 4.98e-04 | 2752.3ms/step | 21.8 samples/s +Step 2620/50462 | Loss: 3.2509 | LR: 4.98e-04 | 2754.3ms/step | 21.8 samples/s +Step 2630/50462 | Loss: 3.2172 | LR: 4.98e-04 | 2751.2ms/step | 21.8 samples/s +Step 2640/50462 | Loss: 3.2446 | LR: 4.98e-04 | 2750.6ms/step | 21.8 samples/s +Step 2650/50462 | Loss: 3.2413 | LR: 4.98e-04 | 2753.6ms/step | 21.8 samples/s +Step 2660/50462 | Loss: 3.2172 | LR: 4.98e-04 | 2749.9ms/step | 21.8 samples/s +Step 2670/50462 | Loss: 3.2744 | LR: 4.98e-04 | 2748.6ms/step | 21.8 samples/s +Step 2680/50462 | Loss: 3.2197 | LR: 4.98e-04 | 2751.7ms/step | 21.8 samples/s +Step 2690/50462 | Loss: 3.2382 | LR: 4.98e-04 | 2752.3ms/step | 21.8 samples/s +Step 2700/50462 | Loss: 3.2237 | LR: 4.98e-04 | 2751.2ms/step | 21.8 samples/s +Step 2710/50462 | Loss: 3.2030 | LR: 4.98e-04 | 2753.0ms/step | 21.8 samples/s +Step 2720/50462 | Loss: 3.2118 | LR: 4.98e-04 | 2754.1ms/step | 21.8 samples/s +Step 2730/50462 | Loss: 3.2279 | LR: 4.98e-04 | 3265.0ms/step | 18.4 samples/s +Step 2740/50462 | Loss: 3.1963 | LR: 4.98e-04 | 2748.0ms/step | 21.8 samples/s +Step 2750/50462 | Loss: 3.2225 | LR: 4.98e-04 | 2750.9ms/step | 21.8 samples/s +Validation at step 2750: Loss = 3.2229, Perplexity = 25.10 +Step 2760/50462 | Loss: 3.2236 | LR: 4.98e-04 | 4236.9ms/step | 14.2 samples/s +Step 2770/50462 | Loss: 3.2081 | LR: 4.98e-04 | 2752.8ms/step | 21.8 samples/s +Step 2780/50462 | Loss: 3.2147 | LR: 4.98e-04 | 3289.5ms/step | 18.2 samples/s +Step 2790/50462 | Loss: 3.1886 | LR: 4.98e-04 | 2750.6ms/step | 21.8 samples/s +Step 2800/50462 | Loss: 3.2108 | LR: 4.98e-04 | 2750.3ms/step | 21.8 samples/s +Step 2810/50462 | Loss: 3.2195 | LR: 4.98e-04 | 2752.1ms/step | 21.8 samples/s +Step 2820/50462 | Loss: 3.2034 | LR: 4.98e-04 | 2753.2ms/step | 21.8 samples/s +Step 2830/50462 | Loss: 3.1613 | LR: 4.98e-04 | 2752.7ms/step | 21.8 samples/s +Step 2840/50462 | Loss: 3.2031 | LR: 4.98e-04 | 2753.9ms/step | 21.8 samples/s +Step 2850/50462 | Loss: 3.2032 | LR: 4.98e-04 | 2753.0ms/step | 21.8 samples/s +Step 2860/50462 | Loss: 3.2109 | LR: 4.98e-04 | 2750.1ms/step | 21.8 samples/s +Step 2870/50462 | Loss: 3.1961 | LR: 4.98e-04 | 2751.8ms/step | 21.8 samples/s +Step 2880/50462 | Loss: 3.1908 | LR: 4.98e-04 | 2752.1ms/step | 21.8 samples/s +Step 2890/50462 | Loss: 3.1746 | LR: 4.98e-04 | 2752.3ms/step | 21.8 samples/s +Step 2900/50462 | Loss: 3.1946 | LR: 4.98e-04 | 2751.9ms/step | 21.8 samples/s +Step 2910/50462 | Loss: 3.2074 | LR: 4.98e-04 | 2752.4ms/step | 21.8 samples/s +Step 2920/50462 | Loss: 3.1985 | LR: 4.98e-04 | 2750.9ms/step | 21.8 samples/s +Step 2930/50462 | Loss: 3.1761 | LR: 4.98e-04 | 2751.8ms/step | 21.8 samples/s +Step 2940/50462 | Loss: 3.1707 | LR: 4.98e-04 | 2752.2ms/step | 21.8 samples/s +Step 2950/50462 | Loss: 3.2039 | LR: 4.98e-04 | 2752.7ms/step | 21.8 samples/s +Step 2960/50462 | Loss: 3.1663 | LR: 4.98e-04 | 2750.3ms/step | 21.8 samples/s +Step 2970/50462 | Loss: 3.1965 | LR: 4.98e-04 | 2750.3ms/step | 21.8 samples/s +Step 2980/50462 | Loss: 3.2059 | LR: 4.98e-04 | 2751.1ms/step | 21.8 samples/s +Step 2990/50462 | Loss: 3.1836 | LR: 4.97e-04 | 2752.6ms/step | 21.8 samples/s +Step 3000/50462 | Loss: 3.1867 | LR: 4.97e-04 | 2751.1ms/step | 21.8 samples/s +Validation at step 3000: Loss = 3.1912, Perplexity = 24.32 +Saved checkpoint to checkpoints/run_20260613_222732/checkpoint-3000 +Step 3010/50462 | Loss: 3.1856 | LR: 4.97e-04 | 5336.6ms/step | 11.2 samples/s +Step 3020/50462 | Loss: 3.1742 | LR: 4.97e-04 | 2747.0ms/step | 21.8 samples/s +Step 3030/50462 | Loss: 3.1650 | LR: 4.97e-04 | 2749.1ms/step | 21.8 samples/s +Traceback (most recent call last): + File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1440, in + main() + File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1437, in main + train(config) + File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1301, in train + outputs = model( + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl + return forward_call(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1832, in forward + else self._run_ddp_forward(*inputs, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1657, in _run_ddp_forward + return self.module(*inputs, **kwargs) # type: ignore[index] + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 473, in __call__ + return super().__call__(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl + return forward_call(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1047, in compile_wrapper + result = fn(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl + return forward_call(*args, **kwargs) + File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 285, in forward + hidden_states, next_cache, all_hidden_states, all_attentions = self.model( + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl + return forward_call(*args, **kwargs) + File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 139, in forward + if has_any_truly_sparse and (attention_mask == 0).any(): + File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 139, in torch_dynamo_resume_in_forward_at_139 + if has_any_truly_sparse and (attention_mask == 0).any(): + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 473, in __call__ + return super().__call__(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl + return forward_call(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1298, in _fn + return fn(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/fx/graph_module.py", line 949, in call_wrapped + return self._wrapped_call(self, *args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/fx/graph_module.py", line 447, in __call__ + return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl + return forward_call(*args, **kwargs) + File ".720", line 656, in forward + submod_16 = self.compiled_submod_16(submod_15, l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_, getitem_15, l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_q_norm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_k_norm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_rotary_emb_buffers_inv_freq_, l_position_ids_, l_sparse_indices_8_kv_num_blocks, l_sparse_indices_8_kv_indices, l_sparse_indices_8_full_kv_num_blocks, l_sparse_indices_8_full_kv_indices, l_sparse_indices_8_q_num_blocks, l_sparse_indices_8_q_indices, l_sparse_indices_8_full_q_num_blocks, l_sparse_indices_8_full_q_indices, l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_); submod_15 = l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = getitem_15 = l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_rotary_emb_buffers_inv_freq_ = l_sparse_indices_8_kv_num_blocks = l_sparse_indices_8_kv_indices = l_sparse_indices_8_full_kv_num_blocks = l_sparse_indices_8_full_kv_indices = l_sparse_indices_8_q_num_blocks = l_sparse_indices_8_q_indices = l_sparse_indices_8_full_q_num_blocks = l_sparse_indices_8_full_q_indices = l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = None + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl + return forward_call(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 215, in forward + x = self.submod(*args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1298, in _fn + return fn(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1273, in forward + return compiled_fn(full_args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 777, in runtime_wrapper + all_outs = compiled_invoker.run(args, on_before_call=exit_prologue) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 506, in run + return call_func_at_runtime_with_args( + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args + out = normalize_as_list(f(args)) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 93, in g + return f(*args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/function.py", line 596, in apply + return super().apply(*args, **kwargs) # type: ignore[misc] + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2844, in forward + fw_outs = call_func_at_runtime_with_args( + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args + out = normalize_as_list(f(args)) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 961, in wrapper + return compiled_fn(runtime_args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 850, in wrapper + return compiled_fn(runtime_args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1055, in inner_fn + outs = compiled_fn(args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 725, in __call__ + return self.current_callable(inputs) + File "/home/navasard/.cache/torch_inductor/oq/coq6impgppmp6dnp26psegq3avd7bsgoow4agrrdejd2tgokhclc.py", line 1390, in call + buf19.copy_(primals_12, False) +KeyboardInterrupt +[rank0]: Traceback (most recent call last): +[rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1440, in +[rank0]: main() +[rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1437, in main +[rank0]: train(config) +[rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1301, in train +[rank0]: outputs = model( +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1832, in forward +[rank0]: else self._run_ddp_forward(*inputs, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1657, in _run_ddp_forward +[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index] +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 473, in __call__ +[rank0]: return super().__call__(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1047, in compile_wrapper +[rank0]: result = fn(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 285, in forward +[rank0]: hidden_states, next_cache, all_hidden_states, all_attentions = self.model( +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 139, in forward +[rank0]: if has_any_truly_sparse and (attention_mask == 0).any(): +[rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 139, in torch_dynamo_resume_in_forward_at_139 +[rank0]: if has_any_truly_sparse and (attention_mask == 0).any(): +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 473, in __call__ +[rank0]: return super().__call__(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1298, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/fx/graph_module.py", line 949, in call_wrapped +[rank0]: return self._wrapped_call(self, *args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/fx/graph_module.py", line 447, in __call__ +[rank0]: return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File ".720", line 656, in forward +[rank0]: submod_16 = self.compiled_submod_16(submod_15, l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_, getitem_15, l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_q_norm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_k_norm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_rotary_emb_buffers_inv_freq_, l_position_ids_, l_sparse_indices_8_kv_num_blocks, l_sparse_indices_8_kv_indices, l_sparse_indices_8_full_kv_num_blocks, l_sparse_indices_8_full_kv_indices, l_sparse_indices_8_q_num_blocks, l_sparse_indices_8_q_indices, l_sparse_indices_8_full_q_num_blocks, l_sparse_indices_8_full_q_indices, l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_); submod_15 = l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = getitem_15 = l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_rotary_emb_buffers_inv_freq_ = l_sparse_indices_8_kv_num_blocks = l_sparse_indices_8_kv_indices = l_sparse_indices_8_full_kv_num_blocks = l_sparse_indices_8_full_kv_indices = l_sparse_indices_8_q_num_blocks = l_sparse_indices_8_q_indices = l_sparse_indices_8_full_q_num_blocks = l_sparse_indices_8_full_q_indices = l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = None +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 215, in forward +[rank0]: x = self.submod(*args) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1298, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1273, in forward +[rank0]: return compiled_fn(full_args) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 777, in runtime_wrapper +[rank0]: all_outs = compiled_invoker.run(args, on_before_call=exit_prologue) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 506, in run +[rank0]: return call_func_at_runtime_with_args( +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args +[rank0]: out = normalize_as_list(f(args)) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 93, in g +[rank0]: return f(*args) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/function.py", line 596, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2844, in forward +[rank0]: fw_outs = call_func_at_runtime_with_args( +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args +[rank0]: out = normalize_as_list(f(args)) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 961, in wrapper +[rank0]: return compiled_fn(runtime_args) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 850, in wrapper +[rank0]: return compiled_fn(runtime_args) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1055, in inner_fn +[rank0]: outs = compiled_fn(args) +[rank0]: File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 725, in __call__ +[rank0]: return self.current_callable(inputs) +[rank0]: File "/home/navasard/.cache/torch_inductor/oq/coq6impgppmp6dnp26psegq3avd7bsgoow4agrrdejd2tgokhclc.py", line 1390, in call +[rank0]: buf19.copy_(primals_12, False) +[rank0]: KeyboardInterrupt +wandb: +wandb: 🚀 View run sparse 85% at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rp96qdel +wandb: Find logs at: wandb/run-20260613_222959-rp96qdel/logs +W0614 00:56:27.057000 2912 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3073 closing signal SIGTERM +W0614 00:56:27.058000 2912 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3074 closing signal SIGTERM +E0614 00:56:27.070000 2912 torch/distributed/elastic/multiprocessing/api.py:988] failed (exitcode: -2) local_rank: 0 (pid: 3072) of binary: /usr/bin/python3 +Traceback (most recent call last): + File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/usr/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1020, in + main() + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 367, in wrapper + return f(*args, **kwargs) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1016, in main + run(args) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1007, in run + elastic_launch( + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 191, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 371, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +====================================================== +SparseAtt/train.py FAILED +------------------------------------------------------ +Failures: +[1]: + time : 2026-06-14_00:56:27 + host : h100-train-3-gpu-0-9 + rank : 1 (local_rank: 1) + exitcode : -15 (pid: 3073) (SIGTERM) + error_file: + traceback : Signal 15 (SIGTERM) received by PID 3073 +[2]: + time : 2026-06-14_00:56:27 + host : h100-train-3-gpu-0-9 + rank : 2 (local_rank: 2) + exitcode : -15 (pid: 3074) (SIGTERM) + error_file: + traceback : Signal 15 (SIGTERM) received by PID 3074 +------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-06-14_00:56:27 + host : h100-train-3-gpu-0-9 + rank : 0 (local_rank: 0) + exitcode : -2 (pid: 3072) (SIGINT) + error_file: + traceback : Signal 2 (SIGINT) received by PID 3072 +====================================================== diff --git a/epfl_final/logs/autolaunch_resampled_watcher.log b/epfl_final/logs/autolaunch_resampled_watcher.log new file mode 100644 index 0000000000000000000000000000000000000000..94dcfaeece6229bdacb419fa117db3a9a5f5e7eb --- /dev/null +++ b/epfl_final/logs/autolaunch_resampled_watcher.log @@ -0,0 +1,380 @@ +=== watcher start Wed Jul 8 08:17:58 AM UTC 2026 pid 17245 === +08:17:58 compute_apps=4 orchestrator=alive zerostreak=0 +08:18:28 compute_apps=4 orchestrator=alive zerostreak=0 +08:18:58 compute_apps=4 orchestrator=alive zerostreak=0 +08:19:28 compute_apps=4 orchestrator=alive zerostreak=0 +08:19:58 compute_apps=4 orchestrator=alive zerostreak=0 +08:20:28 compute_apps=4 orchestrator=alive zerostreak=0 +08:20:58 compute_apps=4 orchestrator=alive zerostreak=0 +08:21:28 compute_apps=4 orchestrator=alive zerostreak=0 +08:21:58 compute_apps=4 orchestrator=alive zerostreak=0 +08:22:28 compute_apps=4 orchestrator=alive zerostreak=0 +08:22:58 compute_apps=4 orchestrator=alive zerostreak=0 +08:23:28 compute_apps=4 orchestrator=alive zerostreak=0 +08:23:58 compute_apps=4 orchestrator=alive zerostreak=0 +08:24:29 compute_apps=4 orchestrator=alive zerostreak=0 +08:24:59 compute_apps=4 orchestrator=alive zerostreak=0 +08:25:29 compute_apps=4 orchestrator=alive zerostreak=0 +08:25:59 compute_apps=4 orchestrator=alive zerostreak=0 +08:26:29 compute_apps=4 orchestrator=alive zerostreak=0 +08:26:59 compute_apps=4 orchestrator=alive zerostreak=0 +08:27:29 compute_apps=4 orchestrator=alive zerostreak=0 +08:27:59 compute_apps=4 orchestrator=alive zerostreak=0 +08:28:29 compute_apps=4 orchestrator=alive zerostreak=0 +08:28:59 compute_apps=4 orchestrator=alive zerostreak=0 +08:29:29 compute_apps=4 orchestrator=alive zerostreak=0 +08:29:59 compute_apps=3 orchestrator=alive zerostreak=0 +08:30:29 compute_apps=4 orchestrator=alive zerostreak=0 +08:30:59 compute_apps=4 orchestrator=alive zerostreak=0 +---- hardened watcher swapped in Wed Jul 8 08:31:31 AM UTC 2026 ---- +08:31:31 === hardened watcher start pid 19286 === +08:31:31 compute_apps=4 orchestrator=alive zerostreak=0 +08:32:01 compute_apps=4 orchestrator=alive zerostreak=0 +08:32:31 compute_apps=4 orchestrator=alive zerostreak=0 +08:33:01 compute_apps=4 orchestrator=alive zerostreak=0 +08:33:31 compute_apps=4 orchestrator=alive zerostreak=0 +08:34:01 compute_apps=4 orchestrator=alive zerostreak=0 +08:34:31 compute_apps=4 orchestrator=alive zerostreak=0 +08:35:01 compute_apps=4 orchestrator=alive zerostreak=0 +08:35:31 compute_apps=4 orchestrator=alive zerostreak=0 +08:36:01 compute_apps=4 orchestrator=alive zerostreak=0 +08:36:31 compute_apps=4 orchestrator=alive zerostreak=0 +08:37:01 compute_apps=4 orchestrator=alive zerostreak=0 +08:37:31 compute_apps=4 orchestrator=alive zerostreak=0 +08:38:02 compute_apps=4 orchestrator=alive zerostreak=0 +08:38:32 compute_apps=4 orchestrator=alive zerostreak=0 +08:39:02 compute_apps=4 orchestrator=alive zerostreak=0 +08:39:32 compute_apps=4 orchestrator=alive zerostreak=0 +08:40:02 compute_apps=4 orchestrator=alive zerostreak=0 +08:40:32 compute_apps=4 orchestrator=alive zerostreak=0 +08:41:02 compute_apps=4 orchestrator=alive zerostreak=0 +08:41:32 compute_apps=4 orchestrator=alive zerostreak=0 +08:42:02 compute_apps=4 orchestrator=alive zerostreak=0 +08:42:32 compute_apps=4 orchestrator=alive zerostreak=0 +08:43:02 compute_apps=4 orchestrator=alive zerostreak=0 +08:43:32 compute_apps=4 orchestrator=alive zerostreak=0 +08:44:02 compute_apps=4 orchestrator=alive zerostreak=0 +08:44:32 compute_apps=4 orchestrator=alive zerostreak=0 +08:45:03 compute_apps=4 orchestrator=alive zerostreak=0 +08:45:33 compute_apps=4 orchestrator=alive zerostreak=0 +08:46:03 compute_apps=4 orchestrator=alive zerostreak=0 +08:46:33 compute_apps=4 orchestrator=alive zerostreak=0 +08:47:03 compute_apps=4 orchestrator=alive zerostreak=0 +08:47:33 compute_apps=4 orchestrator=alive zerostreak=0 +08:48:03 compute_apps=4 orchestrator=alive zerostreak=0 +08:48:33 compute_apps=4 orchestrator=alive zerostreak=0 +08:49:03 compute_apps=4 orchestrator=alive zerostreak=0 +08:49:33 compute_apps=4 orchestrator=alive zerostreak=0 +08:50:03 compute_apps=4 orchestrator=alive zerostreak=0 +08:50:33 compute_apps=4 orchestrator=alive zerostreak=0 +08:51:03 compute_apps=4 orchestrator=alive zerostreak=0 +08:51:33 compute_apps=4 orchestrator=alive zerostreak=0 +08:52:04 compute_apps=4 orchestrator=alive zerostreak=0 +08:52:34 compute_apps=4 orchestrator=alive zerostreak=0 +08:53:04 compute_apps=4 orchestrator=alive zerostreak=0 +08:53:34 compute_apps=4 orchestrator=alive zerostreak=0 +08:54:04 compute_apps=4 orchestrator=alive zerostreak=0 +08:54:34 compute_apps=4 orchestrator=alive zerostreak=0 +08:55:04 compute_apps=4 orchestrator=alive zerostreak=0 +08:55:34 compute_apps=4 orchestrator=alive zerostreak=0 +08:56:04 compute_apps=4 orchestrator=alive zerostreak=0 +08:56:34 compute_apps=4 orchestrator=alive zerostreak=0 +08:57:04 compute_apps=3 orchestrator=alive zerostreak=0 +08:57:34 compute_apps=4 orchestrator=alive zerostreak=0 +08:58:04 compute_apps=4 orchestrator=alive zerostreak=0 +08:58:34 compute_apps=4 orchestrator=alive zerostreak=0 +08:59:04 compute_apps=4 orchestrator=alive zerostreak=0 +08:59:35 compute_apps=4 orchestrator=alive zerostreak=0 +09:00:05 compute_apps=4 orchestrator=alive zerostreak=0 +09:00:35 compute_apps=4 orchestrator=alive zerostreak=0 +09:01:05 compute_apps=4 orchestrator=alive zerostreak=0 +09:01:35 compute_apps=4 orchestrator=alive zerostreak=0 +09:02:05 compute_apps=4 orchestrator=alive zerostreak=0 +09:02:35 compute_apps=4 orchestrator=alive zerostreak=0 +09:03:05 compute_apps=4 orchestrator=alive zerostreak=0 +09:03:35 compute_apps=4 orchestrator=alive zerostreak=0 +09:04:05 compute_apps=4 orchestrator=alive zerostreak=0 +09:04:35 compute_apps=4 orchestrator=alive zerostreak=0 +09:05:05 compute_apps=4 orchestrator=alive zerostreak=0 +09:05:35 compute_apps=4 orchestrator=alive zerostreak=0 +09:06:05 compute_apps=4 orchestrator=alive zerostreak=0 +09:06:36 compute_apps=3 orchestrator=alive zerostreak=0 +09:07:06 compute_apps=4 orchestrator=alive zerostreak=0 +09:07:36 compute_apps=4 orchestrator=alive zerostreak=0 +09:08:06 compute_apps=4 orchestrator=alive zerostreak=0 +09:08:36 compute_apps=3 orchestrator=alive zerostreak=0 +09:09:06 compute_apps=4 orchestrator=alive zerostreak=0 +09:09:36 compute_apps=4 orchestrator=alive zerostreak=0 +09:10:06 compute_apps=4 orchestrator=alive zerostreak=0 +09:10:36 compute_apps=4 orchestrator=alive zerostreak=0 +09:11:06 compute_apps=4 orchestrator=alive zerostreak=0 +09:11:36 compute_apps=4 orchestrator=alive zerostreak=0 +09:12:06 compute_apps=4 orchestrator=alive zerostreak=0 +09:12:36 compute_apps=4 orchestrator=alive zerostreak=0 +09:13:06 compute_apps=4 orchestrator=alive zerostreak=0 +09:13:37 compute_apps=4 orchestrator=alive zerostreak=0 +09:14:07 compute_apps=4 orchestrator=alive zerostreak=0 +09:14:37 compute_apps=4 orchestrator=alive zerostreak=0 +09:15:07 compute_apps=4 orchestrator=alive zerostreak=0 +09:15:37 compute_apps=4 orchestrator=alive zerostreak=0 +09:16:07 compute_apps=3 orchestrator=alive zerostreak=0 +09:16:37 compute_apps=4 orchestrator=alive zerostreak=0 +09:17:07 compute_apps=4 orchestrator=alive zerostreak=0 +09:17:37 compute_apps=4 orchestrator=alive zerostreak=0 +09:18:07 compute_apps=3 orchestrator=alive zerostreak=0 +09:18:37 compute_apps=4 orchestrator=alive zerostreak=0 +09:19:07 compute_apps=4 orchestrator=alive zerostreak=0 +09:19:37 compute_apps=4 orchestrator=alive zerostreak=0 +09:20:07 compute_apps=4 orchestrator=alive zerostreak=0 +09:20:38 compute_apps=4 orchestrator=alive zerostreak=0 +09:21:08 compute_apps=4 orchestrator=alive zerostreak=0 +09:21:38 compute_apps=4 orchestrator=alive zerostreak=0 +09:22:08 compute_apps=4 orchestrator=alive zerostreak=0 +09:22:38 compute_apps=4 orchestrator=alive zerostreak=0 +09:23:08 compute_apps=4 orchestrator=alive zerostreak=0 +09:23:38 compute_apps=4 orchestrator=alive zerostreak=0 +09:24:08 compute_apps=4 orchestrator=alive zerostreak=0 +09:24:38 compute_apps=4 orchestrator=alive zerostreak=0 +09:25:08 compute_apps=4 orchestrator=alive zerostreak=0 +09:25:38 compute_apps=3 orchestrator=alive zerostreak=0 +09:26:08 compute_apps=4 orchestrator=alive zerostreak=0 +09:26:38 compute_apps=4 orchestrator=alive zerostreak=0 +09:27:08 compute_apps=4 orchestrator=alive zerostreak=0 +09:27:39 compute_apps=4 orchestrator=alive zerostreak=0 +09:28:09 compute_apps=3 orchestrator=alive zerostreak=0 +09:28:39 compute_apps=4 orchestrator=alive zerostreak=0 +09:29:09 compute_apps=4 orchestrator=alive zerostreak=0 +09:29:39 compute_apps=4 orchestrator=alive zerostreak=0 +09:30:09 compute_apps=4 orchestrator=alive zerostreak=0 +09:30:39 compute_apps=3 orchestrator=alive zerostreak=0 +09:31:09 compute_apps=4 orchestrator=alive zerostreak=0 +09:31:39 compute_apps=3 orchestrator=alive zerostreak=0 +09:32:09 compute_apps=3 orchestrator=alive zerostreak=0 +09:32:39 compute_apps=3 orchestrator=alive zerostreak=0 +09:33:09 compute_apps=2 orchestrator=alive zerostreak=0 +09:33:39 compute_apps=2 orchestrator=alive zerostreak=0 +09:34:10 compute_apps=2 orchestrator=alive zerostreak=0 +09:34:40 compute_apps=2 orchestrator=alive zerostreak=0 +09:35:10 compute_apps=2 orchestrator=alive zerostreak=0 +09:35:40 compute_apps=2 orchestrator=alive zerostreak=0 +09:36:10 compute_apps=2 orchestrator=alive zerostreak=0 +09:36:40 compute_apps=2 orchestrator=alive zerostreak=0 +09:37:10 compute_apps=2 orchestrator=alive zerostreak=0 +09:37:40 compute_apps=2 orchestrator=alive zerostreak=0 +09:38:10 compute_apps=2 orchestrator=alive zerostreak=0 +09:38:40 compute_apps=2 orchestrator=alive zerostreak=0 +09:39:10 compute_apps=2 orchestrator=alive zerostreak=0 +09:39:40 compute_apps=2 orchestrator=alive zerostreak=0 +09:40:10 compute_apps=1 orchestrator=gone zerostreak=0 +09:40:41 compute_apps=1 orchestrator=gone zerostreak=0 +09:41:11 compute_apps=1 orchestrator=gone zerostreak=0 +09:41:41 compute_apps=1 orchestrator=gone zerostreak=0 +09:42:11 compute_apps=1 orchestrator=gone zerostreak=0 +09:42:41 compute_apps=1 orchestrator=gone zerostreak=0 +09:43:11 compute_apps=1 orchestrator=gone zerostreak=0 +09:43:41 compute_apps=1 orchestrator=gone zerostreak=0 +09:44:11 compute_apps=1 orchestrator=gone zerostreak=0 +09:44:41 compute_apps=1 orchestrator=gone zerostreak=0 +09:45:11 compute_apps=1 orchestrator=gone zerostreak=0 +09:45:41 compute_apps=1 orchestrator=gone zerostreak=0 +09:46:11 compute_apps=1 orchestrator=gone zerostreak=0 +09:46:42 compute_apps=1 orchestrator=gone zerostreak=0 +09:47:12 compute_apps=1 orchestrator=gone zerostreak=0 +09:47:42 compute_apps=1 orchestrator=gone zerostreak=0 +09:48:12 compute_apps=1 orchestrator=gone zerostreak=0 +09:48:42 compute_apps=1 orchestrator=gone zerostreak=0 +09:49:12 compute_apps=1 orchestrator=gone zerostreak=0 +09:49:42 compute_apps=1 orchestrator=gone zerostreak=0 +09:50:12 compute_apps=1 orchestrator=gone zerostreak=0 +09:50:42 compute_apps=1 orchestrator=gone zerostreak=0 +09:51:12 compute_apps=1 orchestrator=gone zerostreak=0 +09:51:42 compute_apps=1 orchestrator=gone zerostreak=0 +09:52:12 compute_apps=1 orchestrator=gone zerostreak=0 +09:52:42 compute_apps=1 orchestrator=gone zerostreak=0 +09:53:12 compute_apps=1 orchestrator=gone zerostreak=0 +09:53:43 compute_apps=1 orchestrator=gone zerostreak=0 +09:54:13 compute_apps=1 orchestrator=gone zerostreak=0 +09:54:43 compute_apps=1 orchestrator=gone zerostreak=0 +09:55:13 compute_apps=1 orchestrator=gone zerostreak=0 +09:55:43 compute_apps=1 orchestrator=gone zerostreak=0 +09:56:13 compute_apps=1 orchestrator=gone zerostreak=0 +09:56:43 compute_apps=1 orchestrator=gone zerostreak=0 +09:57:13 compute_apps=1 orchestrator=gone zerostreak=0 +09:57:43 compute_apps=1 orchestrator=gone zerostreak=0 +09:58:13 compute_apps=1 orchestrator=gone zerostreak=0 +09:58:43 compute_apps=1 orchestrator=gone zerostreak=0 +09:59:13 compute_apps=1 orchestrator=gone zerostreak=0 +09:59:43 compute_apps=1 orchestrator=gone zerostreak=0 +10:00:14 compute_apps=1 orchestrator=gone zerostreak=0 +10:00:44 compute_apps=1 orchestrator=gone zerostreak=0 +10:01:14 compute_apps=1 orchestrator=gone zerostreak=0 +10:01:44 compute_apps=1 orchestrator=gone zerostreak=0 +10:02:14 compute_apps=1 orchestrator=gone zerostreak=0 +10:02:44 compute_apps=1 orchestrator=gone zerostreak=0 +10:03:14 compute_apps=1 orchestrator=gone zerostreak=0 +10:03:44 compute_apps=1 orchestrator=gone zerostreak=0 +10:04:14 compute_apps=1 orchestrator=gone zerostreak=0 +10:04:44 compute_apps=1 orchestrator=gone zerostreak=0 +10:05:14 compute_apps=1 orchestrator=gone zerostreak=0 +10:05:44 compute_apps=1 orchestrator=gone zerostreak=0 +10:06:14 compute_apps=1 orchestrator=gone zerostreak=0 +10:06:45 compute_apps=1 orchestrator=gone zerostreak=0 +10:07:15 compute_apps=1 orchestrator=gone zerostreak=0 +10:07:45 compute_apps=1 orchestrator=gone zerostreak=0 +10:08:15 compute_apps=1 orchestrator=gone zerostreak=0 +10:08:45 compute_apps=1 orchestrator=gone zerostreak=0 +10:09:15 compute_apps=1 orchestrator=gone zerostreak=0 +10:09:45 compute_apps=1 orchestrator=gone zerostreak=0 +10:10:15 compute_apps=1 orchestrator=gone zerostreak=0 +10:10:45 compute_apps=1 orchestrator=gone zerostreak=0 +10:11:15 compute_apps=1 orchestrator=gone zerostreak=0 +10:11:45 compute_apps=1 orchestrator=gone zerostreak=0 +10:12:15 compute_apps=1 orchestrator=gone zerostreak=0 +10:12:45 compute_apps=1 orchestrator=gone zerostreak=0 +10:13:16 compute_apps=1 orchestrator=gone zerostreak=0 +10:13:46 compute_apps=1 orchestrator=gone zerostreak=0 +10:14:16 compute_apps=1 orchestrator=gone zerostreak=0 +10:14:46 compute_apps=1 orchestrator=gone zerostreak=0 +10:15:16 compute_apps=1 orchestrator=gone zerostreak=0 +10:15:46 compute_apps=1 orchestrator=gone zerostreak=0 +10:16:16 compute_apps=1 orchestrator=gone zerostreak=0 +10:16:46 compute_apps=1 orchestrator=gone zerostreak=0 +10:17:16 compute_apps=1 orchestrator=gone zerostreak=0 +10:17:46 compute_apps=1 orchestrator=gone zerostreak=0 +10:18:16 compute_apps=1 orchestrator=gone zerostreak=0 +10:18:46 compute_apps=1 orchestrator=gone zerostreak=0 +10:19:16 compute_apps=1 orchestrator=gone zerostreak=0 +10:19:47 compute_apps=1 orchestrator=gone zerostreak=0 +10:20:17 compute_apps=1 orchestrator=gone zerostreak=0 +10:20:47 compute_apps=1 orchestrator=gone zerostreak=0 +10:21:17 compute_apps=1 orchestrator=gone zerostreak=0 +10:21:47 compute_apps=1 orchestrator=gone zerostreak=0 +10:22:17 compute_apps=1 orchestrator=gone zerostreak=0 +10:22:47 compute_apps=1 orchestrator=gone zerostreak=0 +10:23:17 compute_apps=1 orchestrator=gone zerostreak=0 +10:23:47 compute_apps=1 orchestrator=gone zerostreak=0 +10:24:17 compute_apps=1 orchestrator=gone zerostreak=0 +10:24:47 compute_apps=1 orchestrator=gone zerostreak=0 +10:25:17 compute_apps=1 orchestrator=gone zerostreak=0 +10:25:47 compute_apps=1 orchestrator=gone zerostreak=0 +10:26:17 compute_apps=1 orchestrator=gone zerostreak=0 +10:26:48 compute_apps=1 orchestrator=gone zerostreak=0 +10:27:18 compute_apps=1 orchestrator=gone zerostreak=0 +10:27:48 compute_apps=1 orchestrator=gone zerostreak=0 +10:28:18 compute_apps=1 orchestrator=gone zerostreak=0 +10:28:48 compute_apps=1 orchestrator=gone zerostreak=0 +10:29:18 compute_apps=1 orchestrator=gone zerostreak=0 +10:29:48 compute_apps=1 orchestrator=gone zerostreak=0 +10:30:18 compute_apps=1 orchestrator=gone zerostreak=0 +10:30:48 compute_apps=1 orchestrator=gone zerostreak=0 +10:31:18 compute_apps=1 orchestrator=gone zerostreak=0 +10:31:48 compute_apps=1 orchestrator=gone zerostreak=0 +10:32:18 compute_apps=1 orchestrator=gone zerostreak=0 +10:32:48 compute_apps=1 orchestrator=gone zerostreak=0 +10:33:19 compute_apps=1 orchestrator=gone zerostreak=0 +10:33:49 compute_apps=1 orchestrator=gone zerostreak=0 +10:34:19 compute_apps=1 orchestrator=gone zerostreak=0 +10:34:49 compute_apps=1 orchestrator=gone zerostreak=0 +10:35:19 compute_apps=1 orchestrator=gone zerostreak=0 +10:35:49 compute_apps=1 orchestrator=gone zerostreak=0 +10:36:19 compute_apps=1 orchestrator=gone zerostreak=0 +10:36:49 compute_apps=1 orchestrator=gone zerostreak=0 +10:37:19 compute_apps=1 orchestrator=gone zerostreak=0 +10:37:49 compute_apps=1 orchestrator=gone zerostreak=0 +10:38:19 compute_apps=1 orchestrator=gone zerostreak=0 +10:38:49 compute_apps=1 orchestrator=gone zerostreak=0 +10:39:19 compute_apps=1 orchestrator=gone zerostreak=0 +10:39:50 compute_apps=1 orchestrator=gone zerostreak=0 +10:40:20 compute_apps=1 orchestrator=gone zerostreak=0 +10:40:50 compute_apps=1 orchestrator=gone zerostreak=0 +10:41:20 compute_apps=1 orchestrator=gone zerostreak=0 +10:41:50 compute_apps=1 orchestrator=gone zerostreak=0 +10:42:20 compute_apps=1 orchestrator=gone zerostreak=0 +10:42:50 compute_apps=1 orchestrator=gone zerostreak=0 +10:43:20 compute_apps=1 orchestrator=gone zerostreak=0 +10:43:50 compute_apps=1 orchestrator=gone zerostreak=0 +10:44:20 compute_apps=1 orchestrator=gone zerostreak=0 +10:44:50 compute_apps=1 orchestrator=gone zerostreak=0 +10:45:20 compute_apps=1 orchestrator=gone zerostreak=0 +10:45:50 compute_apps=1 orchestrator=gone zerostreak=0 +10:46:21 compute_apps=1 orchestrator=gone zerostreak=0 +10:46:51 compute_apps=1 orchestrator=gone zerostreak=0 +10:47:21 compute_apps=1 orchestrator=gone zerostreak=0 +10:47:51 compute_apps=1 orchestrator=gone zerostreak=0 +10:48:21 compute_apps=1 orchestrator=gone zerostreak=0 +10:48:51 compute_apps=1 orchestrator=gone zerostreak=0 +10:49:21 compute_apps=1 orchestrator=gone zerostreak=0 +10:49:51 compute_apps=1 orchestrator=gone zerostreak=0 +10:50:21 compute_apps=1 orchestrator=gone zerostreak=0 +10:50:51 compute_apps=1 orchestrator=gone zerostreak=0 +10:51:21 compute_apps=1 orchestrator=gone zerostreak=0 +10:51:51 compute_apps=1 orchestrator=gone zerostreak=0 +10:52:21 compute_apps=1 orchestrator=gone zerostreak=0 +10:52:52 compute_apps=1 orchestrator=gone zerostreak=0 +10:53:22 compute_apps=1 orchestrator=gone zerostreak=0 +10:53:52 compute_apps=1 orchestrator=gone zerostreak=0 +10:54:22 compute_apps=1 orchestrator=gone zerostreak=0 +10:54:52 compute_apps=1 orchestrator=gone zerostreak=0 +10:55:22 compute_apps=1 orchestrator=gone zerostreak=0 +10:55:52 compute_apps=1 orchestrator=gone zerostreak=0 +10:56:22 compute_apps=1 orchestrator=gone zerostreak=0 +10:56:52 compute_apps=1 orchestrator=gone zerostreak=0 +10:57:22 compute_apps=1 orchestrator=gone zerostreak=0 +10:57:52 compute_apps=1 orchestrator=gone zerostreak=0 +10:58:22 compute_apps=1 orchestrator=gone zerostreak=0 +10:58:52 compute_apps=1 orchestrator=gone zerostreak=0 +10:59:23 compute_apps=1 orchestrator=gone zerostreak=0 +10:59:53 compute_apps=1 orchestrator=gone zerostreak=0 +11:00:23 compute_apps=1 orchestrator=gone zerostreak=0 +11:00:53 compute_apps=1 orchestrator=gone zerostreak=0 +11:01:23 compute_apps=1 orchestrator=gone zerostreak=0 +11:01:53 compute_apps=1 orchestrator=gone zerostreak=0 +11:02:23 compute_apps=1 orchestrator=gone zerostreak=0 +11:02:53 compute_apps=1 orchestrator=gone zerostreak=0 +11:03:23 compute_apps=1 orchestrator=gone zerostreak=0 +11:03:53 compute_apps=1 orchestrator=gone zerostreak=0 +11:04:23 compute_apps=1 orchestrator=gone zerostreak=0 +11:04:53 compute_apps=1 orchestrator=gone zerostreak=0 +11:05:23 compute_apps=1 orchestrator=gone zerostreak=0 +11:05:53 compute_apps=1 orchestrator=gone zerostreak=0 +11:06:24 compute_apps=1 orchestrator=gone zerostreak=0 +11:06:54 compute_apps=1 orchestrator=gone zerostreak=0 +11:07:24 compute_apps=1 orchestrator=gone zerostreak=0 +11:07:54 compute_apps=1 orchestrator=gone zerostreak=0 +11:08:24 compute_apps=1 orchestrator=gone zerostreak=0 +11:08:54 compute_apps=1 orchestrator=gone zerostreak=0 +11:09:24 compute_apps=1 orchestrator=gone zerostreak=0 +11:09:54 compute_apps=1 orchestrator=gone zerostreak=0 +11:10:24 compute_apps=1 orchestrator=gone zerostreak=0 +11:10:54 compute_apps=1 orchestrator=gone zerostreak=0 +11:11:24 compute_apps=1 orchestrator=gone zerostreak=0 +11:11:54 compute_apps=1 orchestrator=gone zerostreak=0 +11:12:24 compute_apps=1 orchestrator=gone zerostreak=0 +11:12:55 compute_apps=1 orchestrator=gone zerostreak=0 +11:13:25 compute_apps=1 orchestrator=gone zerostreak=0 +11:13:55 compute_apps=1 orchestrator=gone zerostreak=0 +11:14:25 compute_apps=1 orchestrator=gone zerostreak=0 +11:14:55 compute_apps=1 orchestrator=gone zerostreak=0 +11:15:25 compute_apps=1 orchestrator=gone zerostreak=0 +11:15:55 compute_apps=1 orchestrator=gone zerostreak=0 +11:16:25 compute_apps=1 orchestrator=gone zerostreak=0 +11:16:55 compute_apps=1 orchestrator=gone zerostreak=0 +11:17:25 compute_apps=1 orchestrator=gone zerostreak=0 +11:17:55 compute_apps=1 orchestrator=gone zerostreak=0 +11:18:25 compute_apps=1 orchestrator=gone zerostreak=0 +11:18:55 compute_apps=1 orchestrator=gone zerostreak=0 +11:19:25 compute_apps=1 orchestrator=gone zerostreak=0 +11:19:56 compute_apps=1 orchestrator=gone zerostreak=0 +11:20:26 compute_apps=1 orchestrator=gone zerostreak=0 +11:20:56 compute_apps=0 orchestrator=gone zerostreak=1 +11:21:26 compute_apps=0 orchestrator=gone zerostreak=2 +11:21:56 compute_apps=0 orchestrator=gone zerostreak=3 +11:21:56 ALL CLEAR (90s sustained) +11:21:56 launch attempt 1/3 +11:21:56 LAUNCH: torchrun 4-GPU resampled +11:24:56 VERIFY OK: procs=21 compute_apps=4 +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/hn868f80 +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +11:24:56 === SUCCESS: resampled launched+verified === +11:24:56 === watcher done === diff --git a/epfl_final/logs/eval_30k_battery.log b/epfl_final/logs/eval_30k_battery.log new file mode 100644 index 0000000000000000000000000000000000000000..bd9558188cad623d4d2b1db82c5053462d6de3a9 --- /dev/null +++ b/epfl_final/logs/eval_30k_battery.log @@ -0,0 +1,28 @@ +[21:18:46] ckpt resampled -> /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11700 +[21:18:46] ckpt fixed_s1 -> /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11700 +[21:18:46] ckpt fixed_s2 -> /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11700 +[21:18:46] ckpt dense -> /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11700 +[21:18:46] total jobs=104 already-done=0 gpus=[0, 1, 2] n=100 +[21:18:46] START gpu0 RULER_niah_multikey_resampled_ddense (~104 remaining) +[21:18:46] START gpu1 RULER_niah_multikey_resampled_d120 (~104 remaining) +[21:18:46] START gpu2 RULER_niah_multikey_resampled_d60 (~104 remaining) +[21:20:46] DONE gpu0 RULER_niah_multikey_resampled_ddense rc=0 ok +[21:20:46] START gpu0 RULER_niah_multikey_resampled_d31 (~103 remaining) +[21:36:31] DONE gpu0 RULER_niah_multikey_resampled_d31 rc=0 ok +[21:36:31] START gpu0 RULER_niah_multikey_resampled_d16 (~102 remaining) +[21:45:07] DONE gpu2 RULER_niah_multikey_resampled_d60 rc=0 ok +[21:45:07] START gpu2 RULER_niah_multikey_fixed_s1_ddense (~101 remaining) +[21:46:35] DONE gpu0 RULER_niah_multikey_resampled_d16 rc=0 ok +[21:46:35] START gpu0 RULER_niah_multikey_fixed_s1_d120 (~100 remaining) +[21:46:55] DONE gpu2 RULER_niah_multikey_fixed_s1_ddense rc=0 ok +[21:46:55] START gpu2 RULER_niah_multikey_fixed_s1_d60 (~99 remaining) +[22:07:13] DONE gpu1 RULER_niah_multikey_resampled_d120 rc=0 ok +[22:07:13] START gpu1 RULER_niah_multikey_fixed_s1_d31 (~98 remaining) +[22:13:05] DONE gpu2 RULER_niah_multikey_fixed_s1_d60 rc=0 ok +[22:13:05] START gpu2 RULER_niah_multikey_fixed_s1_d16 (~97 remaining) +[22:22:46] DONE gpu1 RULER_niah_multikey_fixed_s1_d31 rc=0 ok +[22:22:46] START gpu1 RULER_niah_multikey_fixed_s2_ddense (~96 remaining) +[22:23:10] DONE gpu2 RULER_niah_multikey_fixed_s1_d16 rc=0 ok +[22:23:10] START gpu2 RULER_niah_multikey_fixed_s2_d120 (~95 remaining) +[22:24:34] DONE gpu1 RULER_niah_multikey_fixed_s2_ddense rc=0 ok +[22:24:34] START gpu1 RULER_niah_multikey_fixed_s2_d60 (~94 remaining) diff --git a/epfl_final/logs/l30k1_2b_dense.log b/epfl_final/logs/l30k1_2b_dense.log new file mode 100644 index 0000000000000000000000000000000000000000..bd9a57707b2218dbbfe18656539d521afc83294a --- /dev/null +++ b/epfl_final/logs/l30k1_2b_dense.log @@ -0,0 +1,365 @@ +W0708 17:29:07.301000 6079 site-packages/torch/distributed/run.py:851] +W0708 17:29:07.301000 6079 site-packages/torch/distributed/run.py:851] ***************************************** +W0708 17:29:07.301000 6079 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0708 17:29:07.301000 6079 site-packages/torch/distributed/run.py:851] ***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 4, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300", + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 300, + "eval_steps": 500, + "eval_batches": 20, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": true, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "1.2b dense 30k longctx", + "wandb_tags": [ + "qwen3", + "1.2b", + "dense", + "30k", + "longctx", + "b128", + "scaleladder" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300 +Checkpoint activations: True +Checkpoint activations: TrueCheckpoint activations: True + +Checkpoint activations: True +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 1536] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 1536] [1, 30720, 1536] 1,536 True +├─Linear: 1-2 [1, 30720, 1536] [1, 30720, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (Units.GIGABYTES): 1.44 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 98319.73 +Params size (MB): 2871.45 +Estimated Total Size (MB): 101191.43 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... + + +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) + + + +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 187325 +Optimizer steps per epoch: 37465 +Total optimizer steps: 37465 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 3300 (epoch 0, micro_step 16500) +Resuming training from step 3300 +Resuming wandb run cd66e13p +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run cd66e13p +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_173250-cd66e13p +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 1.2b dense 30k longctx +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/cd66e13p +Starting training... +Validation at step 3300: Loss = 2.5783, Perplexity = 13.17 +wandb: WARNING Tried to log to step 3300 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 3301: dense (dense step interval) +Step 3310/37465 | Loss: 2.6606 | LR: 4.94e-04 | 16887.6ms/step | 1.2 samples/s +wandb: WARNING Tried to log to step 3310 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3320/37465 | Loss: 2.3535 | LR: 4.94e-04 | 8333.4ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3320 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3330/37465 | Loss: 2.6802 | LR: 4.94e-04 | 8352.0ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3330 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3340/37465 | Loss: 2.6431 | LR: 4.94e-04 | 8344.2ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3340 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3350/37465 | Loss: 2.9170 | LR: 4.94e-04 | 8354.5ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3350 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3360/37465 | Loss: 2.5131 | LR: 4.94e-04 | 8360.2ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3360 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3370/37465 | Loss: 2.5432 | LR: 4.94e-04 | 8350.8ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3370 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3380/37465 | Loss: 2.5797 | LR: 4.94e-04 | 8388.2ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3380 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3390/37465 | Loss: 2.4650 | LR: 4.94e-04 | 8355.6ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3390 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3400/37465 | Loss: 2.3816 | LR: 4.94e-04 | 8338.4ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3400 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3410/37465 | Loss: 2.4314 | LR: 4.94e-04 | 8332.0ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3410 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3420/37465 | Loss: 2.5678 | LR: 4.94e-04 | 8368.0ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3420 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3430/37465 | Loss: 2.7321 | LR: 4.93e-04 | 8339.8ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 3430 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 3440/37465 | Loss: 2.6158 | LR: 4.93e-04 | 8360.0ms/step | 2.4 samples/s +Step 3450/37465 | Loss: 2.5854 | LR: 4.93e-04 | 8369.7ms/step | 2.4 samples/s +Step 3460/37465 | Loss: 2.4706 | LR: 4.93e-04 | 8346.4ms/step | 2.4 samples/s +Step 3470/37465 | Loss: 2.3381 | LR: 4.93e-04 | 8365.9ms/step | 2.4 samples/s +Step 3480/37465 | Loss: 2.4953 | LR: 4.93e-04 | 8343.1ms/step | 2.4 samples/s +Step 3490/37465 | Loss: 2.6281 | LR: 4.93e-04 | 8368.3ms/step | 2.4 samples/s +Step 3500/37465 | Loss: 2.5841 | LR: 4.93e-04 | 8351.0ms/step | 2.4 samples/s +Validation at step 3500: Loss = 2.5524, Perplexity = 12.84 +Step 3510/37465 | Loss: 2.7820 | LR: 4.93e-04 | 9361.4ms/step | 2.1 samples/s +Step 3520/37465 | Loss: 2.4614 | LR: 4.93e-04 | 8371.8ms/step | 2.4 samples/s +Step 3530/37465 | Loss: 2.5079 | LR: 4.93e-04 | 8353.9ms/step | 2.4 samples/s +Step 3540/37465 | Loss: 2.6692 | LR: 4.93e-04 | 8370.6ms/step | 2.4 samples/s +Step 3550/37465 | Loss: 2.5327 | LR: 4.93e-04 | 8378.5ms/step | 2.4 samples/s +Step 3560/37465 | Loss: 2.4459 | LR: 4.93e-04 | 8348.7ms/step | 2.4 samples/s +Step 3570/37465 | Loss: 2.4465 | LR: 4.93e-04 | 8366.9ms/step | 2.4 samples/s +Step 3580/37465 | Loss: 2.8736 | LR: 4.93e-04 | 8373.4ms/step | 2.4 samples/s +Step 3590/37465 | Loss: 2.5086 | LR: 4.93e-04 | 8355.8ms/step | 2.4 samples/s +Step 3600/37465 | Loss: 2.6325 | LR: 4.93e-04 | 8345.8ms/step | 2.4 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3600 +Step 3610/37465 | Loss: 2.5135 | LR: 4.93e-04 | 9375.9ms/step | 2.1 samples/s +Step 3620/37465 | Loss: 2.4450 | LR: 4.92e-04 | 8396.5ms/step | 2.4 samples/s +Step 3630/37465 | Loss: 2.3774 | LR: 4.92e-04 | 8368.6ms/step | 2.4 samples/s +Step 3640/37465 | Loss: 2.4915 | LR: 4.92e-04 | 8383.0ms/step | 2.4 samples/s +Step 3650/37465 | Loss: 2.5852 | LR: 4.92e-04 | 8368.3ms/step | 2.4 samples/s +Step 3660/37465 | Loss: 2.4715 | LR: 4.92e-04 | 8369.6ms/step | 2.4 samples/s +Step 3670/37465 | Loss: 2.3686 | LR: 4.92e-04 | 8378.3ms/step | 2.4 samples/s +Step 3680/37465 | Loss: 2.6847 | LR: 4.92e-04 | 8372.5ms/step | 2.4 samples/s +Step 3690/37465 | Loss: 2.8277 | LR: 4.92e-04 | 8361.6ms/step | 2.4 samples/s +Step 3700/37465 | Loss: 2.4927 | LR: 4.92e-04 | 8381.6ms/step | 2.4 samples/s +Step 3710/37465 | Loss: 2.7802 | LR: 4.92e-04 | 8385.2ms/step | 2.4 samples/s +Step 3720/37465 | Loss: 2.5450 | LR: 4.92e-04 | 8361.8ms/step | 2.4 samples/s +Step 3730/37465 | Loss: 2.7159 | LR: 4.92e-04 | 8368.8ms/step | 2.4 samples/s +Step 3740/37465 | Loss: 2.3565 | LR: 4.92e-04 | 8356.7ms/step | 2.4 samples/s +Step 3750/37465 | Loss: 2.5005 | LR: 4.92e-04 | 8359.8ms/step | 2.4 samples/s +Step 3760/37465 | Loss: 2.7336 | LR: 4.92e-04 | 8391.1ms/step | 2.4 samples/s +Step 3770/37465 | Loss: 2.7054 | LR: 4.92e-04 | 8368.4ms/step | 2.4 samples/s +Step 3780/37465 | Loss: 2.5842 | LR: 4.92e-04 | 8387.8ms/step | 2.4 samples/s +Step 3790/37465 | Loss: 2.4455 | LR: 4.92e-04 | 8385.2ms/step | 2.4 samples/s +Step 3800/37465 | Loss: 2.5453 | LR: 4.92e-04 | 8380.5ms/step | 2.4 samples/s +Step 3810/37465 | Loss: 2.6389 | LR: 4.91e-04 | 8374.7ms/step | 2.4 samples/s +Step 3820/37465 | Loss: 2.5859 | LR: 4.91e-04 | 8377.2ms/step | 2.4 samples/s +Step 3830/37465 | Loss: 2.7157 | LR: 4.91e-04 | 8396.2ms/step | 2.4 samples/s +Step 3840/37465 | Loss: 2.2813 | LR: 4.91e-04 | 8359.1ms/step | 2.4 samples/s +Step 3850/37465 | Loss: 2.6667 | LR: 4.91e-04 | 8374.8ms/step | 2.4 samples/s +Step 3860/37465 | Loss: 2.3919 | LR: 4.91e-04 | 8379.6ms/step | 2.4 samples/s +Step 3870/37465 | Loss: 2.7215 | LR: 4.91e-04 | 8365.4ms/step | 2.4 samples/s +Step 3880/37465 | Loss: 2.3145 | LR: 4.91e-04 | 8378.0ms/step | 2.4 samples/s +Step 3890/37465 | Loss: 2.3002 | LR: 4.91e-04 | 8357.4ms/step | 2.4 samples/s +Step 3900/37465 | Loss: 2.3930 | LR: 4.91e-04 | 8363.6ms/step | 2.4 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3900 +Step 3910/37465 | Loss: 2.5740 | LR: 4.91e-04 | 9211.5ms/step | 2.2 samples/s +Step 3920/37465 | Loss: 2.5662 | LR: 4.91e-04 | 8365.1ms/step | 2.4 samples/s +Step 3930/37465 | Loss: 2.6290 | LR: 4.91e-04 | 8380.3ms/step | 2.4 samples/s +Step 3940/37465 | Loss: 2.8159 | LR: 4.91e-04 | 8356.3ms/step | 2.4 samples/s +Step 3950/37465 | Loss: 2.3214 | LR: 4.91e-04 | 8372.7ms/step | 2.4 samples/s +Step 3960/37465 | Loss: 2.5481 | LR: 4.91e-04 | 8354.2ms/step | 2.4 samples/s +Step 3970/37465 | Loss: 2.3466 | LR: 4.91e-04 | 8354.4ms/step | 2.4 samples/s +Step 3980/37465 | Loss: 2.5501 | LR: 4.91e-04 | 8370.4ms/step | 2.4 samples/s +Step 3990/37465 | Loss: 2.5272 | LR: 4.90e-04 | 8368.3ms/step | 2.4 samples/s +Step 4000/37465 | Loss: 2.5115 | LR: 4.90e-04 | 8344.0ms/step | 2.4 samples/s +Validation at step 4000: Loss = 2.5080, Perplexity = 12.28 +Step 4010/37465 | Loss: 2.1113 | LR: 4.90e-04 | 9366.2ms/step | 2.1 samples/s +Step 4020/37465 | Loss: 2.2947 | LR: 4.90e-04 | 8339.8ms/step | 2.4 samples/s +Step 4030/37465 | Loss: 2.4372 | LR: 4.90e-04 | 8352.1ms/step | 2.4 samples/s +Step 4040/37465 | Loss: 2.4509 | LR: 4.90e-04 | 8351.1ms/step | 2.4 samples/s +Step 4050/37465 | Loss: 2.6699 | LR: 4.90e-04 | 8366.0ms/step | 2.4 samples/s +Step 4060/37465 | Loss: 2.5102 | LR: 4.90e-04 | 8357.5ms/step | 2.4 samples/s +Step 4070/37465 | Loss: 2.4825 | LR: 4.90e-04 | 8367.2ms/step | 2.4 samples/s +Step 4080/37465 | Loss: 2.4585 | LR: 4.90e-04 | 8340.0ms/step | 2.4 samples/s +Step 4090/37465 | Loss: 2.5597 | LR: 4.90e-04 | 8359.6ms/step | 2.4 samples/s +Step 4100/37465 | Loss: 2.3972 | LR: 4.90e-04 | 8364.1ms/step | 2.4 samples/s +Step 4110/37465 | Loss: 2.5406 | LR: 4.90e-04 | 8352.6ms/step | 2.4 samples/s +Step 4120/37465 | Loss: 2.3038 | LR: 4.90e-04 | 8352.6ms/step | 2.4 samples/s +Step 4130/37465 | Loss: 2.2725 | LR: 4.90e-04 | 8359.4ms/step | 2.4 samples/s +Step 4140/37465 | Loss: 2.5664 | LR: 4.90e-04 | 8358.2ms/step | 2.4 samples/s +Step 4150/37465 | Loss: 2.5551 | LR: 4.89e-04 | 8347.4ms/step | 2.4 samples/s +Step 4160/37465 | Loss: 2.5973 | LR: 4.89e-04 | 8338.1ms/step | 2.4 samples/s +Step 4170/37465 | Loss: 2.4961 | LR: 4.89e-04 | 8352.2ms/step | 2.4 samples/s +Step 4180/37465 | Loss: 2.2842 | LR: 4.89e-04 | 8351.2ms/step | 2.4 samples/s +Step 4190/37465 | Loss: 2.5623 | LR: 4.89e-04 | 8356.2ms/step | 2.4 samples/s +Step 4200/37465 | Loss: 2.5244 | LR: 4.89e-04 | 8359.8ms/step | 2.4 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4200 +Step 4210/37465 | Loss: 2.1315 | LR: 4.89e-04 | 9181.9ms/step | 2.2 samples/s +Step 4220/37465 | Loss: 2.4843 | LR: 4.89e-04 | 8349.2ms/step | 2.4 samples/s +Step 4230/37465 | Loss: 2.3858 | LR: 4.89e-04 | 8357.1ms/step | 2.4 samples/s +Step 4240/37465 | Loss: 2.4300 | LR: 4.89e-04 | 8369.7ms/step | 2.4 samples/s +Step 4250/37465 | Loss: 2.3458 | LR: 4.89e-04 | 8377.7ms/step | 2.4 samples/s +Step 4260/37465 | Loss: 2.4023 | LR: 4.89e-04 | 8367.7ms/step | 2.4 samples/s +Step 4270/37465 | Loss: 2.4831 | LR: 4.89e-04 | 8347.0ms/step | 2.4 samples/s +Step 4280/37465 | Loss: 2.4608 | LR: 4.89e-04 | 8362.1ms/step | 2.4 samples/s +Step 4290/37465 | Loss: 2.5375 | LR: 4.89e-04 | 8377.8ms/step | 2.4 samples/s +Step 4300/37465 | Loss: 2.4230 | LR: 4.89e-04 | 8363.5ms/step | 2.4 samples/s +Step 4310/37465 | Loss: 2.6084 | LR: 4.88e-04 | 8364.0ms/step | 2.4 samples/s +Step 4320/37465 | Loss: 2.3435 | LR: 4.88e-04 | 8351.6ms/step | 2.4 samples/s +Step 4330/37465 | Loss: 2.2068 | LR: 4.88e-04 | 8377.3ms/step | 2.4 samples/s +Step 4340/37465 | Loss: 2.5103 | LR: 4.88e-04 | 8364.9ms/step | 2.4 samples/s +Step 4350/37465 | Loss: 2.6194 | LR: 4.88e-04 | 8333.5ms/step | 2.4 samples/s +Step 4360/37465 | Loss: 2.5892 | LR: 4.88e-04 | 8381.0ms/step | 2.4 samples/s +Step 4370/37465 | Loss: 2.4106 | LR: 4.88e-04 | 8374.3ms/step | 2.4 samples/s +Step 4380/37465 | Loss: 2.4141 | LR: 4.88e-04 | 8380.8ms/step | 2.4 samples/s +Step 4390/37465 | Loss: 2.5619 | LR: 4.88e-04 | 8334.0ms/step | 2.4 samples/s +Step 4400/37465 | Loss: 2.3260 | LR: 4.88e-04 | 8380.4ms/step | 2.4 samples/s +Step 4410/37465 | Loss: 2.5833 | LR: 4.88e-04 | 8362.6ms/step | 2.4 samples/s +Step 4420/37465 | Loss: 2.3065 | LR: 4.88e-04 | 8388.2ms/step | 2.4 samples/s +Step 4430/37465 | Loss: 2.5054 | LR: 4.88e-04 | 8416.8ms/step | 2.4 samples/s +Step 4440/37465 | Loss: 2.3477 | LR: 4.88e-04 | 8370.3ms/step | 2.4 samples/s +Step 4450/37465 | Loss: 2.4271 | LR: 4.88e-04 | 8381.1ms/step | 2.4 samples/s +Step 4460/37465 | Loss: 2.3098 | LR: 4.88e-04 | 8334.6ms/step | 2.4 samples/s +Step 4470/37465 | Loss: 2.3284 | LR: 4.87e-04 | 8364.4ms/step | 2.4 samples/s +Step 4480/37465 | Loss: 2.3389 | LR: 4.87e-04 | 8353.3ms/step | 2.4 samples/s +Step 4490/37465 | Loss: 2.4233 | LR: 4.87e-04 | 8360.4ms/step | 2.4 samples/s +Step 4500/37465 | Loss: 2.6390 | LR: 4.87e-04 | 8391.1ms/step | 2.4 samples/s +Validation at step 4500: Loss = 2.4759, Perplexity = 11.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4500 +Step 4510/37465 | Loss: 2.3560 | LR: 4.87e-04 | 10250.6ms/step | 2.0 samples/s +Step 4520/37465 | Loss: 2.4278 | LR: 4.87e-04 | 8351.7ms/step | 2.4 samples/s +Step 4530/37465 | Loss: 2.5453 | LR: 4.87e-04 | 8374.4ms/step | 2.4 samples/s +Step 4540/37465 | Loss: 2.4651 | LR: 4.87e-04 | 8386.8ms/step | 2.4 samples/s +Step 4550/37465 | Loss: 2.4067 | LR: 4.87e-04 | 8401.2ms/step | 2.4 samples/s +Step 4560/37465 | Loss: 2.4239 | LR: 4.87e-04 | 8392.5ms/step | 2.4 samples/s +Step 4570/37465 | Loss: 2.2015 | LR: 4.87e-04 | 8395.1ms/step | 2.4 samples/s +Step 4580/37465 | Loss: 2.5254 | LR: 4.87e-04 | 8394.8ms/step | 2.4 samples/s +Step 4590/37465 | Loss: 2.4194 | LR: 4.87e-04 | 8396.8ms/step | 2.4 samples/s +Step 4600/37465 | Loss: 2.6416 | LR: 4.87e-04 | 8384.7ms/step | 2.4 samples/s +Step 4610/37465 | Loss: 2.3960 | LR: 4.86e-04 | 8380.4ms/step | 2.4 samples/s +Step 4620/37465 | Loss: 2.5731 | LR: 4.86e-04 | 8385.8ms/step | 2.4 samples/s +Step 4630/37465 | Loss: 2.5278 | LR: 4.86e-04 | 8355.9ms/step | 2.4 samples/s +Step 4640/37465 | Loss: 2.3128 | LR: 4.86e-04 | 8368.9ms/step | 2.4 samples/s +Step 4650/37465 | Loss: 2.2687 | LR: 4.86e-04 | 8420.4ms/step | 2.4 samples/s +Step 4660/37465 | Loss: 2.4507 | LR: 4.86e-04 | 8403.7ms/step | 2.4 samples/s +Step 4670/37465 | Loss: 2.4984 | LR: 4.86e-04 | 8415.4ms/step | 2.4 samples/s +Step 4680/37465 | Loss: 2.4701 | LR: 4.86e-04 | 8359.8ms/step | 2.4 samples/s +Step 4690/37465 | Loss: 2.6007 | LR: 4.86e-04 | 8353.6ms/step | 2.4 samples/s +Step 4700/37465 | Loss: 2.4826 | LR: 4.86e-04 | 8402.5ms/step | 2.4 samples/s +Step 4710/37465 | Loss: 2.2913 | LR: 4.86e-04 | 8369.6ms/step | 2.4 samples/s +Step 4720/37465 | Loss: 2.5531 | LR: 4.86e-04 | 8378.1ms/step | 2.4 samples/s +Step 4730/37465 | Loss: 2.3186 | LR: 4.86e-04 | 8362.9ms/step | 2.4 samples/s +Step 4740/37465 | Loss: 2.4826 | LR: 4.86e-04 | 8392.5ms/step | 2.4 samples/s +Step 4750/37465 | Loss: 2.4423 | LR: 4.85e-04 | 8381.7ms/step | 2.4 samples/s +Step 4760/37465 | Loss: 2.3839 | LR: 4.85e-04 | 8358.8ms/step | 2.4 samples/s +Step 4770/37465 | Loss: 2.3475 | LR: 4.85e-04 | 8386.0ms/step | 2.4 samples/s +Step 4780/37465 | Loss: 2.5026 | LR: 4.85e-04 | 8373.9ms/step | 2.4 samples/s +Step 4790/37465 | Loss: 2.6148 | LR: 4.85e-04 | 8389.5ms/step | 2.4 samples/s +Step 4800/37465 | Loss: 2.4976 | LR: 4.85e-04 | 8363.1ms/step | 2.4 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4800 +Step 4810/37465 | Loss: 2.5211 | LR: 4.85e-04 | 9312.5ms/step | 2.1 samples/s diff --git a/epfl_final/logs/l30k1_2b_dense.log.old2 b/epfl_final/logs/l30k1_2b_dense.log.old2 new file mode 100644 index 0000000000000000000000000000000000000000..9ad73f302e7d4c1e5d58a02899e3366cc31a33a1 --- /dev/null +++ b/epfl_final/logs/l30k1_2b_dense.log.old2 @@ -0,0 +1,222 @@ +W0708 06:36:11.151000 2475 site-packages/torch/distributed/run.py:851] +W0708 06:36:11.151000 2475 site-packages/torch/distributed/run.py:851] ***************************************** +W0708 06:36:11.151000 2475 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0708 06:36:11.151000 2475 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063617 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 4, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063617", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 300, + "eval_steps": 500, + "eval_batches": 20, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": true, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "1.2b dense 30k longctx", + "wandb_tags": [ + "qwen3", + "1.2b", + "dense", + "30k", + "longctx", + "b128", + "scaleladder" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: True +Checkpoint activations: True +Checkpoint activations: True +Initializing model from scratch (random weights)... +Checkpoint activations: True +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 1536] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 1536] [1, 30720, 1536] 1,536 True +├─Linear: 1-2 [1, 30720, 1536] [1, 30720, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (Units.GIGABYTES): 1.44 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 98319.73 +Params size (MB): 2871.45 +Estimated Total Size (MB): 101191.43 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 187325 +Optimizer steps per epoch: 37465 +Total optimizer steps: 37465 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_063704-t6a1jz9b +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 1.2b dense 30k longctx +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/t6a1jz9b +Starting training... +Switching attention mode for step 1: dense (dense step interval) +Step 10/37465 | Loss: 12.2534 | LR: 6.67e-06 | 13041.4ms/step | 1.5 samples/s +Step 20/37465 | Loss: 12.1201 | LR: 1.33e-05 | 6349.5ms/step | 3.1 samples/s +Step 30/37465 | Loss: 11.6859 | LR: 2.00e-05 | 6356.2ms/step | 3.1 samples/s +Step 40/37465 | Loss: 10.6683 | LR: 2.67e-05 | 6340.9ms/step | 3.2 samples/s +Step 50/37465 | Loss: 10.0551 | LR: 3.33e-05 | 6325.8ms/step | 3.2 samples/s +W0708 06:43:38.137000 2475 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 2475 got signal: 15 diff --git a/epfl_final/logs/l30k1_2b_dense.log.reclaim_20260708_172842 b/epfl_final/logs/l30k1_2b_dense.log.reclaim_20260708_172842 new file mode 100644 index 0000000000000000000000000000000000000000..b16861853c81283f4186aedc6a00133a6e22234a --- /dev/null +++ b/epfl_final/logs/l30k1_2b_dense.log.reclaim_20260708_172842 @@ -0,0 +1,546 @@ +W0708 06:51:07.977000 139 site-packages/torch/distributed/run.py:851] +W0708 06:51:07.977000 139 site-packages/torch/distributed/run.py:851] ***************************************** +W0708 06:51:07.977000 139 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0708 06:51:07.977000 139 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 4, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 300, + "eval_steps": 500, + "eval_batches": 20, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": true, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "1.2b dense 30k longctx", + "wandb_tags": [ + "qwen3", + "1.2b", + "dense", + "30k", + "longctx", + "b128", + "scaleladder" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: True +Checkpoint activations: True +Checkpoint activations: True +Checkpoint activations: True +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 1536] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 1536] [1, 30720, 1536] 1,536 True +├─Linear: 1-2 [1, 30720, 1536] [1, 30720, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (Units.GIGABYTES): 1.44 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 98319.73 +Params size (MB): 2871.45 +Estimated Total Size (MB): 101191.43 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... + +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) + +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 187325 +Optimizer steps per epoch: 37465 +Total optimizer steps: 37465 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run cd66e13p +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_065216-cd66e13p +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 1.2b dense 30k longctx +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/cd66e13p +Starting training... +Switching attention mode for step 1: dense (dense step interval) +Step 10/37465 | Loss: 12.2534 | LR: 6.67e-06 | 16466.7ms/step | 1.2 samples/s +Step 20/37465 | Loss: 12.1201 | LR: 1.33e-05 | 6357.6ms/step | 3.1 samples/s +Step 30/37465 | Loss: 11.6858 | LR: 2.00e-05 | 6363.5ms/step | 3.1 samples/s +Step 40/37465 | Loss: 10.6683 | LR: 2.67e-05 | 6353.9ms/step | 3.1 samples/s +Step 50/37465 | Loss: 10.0551 | LR: 3.33e-05 | 6330.1ms/step | 3.2 samples/s +Step 60/37465 | Loss: 9.5722 | LR: 4.00e-05 | 6319.8ms/step | 3.2 samples/s +Step 70/37465 | Loss: 9.1242 | LR: 4.67e-05 | 6319.8ms/step | 3.2 samples/s +Step 80/37465 | Loss: 8.6455 | LR: 5.33e-05 | 6326.9ms/step | 3.2 samples/s +Step 90/37465 | Loss: 8.0658 | LR: 6.00e-05 | 6329.3ms/step | 3.2 samples/s +Step 100/37465 | Loss: 7.7199 | LR: 6.67e-05 | 6327.5ms/step | 3.2 samples/s +Step 110/37465 | Loss: 7.3702 | LR: 7.33e-05 | 6331.3ms/step | 3.2 samples/s +Step 120/37465 | Loss: 7.1084 | LR: 8.00e-05 | 6339.4ms/step | 3.2 samples/s +Step 130/37465 | Loss: 6.9197 | LR: 8.67e-05 | 6338.2ms/step | 3.2 samples/s +Step 140/37465 | Loss: 6.8033 | LR: 9.33e-05 | 6347.0ms/step | 3.2 samples/s +Step 150/37465 | Loss: 6.8035 | LR: 1.00e-04 | 6351.2ms/step | 3.1 samples/s +Step 160/37465 | Loss: 6.6536 | LR: 1.07e-04 | 6354.2ms/step | 3.1 samples/s +Step 170/37465 | Loss: 6.3240 | LR: 1.13e-04 | 6361.6ms/step | 3.1 samples/s +Step 180/37465 | Loss: 6.1787 | LR: 1.20e-04 | 6363.3ms/step | 3.1 samples/s +Step 190/37465 | Loss: 6.2520 | LR: 1.27e-04 | 6359.6ms/step | 3.1 samples/s +Step 200/37465 | Loss: 6.0200 | LR: 1.33e-04 | 6364.1ms/step | 3.1 samples/s +Step 210/37465 | Loss: 6.0067 | LR: 1.40e-04 | 6360.8ms/step | 3.1 samples/s +Step 220/37465 | Loss: 5.9344 | LR: 1.47e-04 | 6362.6ms/step | 3.1 samples/s +Step 230/37465 | Loss: 5.9027 | LR: 1.53e-04 | 6366.6ms/step | 3.1 samples/s +Step 240/37465 | Loss: 5.8670 | LR: 1.60e-04 | 6356.6ms/step | 3.1 samples/s +Step 250/37465 | Loss: 5.8162 | LR: 1.67e-04 | 6368.7ms/step | 3.1 samples/s +Step 260/37465 | Loss: 5.6788 | LR: 1.73e-04 | 6369.0ms/step | 3.1 samples/s +Step 270/37465 | Loss: 5.7045 | LR: 1.80e-04 | 6368.3ms/step | 3.1 samples/s +Step 280/37465 | Loss: 5.7118 | LR: 1.87e-04 | 6360.1ms/step | 3.1 samples/s +Step 290/37465 | Loss: 5.5616 | LR: 1.93e-04 | 6365.8ms/step | 3.1 samples/s +Step 300/37465 | Loss: 5.3729 | LR: 2.00e-04 | 6371.3ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-300 +Step 310/37465 | Loss: 5.2669 | LR: 2.07e-04 | 7332.6ms/step | 2.7 samples/s +Step 320/37465 | Loss: 5.3585 | LR: 2.13e-04 | 6363.0ms/step | 3.1 samples/s +Step 330/37465 | Loss: 5.5393 | LR: 2.20e-04 | 6364.3ms/step | 3.1 samples/s +Step 340/37465 | Loss: 5.3772 | LR: 2.27e-04 | 6362.6ms/step | 3.1 samples/s +Step 350/37465 | Loss: 5.2125 | LR: 2.33e-04 | 6367.0ms/step | 3.1 samples/s +Step 360/37465 | Loss: 5.2798 | LR: 2.40e-04 | 6363.3ms/step | 3.1 samples/s +Step 370/37465 | Loss: 5.1172 | LR: 2.47e-04 | 6363.6ms/step | 3.1 samples/s +Step 380/37465 | Loss: 5.3529 | LR: 2.53e-04 | 6367.7ms/step | 3.1 samples/s +Step 390/37465 | Loss: 5.2645 | LR: 2.60e-04 | 6367.3ms/step | 3.1 samples/s +Step 400/37465 | Loss: 5.2519 | LR: 2.67e-04 | 6365.2ms/step | 3.1 samples/s +Step 410/37465 | Loss: 5.0620 | LR: 2.73e-04 | 6366.8ms/step | 3.1 samples/s +Step 420/37465 | Loss: 4.8713 | LR: 2.80e-04 | 6364.6ms/step | 3.1 samples/s +Step 430/37465 | Loss: 4.9729 | LR: 2.87e-04 | 6365.5ms/step | 3.1 samples/s +Step 440/37465 | Loss: 4.8967 | LR: 2.93e-04 | 6369.5ms/step | 3.1 samples/s +Step 450/37465 | Loss: 4.8868 | LR: 3.00e-04 | 6367.4ms/step | 3.1 samples/s +Step 460/37465 | Loss: 4.6993 | LR: 3.07e-04 | 6368.1ms/step | 3.1 samples/s +Step 470/37465 | Loss: 4.7116 | LR: 3.13e-04 | 6370.6ms/step | 3.1 samples/s +Step 480/37465 | Loss: 4.8928 | LR: 3.20e-04 | 6368.0ms/step | 3.1 samples/s +Step 490/37465 | Loss: 4.6241 | LR: 3.27e-04 | 6368.7ms/step | 3.1 samples/s +Step 500/37465 | Loss: 4.8609 | LR: 3.33e-04 | 6369.7ms/step | 3.1 samples/s +Validation at step 500: Loss = 4.6498, Perplexity = 104.56 +Step 510/37465 | Loss: 4.4969 | LR: 3.40e-04 | 10432.4ms/step | 1.9 samples/s +Step 520/37465 | Loss: 4.6458 | LR: 3.47e-04 | 6360.4ms/step | 3.1 samples/s +Step 530/37465 | Loss: 4.8233 | LR: 3.53e-04 | 6367.5ms/step | 3.1 samples/s +Step 540/37465 | Loss: 4.9458 | LR: 3.60e-04 | 6369.9ms/step | 3.1 samples/s +Step 550/37465 | Loss: 4.4279 | LR: 3.67e-04 | 6367.6ms/step | 3.1 samples/s +Step 560/37465 | Loss: 4.4328 | LR: 3.73e-04 | 6368.9ms/step | 3.1 samples/s +Step 570/37465 | Loss: 4.4073 | LR: 3.80e-04 | 6368.9ms/step | 3.1 samples/s +Step 580/37465 | Loss: 4.0842 | LR: 3.87e-04 | 6369.7ms/step | 3.1 samples/s +Step 590/37465 | Loss: 4.4378 | LR: 3.93e-04 | 6371.7ms/step | 3.1 samples/s +Step 600/37465 | Loss: 4.5479 | LR: 4.00e-04 | 6371.6ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-600 +Step 610/37465 | Loss: 4.2887 | LR: 4.07e-04 | 7304.6ms/step | 2.7 samples/s +Step 620/37465 | Loss: 4.1438 | LR: 4.13e-04 | 6380.9ms/step | 3.1 samples/s +Step 630/37465 | Loss: 4.2081 | LR: 4.20e-04 | 6371.7ms/step | 3.1 samples/s +Step 640/37465 | Loss: 4.2991 | LR: 4.27e-04 | 6372.2ms/step | 3.1 samples/s +Step 650/37465 | Loss: 4.2150 | LR: 4.33e-04 | 6368.2ms/step | 3.1 samples/s +Step 660/37465 | Loss: 4.3003 | LR: 4.40e-04 | 6376.6ms/step | 3.1 samples/s +Step 670/37465 | Loss: 4.4093 | LR: 4.47e-04 | 6375.2ms/step | 3.1 samples/s +Step 680/37465 | Loss: 4.1509 | LR: 4.53e-04 | 6370.9ms/step | 3.1 samples/s +Step 690/37465 | Loss: 3.9877 | LR: 4.60e-04 | 6374.5ms/step | 3.1 samples/s +Step 700/37465 | Loss: 4.1206 | LR: 4.67e-04 | 6372.4ms/step | 3.1 samples/s +Step 710/37465 | Loss: 4.0514 | LR: 4.73e-04 | 6374.8ms/step | 3.1 samples/s +Step 720/37465 | Loss: 3.9697 | LR: 4.80e-04 | 6372.0ms/step | 3.1 samples/s +Step 730/37465 | Loss: 3.5586 | LR: 4.87e-04 | 6372.2ms/step | 3.1 samples/s +Step 740/37465 | Loss: 4.0557 | LR: 4.93e-04 | 6369.6ms/step | 3.1 samples/s +Step 750/37465 | Loss: 3.7544 | LR: 5.00e-04 | 6375.6ms/step | 3.1 samples/s +Step 760/37465 | Loss: 3.6661 | LR: 5.00e-04 | 6374.7ms/step | 3.1 samples/s +Step 770/37465 | Loss: 3.6655 | LR: 5.00e-04 | 6379.4ms/step | 3.1 samples/s +Step 780/37465 | Loss: 3.7366 | LR: 5.00e-04 | 6373.3ms/step | 3.1 samples/s +Step 790/37465 | Loss: 3.3627 | LR: 5.00e-04 | 6377.5ms/step | 3.1 samples/s +Step 800/37465 | Loss: 3.7698 | LR: 5.00e-04 | 6377.4ms/step | 3.1 samples/s +Step 810/37465 | Loss: 3.8066 | LR: 5.00e-04 | 6371.7ms/step | 3.1 samples/s +Step 820/37465 | Loss: 3.6056 | LR: 5.00e-04 | 6376.1ms/step | 3.1 samples/s +Step 830/37465 | Loss: 3.6018 | LR: 5.00e-04 | 6376.6ms/step | 3.1 samples/s +Step 840/37465 | Loss: 3.7742 | LR: 5.00e-04 | 6375.5ms/step | 3.1 samples/s +Step 850/37465 | Loss: 3.4180 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s +Step 860/37465 | Loss: 3.6353 | LR: 5.00e-04 | 6374.2ms/step | 3.1 samples/s +Step 870/37465 | Loss: 3.5633 | LR: 5.00e-04 | 6373.2ms/step | 3.1 samples/s +Step 880/37465 | Loss: 3.4691 | LR: 5.00e-04 | 6374.4ms/step | 3.1 samples/s +Step 890/37465 | Loss: 3.5110 | LR: 5.00e-04 | 6370.8ms/step | 3.1 samples/s +Step 900/37465 | Loss: 3.1987 | LR: 5.00e-04 | 6375.8ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-900 +Step 910/37465 | Loss: 3.2169 | LR: 5.00e-04 | 7315.5ms/step | 2.7 samples/s +Step 920/37465 | Loss: 3.3605 | LR: 5.00e-04 | 6378.5ms/step | 3.1 samples/s +Step 930/37465 | Loss: 3.2232 | LR: 5.00e-04 | 6380.6ms/step | 3.1 samples/s +Step 940/37465 | Loss: 3.4859 | LR: 5.00e-04 | 6380.2ms/step | 3.1 samples/s +Step 950/37465 | Loss: 3.7345 | LR: 5.00e-04 | 6381.3ms/step | 3.1 samples/s +Step 960/37465 | Loss: 3.0550 | LR: 5.00e-04 | 6386.2ms/step | 3.1 samples/s +Step 970/37465 | Loss: 3.4433 | LR: 5.00e-04 | 6389.7ms/step | 3.1 samples/s +Step 980/37465 | Loss: 3.2395 | LR: 5.00e-04 | 6383.5ms/step | 3.1 samples/s +Step 990/37465 | Loss: 3.2277 | LR: 5.00e-04 | 6383.7ms/step | 3.1 samples/s +Step 1000/37465 | Loss: 3.0568 | LR: 5.00e-04 | 6387.2ms/step | 3.1 samples/s +Validation at step 1000: Loss = 3.2705, Perplexity = 26.32 +Step 1010/37465 | Loss: 3.1221 | LR: 5.00e-04 | 7136.6ms/step | 2.8 samples/s +Step 1020/37465 | Loss: 3.3939 | LR: 5.00e-04 | 6387.9ms/step | 3.1 samples/s +Step 1030/37465 | Loss: 2.9741 | LR: 5.00e-04 | 6392.5ms/step | 3.1 samples/s +Step 1040/37465 | Loss: 3.3890 | LR: 5.00e-04 | 6384.8ms/step | 3.1 samples/s +Step 1050/37465 | Loss: 3.1908 | LR: 5.00e-04 | 6385.4ms/step | 3.1 samples/s +Step 1060/37465 | Loss: 3.2037 | LR: 5.00e-04 | 6391.9ms/step | 3.1 samples/s +Step 1070/37465 | Loss: 3.1875 | LR: 5.00e-04 | 6389.5ms/step | 3.1 samples/s +Step 1080/37465 | Loss: 3.1991 | LR: 5.00e-04 | 6397.2ms/step | 3.1 samples/s +Step 1090/37465 | Loss: 3.1953 | LR: 5.00e-04 | 6388.9ms/step | 3.1 samples/s +Step 1100/37465 | Loss: 3.0628 | LR: 5.00e-04 | 6385.3ms/step | 3.1 samples/s +Step 1110/37465 | Loss: 2.9822 | LR: 5.00e-04 | 6388.1ms/step | 3.1 samples/s +Step 1120/37465 | Loss: 2.9275 | LR: 5.00e-04 | 6390.9ms/step | 3.1 samples/s +Step 1130/37465 | Loss: 3.1709 | LR: 5.00e-04 | 6384.2ms/step | 3.1 samples/s +Step 1140/37465 | Loss: 3.1406 | LR: 5.00e-04 | 6396.0ms/step | 3.1 samples/s +Step 1150/37465 | Loss: 2.9701 | LR: 5.00e-04 | 6386.2ms/step | 3.1 samples/s +Step 1160/37465 | Loss: 3.2755 | LR: 5.00e-04 | 6389.9ms/step | 3.1 samples/s +Step 1170/37465 | Loss: 2.8357 | LR: 5.00e-04 | 6391.5ms/step | 3.1 samples/s +Step 1180/37465 | Loss: 3.2176 | LR: 5.00e-04 | 6391.0ms/step | 3.1 samples/s +Step 1190/37465 | Loss: 3.0387 | LR: 5.00e-04 | 6388.0ms/step | 3.1 samples/s +Step 1200/37465 | Loss: 2.7643 | LR: 5.00e-04 | 6386.4ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1200 +Step 1210/37465 | Loss: 3.1515 | LR: 5.00e-04 | 7315.0ms/step | 2.7 samples/s +Step 1220/37465 | Loss: 3.2124 | LR: 5.00e-04 | 6384.4ms/step | 3.1 samples/s +Step 1230/37465 | Loss: 2.8344 | LR: 5.00e-04 | 6379.6ms/step | 3.1 samples/s +Step 1240/37465 | Loss: 3.0946 | LR: 5.00e-04 | 6375.6ms/step | 3.1 samples/s +Step 1250/37465 | Loss: 2.9099 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s +Step 1260/37465 | Loss: 2.9297 | LR: 5.00e-04 | 6374.3ms/step | 3.1 samples/s +Step 1270/37465 | Loss: 3.0279 | LR: 5.00e-04 | 6376.6ms/step | 3.1 samples/s +Step 1280/37465 | Loss: 2.8854 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s +Step 1290/37465 | Loss: 2.9985 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s +Step 1300/37465 | Loss: 3.3021 | LR: 5.00e-04 | 6378.0ms/step | 3.1 samples/s +Step 1310/37465 | Loss: 3.0772 | LR: 5.00e-04 | 6379.3ms/step | 3.1 samples/s +Step 1320/37465 | Loss: 2.9217 | LR: 5.00e-04 | 6377.9ms/step | 3.1 samples/s +Step 1330/37465 | Loss: 3.1318 | LR: 5.00e-04 | 6378.1ms/step | 3.1 samples/s +Step 1340/37465 | Loss: 3.0401 | LR: 5.00e-04 | 6380.0ms/step | 3.1 samples/s +Step 1350/37465 | Loss: 2.9640 | LR: 5.00e-04 | 6379.0ms/step | 3.1 samples/s +Step 1360/37465 | Loss: 2.8963 | LR: 5.00e-04 | 6376.5ms/step | 3.1 samples/s +Step 1370/37465 | Loss: 3.2083 | LR: 5.00e-04 | 6380.2ms/step | 3.1 samples/s +Step 1380/37465 | Loss: 2.9132 | LR: 5.00e-04 | 6379.1ms/step | 3.1 samples/s +Step 1390/37465 | Loss: 2.8943 | LR: 5.00e-04 | 6374.9ms/step | 3.1 samples/s +Step 1400/37465 | Loss: 2.9270 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s +Step 1410/37465 | Loss: 3.1229 | LR: 5.00e-04 | 6381.8ms/step | 3.1 samples/s +Step 1420/37465 | Loss: 2.7350 | LR: 5.00e-04 | 6381.4ms/step | 3.1 samples/s +Step 1430/37465 | Loss: 3.2892 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s +Step 1440/37465 | Loss: 2.9455 | LR: 5.00e-04 | 6382.3ms/step | 3.1 samples/s +Step 1450/37465 | Loss: 3.0110 | LR: 5.00e-04 | 6375.8ms/step | 3.1 samples/s +Step 1460/37465 | Loss: 3.0622 | LR: 5.00e-04 | 6377.9ms/step | 3.1 samples/s +Step 1470/37465 | Loss: 2.9322 | LR: 5.00e-04 | 6379.8ms/step | 3.1 samples/s +Step 1480/37465 | Loss: 2.7953 | LR: 5.00e-04 | 6377.0ms/step | 3.1 samples/s +Step 1490/37465 | Loss: 2.9015 | LR: 4.99e-04 | 6379.9ms/step | 3.1 samples/s +Step 1500/37465 | Loss: 2.8845 | LR: 4.99e-04 | 6381.4ms/step | 3.1 samples/s +Validation at step 1500: Loss = 2.9509, Perplexity = 19.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1500 +Step 1510/37465 | Loss: 2.9572 | LR: 4.99e-04 | 8029.5ms/step | 2.5 samples/s +Step 1520/37465 | Loss: 2.8701 | LR: 4.99e-04 | 6387.5ms/step | 3.1 samples/s +Step 1530/37465 | Loss: 2.8017 | LR: 4.99e-04 | 6387.6ms/step | 3.1 samples/s +Step 1540/37465 | Loss: 2.7911 | LR: 4.99e-04 | 6386.7ms/step | 3.1 samples/s +Step 1550/37465 | Loss: 3.0827 | LR: 4.99e-04 | 6385.9ms/step | 3.1 samples/s +Step 1560/37465 | Loss: 2.9479 | LR: 4.99e-04 | 6387.4ms/step | 3.1 samples/s +Step 1570/37465 | Loss: 2.7598 | LR: 4.99e-04 | 6390.2ms/step | 3.1 samples/s +Step 1580/37465 | Loss: 3.1258 | LR: 4.99e-04 | 6387.3ms/step | 3.1 samples/s +Step 1590/37465 | Loss: 2.9865 | LR: 4.99e-04 | 6389.2ms/step | 3.1 samples/s +Step 1600/37465 | Loss: 3.2862 | LR: 4.99e-04 | 6383.2ms/step | 3.1 samples/s +Step 1610/37465 | Loss: 2.7886 | LR: 4.99e-04 | 6386.6ms/step | 3.1 samples/s +Step 1620/37465 | Loss: 3.0593 | LR: 4.99e-04 | 6384.5ms/step | 3.1 samples/s +Step 1630/37465 | Loss: 2.9216 | LR: 4.99e-04 | 6390.2ms/step | 3.1 samples/s +Step 1640/37465 | Loss: 2.8619 | LR: 4.99e-04 | 6391.2ms/step | 3.1 samples/s +Step 1650/37465 | Loss: 2.6076 | LR: 4.99e-04 | 6387.9ms/step | 3.1 samples/s +Step 1660/37465 | Loss: 2.7470 | LR: 4.99e-04 | 6386.2ms/step | 3.1 samples/s +Step 1670/37465 | Loss: 2.7488 | LR: 4.99e-04 | 6394.0ms/step | 3.1 samples/s +Step 1680/37465 | Loss: 3.0364 | LR: 4.99e-04 | 6389.6ms/step | 3.1 samples/s +Step 1690/37465 | Loss: 2.9046 | LR: 4.99e-04 | 6398.3ms/step | 3.1 samples/s +Step 1700/37465 | Loss: 2.9512 | LR: 4.99e-04 | 6388.5ms/step | 3.1 samples/s +Step 1710/37465 | Loss: 2.9809 | LR: 4.99e-04 | 6382.4ms/step | 3.1 samples/s +Step 1720/37465 | Loss: 3.0311 | LR: 4.99e-04 | 6392.7ms/step | 3.1 samples/s +Step 1730/37465 | Loss: 2.7612 | LR: 4.99e-04 | 6390.9ms/step | 3.1 samples/s +Step 1740/37465 | Loss: 2.9211 | LR: 4.99e-04 | 6379.4ms/step | 3.1 samples/s +Step 1750/37465 | Loss: 2.9566 | LR: 4.99e-04 | 6387.7ms/step | 3.1 samples/s +Step 1760/37465 | Loss: 2.8662 | LR: 4.99e-04 | 6387.3ms/step | 3.1 samples/s +Step 1770/37465 | Loss: 2.8642 | LR: 4.99e-04 | 6385.7ms/step | 3.1 samples/s +Step 1780/37465 | Loss: 2.7108 | LR: 4.99e-04 | 6390.8ms/step | 3.1 samples/s +Step 1790/37465 | Loss: 2.9409 | LR: 4.99e-04 | 6388.7ms/step | 3.1 samples/s +Step 1800/37465 | Loss: 2.6069 | LR: 4.99e-04 | 6392.8ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1800 +Step 1810/37465 | Loss: 2.9186 | LR: 4.99e-04 | 7290.9ms/step | 2.7 samples/s +Step 1820/37465 | Loss: 2.9307 | LR: 4.99e-04 | 6389.4ms/step | 3.1 samples/s +Step 1830/37465 | Loss: 2.8242 | LR: 4.99e-04 | 6394.3ms/step | 3.1 samples/s +Step 1840/37465 | Loss: 2.9409 | LR: 4.99e-04 | 6387.8ms/step | 3.1 samples/s +Step 1850/37465 | Loss: 2.7138 | LR: 4.99e-04 | 6391.3ms/step | 3.1 samples/s +Step 1860/37465 | Loss: 2.7519 | LR: 4.99e-04 | 6391.2ms/step | 3.1 samples/s +Step 1870/37465 | Loss: 3.1458 | LR: 4.99e-04 | 6386.0ms/step | 3.1 samples/s +Step 1880/37465 | Loss: 2.8918 | LR: 4.99e-04 | 6383.1ms/step | 3.1 samples/s +Step 1890/37465 | Loss: 2.8751 | LR: 4.99e-04 | 6379.4ms/step | 3.1 samples/s +Step 1900/37465 | Loss: 2.7494 | LR: 4.99e-04 | 6380.6ms/step | 3.1 samples/s +Step 1910/37465 | Loss: 2.7345 | LR: 4.99e-04 | 6379.9ms/step | 3.1 samples/s +Step 1920/37465 | Loss: 2.9475 | LR: 4.99e-04 | 6382.4ms/step | 3.1 samples/s +Step 1930/37465 | Loss: 2.8216 | LR: 4.99e-04 | 6382.3ms/step | 3.1 samples/s +Step 1940/37465 | Loss: 2.8334 | LR: 4.99e-04 | 6382.6ms/step | 3.1 samples/s +Step 1950/37465 | Loss: 2.7589 | LR: 4.99e-04 | 6384.2ms/step | 3.1 samples/s +Step 1960/37465 | Loss: 2.7681 | LR: 4.99e-04 | 6381.6ms/step | 3.1 samples/s +Step 1970/37465 | Loss: 2.9729 | LR: 4.99e-04 | 6382.0ms/step | 3.1 samples/s +Step 1980/37465 | Loss: 2.7409 | LR: 4.99e-04 | 6385.0ms/step | 3.1 samples/s +Step 1990/37465 | Loss: 2.6951 | LR: 4.99e-04 | 6389.5ms/step | 3.1 samples/s +Step 2000/37465 | Loss: 2.7914 | LR: 4.99e-04 | 6383.1ms/step | 3.1 samples/s +Validation at step 2000: Loss = 2.7777, Perplexity = 16.08 +Step 2010/37465 | Loss: 2.7050 | LR: 4.99e-04 | 7128.8ms/step | 2.8 samples/s +Step 2020/37465 | Loss: 2.7686 | LR: 4.99e-04 | 6384.7ms/step | 3.1 samples/s +Step 2030/37465 | Loss: 2.6295 | LR: 4.99e-04 | 6385.0ms/step | 3.1 samples/s +Step 2040/37465 | Loss: 2.6745 | LR: 4.98e-04 | 6379.5ms/step | 3.1 samples/s +Step 2050/37465 | Loss: 2.7631 | LR: 4.98e-04 | 6382.7ms/step | 3.1 samples/s +Step 2060/37465 | Loss: 2.6218 | LR: 4.98e-04 | 6384.9ms/step | 3.1 samples/s +Step 2070/37465 | Loss: 2.6614 | LR: 4.98e-04 | 6386.5ms/step | 3.1 samples/s +Step 2080/37465 | Loss: 2.8349 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s +Step 2090/37465 | Loss: 2.8247 | LR: 4.98e-04 | 6382.2ms/step | 3.1 samples/s +Step 2100/37465 | Loss: 2.9305 | LR: 4.98e-04 | 6379.0ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2100 +Step 2110/37465 | Loss: 2.8710 | LR: 4.98e-04 | 7287.2ms/step | 2.7 samples/s +Step 2120/37465 | Loss: 2.7928 | LR: 4.98e-04 | 6384.3ms/step | 3.1 samples/s +Step 2130/37465 | Loss: 2.5135 | LR: 4.98e-04 | 6387.6ms/step | 3.1 samples/s +Step 2140/37465 | Loss: 2.8393 | LR: 4.98e-04 | 6384.1ms/step | 3.1 samples/s +Step 2150/37465 | Loss: 2.7712 | LR: 4.98e-04 | 6387.7ms/step | 3.1 samples/s +Step 2160/37465 | Loss: 2.6872 | LR: 4.98e-04 | 6382.7ms/step | 3.1 samples/s +Step 2170/37465 | Loss: 2.7883 | LR: 4.98e-04 | 6379.3ms/step | 3.1 samples/s +Step 2180/37465 | Loss: 2.8064 | LR: 4.98e-04 | 6374.1ms/step | 3.1 samples/s +Step 2190/37465 | Loss: 2.7624 | LR: 4.98e-04 | 6382.0ms/step | 3.1 samples/s +Step 2200/37465 | Loss: 2.6835 | LR: 4.98e-04 | 6383.3ms/step | 3.1 samples/s +Step 2210/37465 | Loss: 2.6176 | LR: 4.98e-04 | 6382.6ms/step | 3.1 samples/s +Step 2220/37465 | Loss: 2.8132 | LR: 4.98e-04 | 6382.2ms/step | 3.1 samples/s +Step 2230/37465 | Loss: 2.8084 | LR: 4.98e-04 | 6378.5ms/step | 3.1 samples/s +Step 2240/37465 | Loss: 2.8041 | LR: 4.98e-04 | 6377.2ms/step | 3.1 samples/s +Step 2250/37465 | Loss: 2.6921 | LR: 4.98e-04 | 6380.4ms/step | 3.1 samples/s +Step 2260/37465 | Loss: 2.8456 | LR: 4.98e-04 | 6376.4ms/step | 3.1 samples/s +Step 2270/37465 | Loss: 2.7451 | LR: 4.98e-04 | 6379.1ms/step | 3.1 samples/s +Step 2280/37465 | Loss: 2.5784 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s +Step 2290/37465 | Loss: 2.7031 | LR: 4.98e-04 | 6381.7ms/step | 3.1 samples/s +Step 2300/37465 | Loss: 2.8041 | LR: 4.98e-04 | 6377.0ms/step | 3.1 samples/s +Step 2310/37465 | Loss: 2.8593 | LR: 4.98e-04 | 6379.1ms/step | 3.1 samples/s +Step 2320/37465 | Loss: 2.7207 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s +Step 2330/37465 | Loss: 2.9899 | LR: 4.98e-04 | 6372.6ms/step | 3.1 samples/s +Step 2340/37465 | Loss: 2.5633 | LR: 4.98e-04 | 6377.4ms/step | 3.1 samples/s +Step 2350/37465 | Loss: 2.9726 | LR: 4.98e-04 | 6379.6ms/step | 3.1 samples/s +Step 2360/37465 | Loss: 2.5830 | LR: 4.98e-04 | 6378.0ms/step | 3.1 samples/s +Step 2370/37465 | Loss: 2.8398 | LR: 4.98e-04 | 6378.2ms/step | 3.1 samples/s +Step 2380/37465 | Loss: 2.6590 | LR: 4.98e-04 | 6377.3ms/step | 3.1 samples/s +Step 2390/37465 | Loss: 2.5337 | LR: 4.98e-04 | 6380.3ms/step | 3.1 samples/s +Step 2400/37465 | Loss: 2.5193 | LR: 4.98e-04 | 6379.5ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2400 +Step 2410/37465 | Loss: 2.6855 | LR: 4.97e-04 | 7312.0ms/step | 2.7 samples/s +Step 2420/37465 | Loss: 2.8776 | LR: 4.97e-04 | 6381.3ms/step | 3.1 samples/s +Step 2430/37465 | Loss: 2.8649 | LR: 4.97e-04 | 6384.6ms/step | 3.1 samples/s +Step 2440/37465 | Loss: 2.8029 | LR: 4.97e-04 | 6385.3ms/step | 3.1 samples/s +Step 2450/37465 | Loss: 2.7996 | LR: 4.97e-04 | 6390.0ms/step | 3.1 samples/s +Step 2460/37465 | Loss: 2.7896 | LR: 4.97e-04 | 6391.3ms/step | 3.1 samples/s +Step 2470/37465 | Loss: 2.8103 | LR: 4.97e-04 | 6381.5ms/step | 3.1 samples/s +Step 2480/37465 | Loss: 2.6114 | LR: 4.97e-04 | 6383.0ms/step | 3.1 samples/s +Step 2490/37465 | Loss: 2.6087 | LR: 4.97e-04 | 6371.4ms/step | 3.1 samples/s +Step 2500/37465 | Loss: 2.7416 | LR: 4.97e-04 | 6382.6ms/step | 3.1 samples/s +Validation at step 2500: Loss = 2.6779, Perplexity = 14.55 +Step 2510/37465 | Loss: 2.8022 | LR: 4.97e-04 | 7132.4ms/step | 2.8 samples/s +Step 2520/37465 | Loss: 2.4969 | LR: 4.97e-04 | 6380.0ms/step | 3.1 samples/s +Step 2530/37465 | Loss: 2.6487 | LR: 4.97e-04 | 6378.3ms/step | 3.1 samples/s +Step 2540/37465 | Loss: 2.9364 | LR: 4.97e-04 | 6381.9ms/step | 3.1 samples/s +Step 2550/37465 | Loss: 2.4333 | LR: 4.97e-04 | 6374.7ms/step | 3.1 samples/s +Step 2560/37465 | Loss: 2.7193 | LR: 4.97e-04 | 6375.6ms/step | 3.1 samples/s +Step 2570/37465 | Loss: 2.6816 | LR: 4.97e-04 | 6378.3ms/step | 3.1 samples/s +Step 2580/37465 | Loss: 2.4674 | LR: 4.97e-04 | 6378.2ms/step | 3.1 samples/s +Step 2590/37465 | Loss: 2.7053 | LR: 4.97e-04 | 6378.6ms/step | 3.1 samples/s +Step 2600/37465 | Loss: 2.8359 | LR: 4.97e-04 | 6378.8ms/step | 3.1 samples/s +Step 2610/37465 | Loss: 2.5548 | LR: 4.97e-04 | 6379.4ms/step | 3.1 samples/s +Step 2620/37465 | Loss: 2.8504 | LR: 4.97e-04 | 6382.6ms/step | 3.1 samples/s +Step 2630/37465 | Loss: 2.6607 | LR: 4.97e-04 | 6385.9ms/step | 3.1 samples/s +Step 2640/37465 | Loss: 2.5939 | LR: 4.97e-04 | 6377.5ms/step | 3.1 samples/s +Step 2650/37465 | Loss: 2.5936 | LR: 4.97e-04 | 6378.4ms/step | 3.1 samples/s +Step 2660/37465 | Loss: 2.4835 | LR: 4.97e-04 | 6379.9ms/step | 3.1 samples/s +Step 2670/37465 | Loss: 2.5658 | LR: 4.97e-04 | 6374.7ms/step | 3.1 samples/s +Step 2680/37465 | Loss: 2.8609 | LR: 4.97e-04 | 6379.0ms/step | 3.1 samples/s +Step 2690/37465 | Loss: 2.5883 | LR: 4.97e-04 | 6382.0ms/step | 3.1 samples/s +Step 2700/37465 | Loss: 2.7509 | LR: 4.97e-04 | 6381.7ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2700 +Step 2710/37465 | Loss: 2.5879 | LR: 4.96e-04 | 7293.1ms/step | 2.7 samples/s +Step 2720/37465 | Loss: 2.5680 | LR: 4.96e-04 | 6377.8ms/step | 3.1 samples/s +Step 2730/37465 | Loss: 2.6055 | LR: 4.96e-04 | 6377.1ms/step | 3.1 samples/s +Step 2740/37465 | Loss: 2.6871 | LR: 4.96e-04 | 6382.4ms/step | 3.1 samples/s +Step 2750/37465 | Loss: 2.5974 | LR: 4.96e-04 | 6387.7ms/step | 3.1 samples/s +Step 2760/37465 | Loss: 2.8237 | LR: 4.96e-04 | 6378.7ms/step | 3.1 samples/s +Step 2770/37465 | Loss: 2.6716 | LR: 4.96e-04 | 6382.1ms/step | 3.1 samples/s +Step 2780/37465 | Loss: 2.7498 | LR: 4.96e-04 | 6380.9ms/step | 3.1 samples/s +Step 2790/37465 | Loss: 2.4863 | LR: 4.96e-04 | 6380.2ms/step | 3.1 samples/s +Step 2800/37465 | Loss: 2.5453 | LR: 4.96e-04 | 6383.2ms/step | 3.1 samples/s +Step 2810/37465 | Loss: 2.5213 | LR: 4.96e-04 | 6386.4ms/step | 3.1 samples/s +Step 2820/37465 | Loss: 2.3568 | LR: 4.96e-04 | 6391.6ms/step | 3.1 samples/s +Step 2830/37465 | Loss: 2.4480 | LR: 4.96e-04 | 6387.9ms/step | 3.1 samples/s +Step 2840/37465 | Loss: 2.8941 | LR: 4.96e-04 | 6383.9ms/step | 3.1 samples/s +Step 2850/37465 | Loss: 2.5534 | LR: 4.96e-04 | 6384.4ms/step | 3.1 samples/s +Step 2860/37465 | Loss: 2.5181 | LR: 4.96e-04 | 6387.4ms/step | 3.1 samples/s +Step 2870/37465 | Loss: 2.7222 | LR: 4.96e-04 | 6384.2ms/step | 3.1 samples/s +Step 2880/37465 | Loss: 2.6761 | LR: 4.96e-04 | 6383.9ms/step | 3.1 samples/s +Step 2890/37465 | Loss: 2.6068 | LR: 4.96e-04 | 6388.8ms/step | 3.1 samples/s +Step 2900/37465 | Loss: 2.5596 | LR: 4.96e-04 | 6387.7ms/step | 3.1 samples/s +Step 2910/37465 | Loss: 2.6637 | LR: 4.96e-04 | 6385.2ms/step | 3.1 samples/s +Step 2920/37465 | Loss: 2.4493 | LR: 4.96e-04 | 6381.5ms/step | 3.1 samples/s +Step 2930/37465 | Loss: 2.5966 | LR: 4.96e-04 | 6384.4ms/step | 3.1 samples/s +Step 2940/37465 | Loss: 2.5092 | LR: 4.96e-04 | 6381.1ms/step | 3.1 samples/s +Step 2950/37465 | Loss: 2.5356 | LR: 4.96e-04 | 6383.8ms/step | 3.1 samples/s +Step 2960/37465 | Loss: 2.5739 | LR: 4.96e-04 | 6386.1ms/step | 3.1 samples/s +Step 2970/37465 | Loss: 2.6963 | LR: 4.96e-04 | 6387.8ms/step | 3.1 samples/s +Step 2980/37465 | Loss: 2.3348 | LR: 4.95e-04 | 6385.2ms/step | 3.1 samples/s +Step 2990/37465 | Loss: 2.7376 | LR: 4.95e-04 | 6388.7ms/step | 3.1 samples/s +Step 3000/37465 | Loss: 2.6397 | LR: 4.95e-04 | 6387.3ms/step | 3.1 samples/s +Validation at step 3000: Loss = 2.6081, Perplexity = 13.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3000 +Step 3010/37465 | Loss: 2.6149 | LR: 4.95e-04 | 8062.9ms/step | 2.5 samples/s +Step 3020/37465 | Loss: 2.6029 | LR: 4.95e-04 | 6386.1ms/step | 3.1 samples/s +Step 3030/37465 | Loss: 2.8015 | LR: 4.95e-04 | 6384.6ms/step | 3.1 samples/s +Step 3040/37465 | Loss: 2.7272 | LR: 4.95e-04 | 6380.3ms/step | 3.1 samples/s +Step 3050/37465 | Loss: 2.6205 | LR: 4.95e-04 | 6380.2ms/step | 3.1 samples/s +Step 3060/37465 | Loss: 2.6117 | LR: 4.95e-04 | 6386.5ms/step | 3.1 samples/s +Step 3070/37465 | Loss: 2.4390 | LR: 4.95e-04 | 6384.2ms/step | 3.1 samples/s +Step 3080/37465 | Loss: 2.3941 | LR: 4.95e-04 | 6385.5ms/step | 3.1 samples/s +Step 3090/37465 | Loss: 2.5643 | LR: 4.95e-04 | 6385.4ms/step | 3.1 samples/s +Step 3100/37465 | Loss: 2.6672 | LR: 4.95e-04 | 6382.4ms/step | 3.1 samples/s +Step 3110/37465 | Loss: 2.6058 | LR: 4.95e-04 | 6380.2ms/step | 3.1 samples/s +Step 3120/37465 | Loss: 2.4922 | LR: 4.95e-04 | 6381.2ms/step | 3.1 samples/s +Step 3130/37465 | Loss: 2.5960 | LR: 4.95e-04 | 6383.0ms/step | 3.1 samples/s +Step 3140/37465 | Loss: 2.4059 | LR: 4.95e-04 | 6378.9ms/step | 3.1 samples/s +Step 3150/37465 | Loss: 2.3778 | LR: 4.95e-04 | 6382.1ms/step | 3.1 samples/s +Step 3160/37465 | Loss: 2.5527 | LR: 4.95e-04 | 6374.2ms/step | 3.1 samples/s +Step 3170/37465 | Loss: 2.8536 | LR: 4.95e-04 | 6381.6ms/step | 3.1 samples/s +Step 3180/37465 | Loss: 2.7932 | LR: 4.95e-04 | 6378.5ms/step | 3.1 samples/s +Step 3190/37465 | Loss: 2.2729 | LR: 4.95e-04 | 6379.2ms/step | 3.1 samples/s +Step 3200/37465 | Loss: 2.4523 | LR: 4.95e-04 | 6376.7ms/step | 3.1 samples/s +Step 3210/37465 | Loss: 2.3870 | LR: 4.94e-04 | 6389.6ms/step | 3.1 samples/s +Step 3220/37465 | Loss: 2.6399 | LR: 4.94e-04 | 6385.6ms/step | 3.1 samples/s +Step 3230/37465 | Loss: 2.6069 | LR: 4.94e-04 | 6382.8ms/step | 3.1 samples/s +Step 3240/37465 | Loss: 2.4823 | LR: 4.94e-04 | 6378.3ms/step | 3.1 samples/s +Step 3250/37465 | Loss: 2.4919 | LR: 4.94e-04 | 6378.8ms/step | 3.1 samples/s +Step 3260/37465 | Loss: 2.6710 | LR: 4.94e-04 | 6378.5ms/step | 3.1 samples/s +Step 3270/37465 | Loss: 2.6290 | LR: 4.94e-04 | 6377.1ms/step | 3.1 samples/s +Step 3280/37465 | Loss: 2.5979 | LR: 4.94e-04 | 6382.3ms/step | 3.1 samples/s +Step 3290/37465 | Loss: 2.4953 | LR: 4.94e-04 | 6383.3ms/step | 3.1 samples/s +Step 3300/37465 | Loss: 2.4728 | LR: 4.94e-04 | 6378.9ms/step | 3.1 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300 +Step 3310/37465 | Loss: 2.6605 | LR: 4.94e-04 | 7302.7ms/step | 2.7 samples/s +Step 3320/37465 | Loss: 2.3535 | LR: 4.94e-04 | 6384.4ms/step | 3.1 samples/s +Step 3330/37465 | Loss: 2.6802 | LR: 4.94e-04 | 6381.3ms/step | 3.1 samples/s +Step 3340/37465 | Loss: 2.6431 | LR: 4.94e-04 | 6383.2ms/step | 3.1 samples/s +Step 3350/37465 | Loss: 2.9169 | LR: 4.94e-04 | 6392.3ms/step | 3.1 samples/s +Step 3360/37465 | Loss: 2.5130 | LR: 4.94e-04 | 6380.6ms/step | 3.1 samples/s +Step 3370/37465 | Loss: 2.5432 | LR: 4.94e-04 | 6380.1ms/step | 3.1 samples/s +Step 3380/37465 | Loss: 2.5796 | LR: 4.94e-04 | 6382.4ms/step | 3.1 samples/s +Step 3390/37465 | Loss: 2.4650 | LR: 4.94e-04 | 6383.2ms/step | 3.1 samples/s +Step 3400/37465 | Loss: 2.3815 | LR: 4.94e-04 | 6386.5ms/step | 3.1 samples/s +Step 3410/37465 | Loss: 2.4313 | LR: 4.94e-04 | 6382.9ms/step | 3.1 samples/s +Step 3420/37465 | Loss: 2.5677 | LR: 4.94e-04 | 6383.0ms/step | 3.1 samples/s +Step 3430/37465 | Loss: 2.7321 | LR: 4.93e-04 | 6379.4ms/step | 3.1 samples/s +Step 3440/37465 | Loss: 2.6157 | LR: 4.93e-04 | 6381.4ms/step | 3.1 samples/s diff --git a/epfl_final/logs/l30k1_2b_dense.log.rescheduled1 b/epfl_final/logs/l30k1_2b_dense.log.rescheduled1 new file mode 100644 index 0000000000000000000000000000000000000000..870d064f6e8300e96bbbe98ac5913436bebaaf86 --- /dev/null +++ b/epfl_final/logs/l30k1_2b_dense.log.rescheduled1 @@ -0,0 +1,219 @@ +W0708 05:43:00.370000 206 site-packages/torch/distributed/run.py:851] +W0708 05:43:00.370000 206 site-packages/torch/distributed/run.py:851] ***************************************** +W0708 05:43:00.370000 206 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0708 05:43:00.370000 206 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_054311 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 4, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_054311", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 500, + "eval_batches": 20, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": true, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "1.2b dense 30k longctx", + "wandb_tags": [ + "qwen3", + "1.2b", + "dense", + "30k", + "longctx", + "b128", + "scaleladder" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: TrueCheckpoint activations: True + +Checkpoint activations: True +Checkpoint activations: True +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 1536] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 1536] [1, 30720, 1536] 1,536 True +├─Linear: 1-2 [1, 30720, 1536] [1, 30720, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (Units.GIGABYTES): 1.44 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 98319.73 +Params size (MB): 2871.45 +Estimated Total Size (MB): 101191.43 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) + +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 187325 +Optimizer steps per epoch: 37465 +Total optimizer steps: 37465 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_054405-43sa4ouk +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 1.2b dense 30k longctx +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/43sa4ouk +Starting training... +Switching attention mode for step 1: dense (dense step interval) +Step 10/37465 | Loss: 12.2534 | LR: 6.67e-06 | 16167.7ms/step | 1.2 samples/s +Step 20/37465 | Loss: 12.1201 | LR: 1.33e-05 | 6304.5ms/step | 3.2 samples/s +Step 30/37465 | Loss: 11.6858 | LR: 2.00e-05 | 6309.1ms/step | 3.2 samples/s +Step 40/37465 | Loss: 10.6684 | LR: 2.67e-05 | 6298.4ms/step | 3.2 samples/s +Step 50/37465 | Loss: 10.0551 | LR: 3.33e-05 | 6286.7ms/step | 3.2 samples/s +Step 60/37465 | Loss: 9.5722 | LR: 4.00e-05 | 6268.7ms/step | 3.2 samples/s +Step 70/37465 | Loss: 9.1242 | LR: 4.67e-05 | 6271.1ms/step | 3.2 samples/s +Step 80/37465 | Loss: 8.6456 | LR: 5.33e-05 | 6278.9ms/step | 3.2 samples/s +Step 90/37465 | Loss: 8.0658 | LR: 6.00e-05 | 6281.1ms/step | 3.2 samples/s +Step 100/37465 | Loss: 7.7196 | LR: 6.67e-05 | 6269.8ms/step | 3.2 samples/s +Step 110/37465 | Loss: 7.3699 | LR: 7.33e-05 | 6276.4ms/step | 3.2 samples/s +Step 120/37465 | Loss: 7.1081 | LR: 8.00e-05 | 6290.5ms/step | 3.2 samples/s +Step 130/37465 | Loss: 6.9196 | LR: 8.67e-05 | 6302.0ms/step | 3.2 samples/s +Step 140/37465 | Loss: 6.8029 | LR: 9.33e-05 | 6285.6ms/step | 3.2 samples/s +Step 150/37465 | Loss: 6.8077 | LR: 1.00e-04 | 6286.5ms/step | 3.2 samples/s +Step 160/37465 | Loss: 6.6545 | LR: 1.07e-04 | 6303.8ms/step | 3.2 samples/s +Step 170/37465 | Loss: 6.3287 | LR: 1.13e-04 | 6301.6ms/step | 3.2 samples/s +Step 180/37465 | Loss: 6.1895 | LR: 1.20e-04 | 6295.4ms/step | 3.2 samples/s +Step 190/37465 | Loss: 6.2694 | LR: 1.27e-04 | 6300.7ms/step | 3.2 samples/s +Step 200/37465 | Loss: 6.0386 | LR: 1.33e-04 | 6304.3ms/step | 3.2 samples/s +Step 210/37465 | Loss: 6.0358 | LR: 1.40e-04 | 6303.8ms/step | 3.2 samples/s +Step 220/37465 | Loss: 5.9696 | LR: 1.47e-04 | 6309.1ms/step | 3.2 samples/s +Step 230/37465 | Loss: 5.9117 | LR: 1.53e-04 | 6327.0ms/step | 3.2 samples/s +Step 240/37465 | Loss: 5.8905 | LR: 1.60e-04 | 6389.6ms/step | 3.1 samples/s +Step 250/37465 | Loss: 5.8312 | LR: 1.67e-04 | 6367.4ms/step | 3.1 samples/s +Step 260/37465 | Loss: 5.6800 | LR: 1.73e-04 | 6363.2ms/step | 3.1 samples/s +Step 270/37465 | Loss: 5.7001 | LR: 1.80e-04 | 6299.8ms/step | 3.2 samples/s +Step 280/37465 | Loss: 5.7136 | LR: 1.87e-04 | 6306.0ms/step | 3.2 samples/s +Step 290/37465 | Loss: 5.4759 | LR: 1.93e-04 | 6300.9ms/step | 3.2 samples/s +Step 300/37465 | Loss: 5.3876 | LR: 2.00e-04 | 6309.8ms/step | 3.2 samples/s +Step 310/37465 | Loss: 5.2540 | LR: 2.07e-04 | 6324.4ms/step | 3.2 samples/s +Step 320/37465 | Loss: 5.3460 | LR: 2.13e-04 | 6306.3ms/step | 3.2 samples/s +Step 330/37465 | Loss: 5.5442 | LR: 2.20e-04 | 6306.8ms/step | 3.2 samples/s +Step 340/37465 | Loss: 5.3776 | LR: 2.27e-04 | 6312.8ms/step | 3.2 samples/s +Step 350/37465 | Loss: 5.2115 | LR: 2.33e-04 | 6318.3ms/step | 3.2 samples/s diff --git a/epfl_final/logs/l30k1_2b_dense.log.savestep_restart b/epfl_final/logs/l30k1_2b_dense.log.savestep_restart new file mode 100644 index 0000000000000000000000000000000000000000..72dd60a3a33d26c07bb0f5c2a974543a0ebd3ded --- /dev/null +++ b/epfl_final/logs/l30k1_2b_dense.log.savestep_restart @@ -0,0 +1,218 @@ +W0708 06:32:22.733000 139 site-packages/torch/distributed/run.py:851] +W0708 06:32:22.733000 139 site-packages/torch/distributed/run.py:851] ***************************************** +W0708 06:32:22.733000 139 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0708 06:32:22.733000 139 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063234 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 4, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063234", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 500, + "eval_batches": 20, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": true, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "1.2b dense 30k longctx", + "wandb_tags": [ + "qwen3", + "1.2b", + "dense", + "30k", + "longctx", + "b128", + "scaleladder" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: TrueCheckpoint activations: True + +Initializing model from scratch (random weights)... +Checkpoint activations: True +Checkpoint activations: True +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 1536] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 1536] 233,373,696 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 1536] [1, 30720, 1536] 34,606,336 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 1536] [1, 30720, 1536] 1,536 True +├─Linear: 1-2 [1, 30720, 1536] [1, 30720, 151936] 233,373,696 True +================================================================================================================================================================ +Total params: 1,435,726,336 +Trainable params: 1,435,726,336 +Non-trainable params: 0 +Total mult-adds (Units.GIGABYTES): 1.44 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 98319.73 +Params size (MB): 2871.45 +Estimated Total Size (MB): 101191.43 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) + + +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 187325 +Optimizer steps per epoch: 37465 +Total optimizer steps: 37465 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run i61v7k8r +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_063331-i61v7k8r +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 1.2b dense 30k longctx +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i61v7k8r +Starting training... +Switching attention mode for step 1: dense (dense step interval) +W0708 06:35:46.667000 139 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 139 got signal: 15 diff --git a/epfl_final/logs/l30k1_2b_resampled.log b/epfl_final/logs/l30k1_2b_resampled.log new file mode 100644 index 0000000000000000000000000000000000000000..e6bad8a0492b056a01d4d4391560423f4c883443 --- /dev/null +++ b/epfl_final/logs/l30k1_2b_resampled.log @@ -0,0 +1,374 @@ +W0709 00:04:52.738000 161 site-packages/torch/distributed/run.py:851] +W0709 00:04:52.738000 161 site-packages/torch/distributed/run.py:851] ***************************************** +W0709 00:04:52.738000 161 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0709 00:04:52.738000 161 site-packages/torch/distributed/run.py:851] ***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 4, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200", + "force_new_run": false, + "num_epochs": 1, + "max_steps": null, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 300, + "eval_steps": 500, + "eval_batches": 20, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": true, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "1.2b resampled 30k longctx", + "wandb_tags": [ + "qwen3", + "1.2b", + "resampled", + "30k", + "longctx", + "b128", + "scaleladder" + ], + "hidden_size": 1536, + "intermediate_size": 6144, + "num_attention_heads": 12, + "num_key_value_heads": 4, + "num_hidden_layers": null, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120", + "sparse_mask_seed": 0, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: True +Checkpoint activations: True +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200 +Checkpoint activations: True +Checkpoint activations: True + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 1536 + num_hidden_layers: 28 + num_attention_heads: 12 + num_key_value_heads: 4 + head_dim: 128 + intermediate_size: 6144 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1000 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 1,202,352,640 + Trainable parameters: 1,202,352,640 + Non-trainable parameters: 0 + Model size (BF16): 2.40 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +Could not generate detailed summary: Failed to run torchinfo. See above stack traces for more details. Executed layers up to: [Embedding: 2, Qwen3RMSNorm: 4, Linear: 5, Linear: 5, Linear: 5, Qwen3RMSNorm: 5, Qwen3RMSNorm: 5, Qwen3RotaryEmbedding: 5] + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) + +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 187325 +Optimizer steps per epoch: 37465 +Total optimizer steps: 37465 +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 4200 (epoch 0, micro_step 21000) +Resuming training from step 4200 +Resuming wandb run hn868f80 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run hn868f80 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260709_000736-hn868f80 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 1.2b resampled 30k longctx +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/hn868f80 +Starting training... +Validation at step 4200: Loss = 2.5215, Perplexity = 12.45 +wandb: WARNING Tried to log to step 4200 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 4201: sparse_stochastic (configured sparse attention) +Step 4210/37465 | Loss: 2.1631 | LR: 4.89e-04 | 25283.0ms/step | 0.8 samples/s +wandb: WARNING Tried to log to step 4210 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4220/37465 | Loss: 2.5147 | LR: 4.89e-04 | 8085.2ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4220 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4230/37465 | Loss: 2.4117 | LR: 4.89e-04 | 8214.6ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 4230 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4240/37465 | Loss: 2.4558 | LR: 4.89e-04 | 8096.8ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4240 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4250/37465 | Loss: 2.3723 | LR: 4.89e-04 | 8096.8ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4250 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4260/37465 | Loss: 2.4316 | LR: 4.89e-04 | 8094.9ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4260 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4270/37465 | Loss: 2.5079 | LR: 4.89e-04 | 8092.9ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4270 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4280/37465 | Loss: 2.4872 | LR: 4.89e-04 | 8092.7ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4280 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4290/37465 | Loss: 2.5664 | LR: 4.89e-04 | 8095.6ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4290 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4300/37465 | Loss: 2.4478 | LR: 4.89e-04 | 8098.4ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4300 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4310/37465 | Loss: 2.6316 | LR: 4.88e-04 | 8092.3ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4310 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4320/37465 | Loss: 2.3660 | LR: 4.88e-04 | 8094.7ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4320 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4330/37465 | Loss: 2.2324 | LR: 4.88e-04 | 8092.0ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4330 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4340/37465 | Loss: 2.5369 | LR: 4.88e-04 | 8090.7ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4340 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4350/37465 | Loss: 2.6448 | LR: 4.88e-04 | 8231.6ms/step | 2.4 samples/s +wandb: WARNING Tried to log to step 4350 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4360/37465 | Loss: 2.6113 | LR: 4.88e-04 | 8074.1ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4360 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4370/37465 | Loss: 2.4337 | LR: 4.88e-04 | 8078.8ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4370 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4380/37465 | Loss: 2.4301 | LR: 4.88e-04 | 8086.8ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4380 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4390/37465 | Loss: 2.5853 | LR: 4.88e-04 | 8088.1ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4390 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4400/37465 | Loss: 2.3710 | LR: 4.88e-04 | 8088.4ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4400 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4410/37465 | Loss: 2.6066 | LR: 4.88e-04 | 8090.8ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4410 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4420/37465 | Loss: 2.3306 | LR: 4.88e-04 | 8089.6ms/step | 2.5 samples/s +wandb: WARNING Tried to log to step 4420 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 4430/37465 | Loss: 2.5145 | LR: 4.88e-04 | 8089.5ms/step | 2.5 samples/s +Step 4440/37465 | Loss: 2.3588 | LR: 4.88e-04 | 8090.0ms/step | 2.5 samples/s +Step 4450/37465 | Loss: 2.4488 | LR: 4.88e-04 | 8074.7ms/step | 2.5 samples/s +Step 4460/37465 | Loss: 2.3425 | LR: 4.88e-04 | 8086.2ms/step | 2.5 samples/s +Step 4470/37465 | Loss: 2.3559 | LR: 4.87e-04 | 8088.1ms/step | 2.5 samples/s +Step 4480/37465 | Loss: 2.3664 | LR: 4.87e-04 | 8088.8ms/step | 2.5 samples/s +Step 4490/37465 | Loss: 2.4456 | LR: 4.87e-04 | 8088.2ms/step | 2.5 samples/s +Step 4500/37465 | Loss: 2.6641 | LR: 4.87e-04 | 8087.2ms/step | 2.5 samples/s +Validation at step 4500: Loss = 2.5009, Perplexity = 12.19 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4500 +Step 4510/37465 | Loss: 2.3789 | LR: 4.87e-04 | 9685.6ms/step | 2.1 samples/s +Step 4520/37465 | Loss: 2.4548 | LR: 4.87e-04 | 8075.6ms/step | 2.5 samples/s +Step 4530/37465 | Loss: 2.5635 | LR: 4.87e-04 | 8087.1ms/step | 2.5 samples/s +Step 4540/37465 | Loss: 2.4873 | LR: 4.87e-04 | 8087.0ms/step | 2.5 samples/s +Step 4550/37465 | Loss: 2.4339 | LR: 4.87e-04 | 8097.0ms/step | 2.5 samples/s +Step 4560/37465 | Loss: 2.4475 | LR: 4.87e-04 | 8085.9ms/step | 2.5 samples/s +Step 4570/37465 | Loss: 2.2232 | LR: 4.87e-04 | 8090.2ms/step | 2.5 samples/s +Step 4580/37465 | Loss: 2.5504 | LR: 4.87e-04 | 8091.0ms/step | 2.5 samples/s +Step 4590/37465 | Loss: 2.4426 | LR: 4.87e-04 | 8087.9ms/step | 2.5 samples/s +Step 4600/37465 | Loss: 2.6598 | LR: 4.87e-04 | 8084.2ms/step | 2.5 samples/s +Step 4610/37465 | Loss: 2.4243 | LR: 4.86e-04 | 8088.4ms/step | 2.5 samples/s +Step 4620/37465 | Loss: 2.5928 | LR: 4.86e-04 | 8089.5ms/step | 2.5 samples/s +Step 4630/37465 | Loss: 2.5539 | LR: 4.86e-04 | 8079.4ms/step | 2.5 samples/s +Step 4640/37465 | Loss: 2.3340 | LR: 4.86e-04 | 8086.4ms/step | 2.5 samples/s +Step 4650/37465 | Loss: 2.2886 | LR: 4.86e-04 | 8082.5ms/step | 2.5 samples/s +Step 4660/37465 | Loss: 2.4744 | LR: 4.86e-04 | 8080.6ms/step | 2.5 samples/s +Step 4670/37465 | Loss: 2.5130 | LR: 4.86e-04 | 8085.7ms/step | 2.5 samples/s +Step 4680/37465 | Loss: 2.4986 | LR: 4.86e-04 | 8082.6ms/step | 2.5 samples/s +Step 4690/37465 | Loss: 2.6264 | LR: 4.86e-04 | 8083.7ms/step | 2.5 samples/s +Step 4700/37465 | Loss: 2.5099 | LR: 4.86e-04 | 8084.9ms/step | 2.5 samples/s +Step 4710/37465 | Loss: 2.3189 | LR: 4.86e-04 | 8083.4ms/step | 2.5 samples/s +Step 4720/37465 | Loss: 2.5777 | LR: 4.86e-04 | 8083.3ms/step | 2.5 samples/s +Step 4730/37465 | Loss: 2.3423 | LR: 4.86e-04 | 8081.6ms/step | 2.5 samples/s +Step 4740/37465 | Loss: 2.5076 | LR: 4.86e-04 | 8083.0ms/step | 2.5 samples/s +Step 4750/37465 | Loss: 2.4668 | LR: 4.85e-04 | 8084.6ms/step | 2.5 samples/s +Step 4760/37465 | Loss: 2.4106 | LR: 4.85e-04 | 8079.4ms/step | 2.5 samples/s +Step 4770/37465 | Loss: 2.3726 | LR: 4.85e-04 | 8082.2ms/step | 2.5 samples/s +Step 4780/37465 | Loss: 2.5252 | LR: 4.85e-04 | 8103.5ms/step | 2.5 samples/s +Step 4790/37465 | Loss: 2.6367 | LR: 4.85e-04 | 8083.1ms/step | 2.5 samples/s +Step 4800/37465 | Loss: 2.5199 | LR: 4.85e-04 | 8084.3ms/step | 2.5 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4800 +Step 4810/37465 | Loss: 2.5485 | LR: 4.85e-04 | 8989.9ms/step | 2.2 samples/s +Step 4820/37465 | Loss: 2.5484 | LR: 4.85e-04 | 8080.0ms/step | 2.5 samples/s +Step 4830/37465 | Loss: 2.5447 | LR: 4.85e-04 | 8070.9ms/step | 2.5 samples/s +Step 4840/37465 | Loss: 2.3650 | LR: 4.85e-04 | 8066.4ms/step | 2.5 samples/s +Step 4850/37465 | Loss: 2.5167 | LR: 4.85e-04 | 8067.6ms/step | 2.5 samples/s +Step 4860/37465 | Loss: 2.2573 | LR: 4.85e-04 | 8067.0ms/step | 2.5 samples/s +Step 4870/37465 | Loss: 2.5819 | LR: 4.85e-04 | 8069.0ms/step | 2.5 samples/s +Step 4880/37465 | Loss: 2.8365 | LR: 4.85e-04 | 8070.2ms/step | 2.5 samples/s +Step 4890/37465 | Loss: 2.6807 | LR: 4.84e-04 | 8069.4ms/step | 2.5 samples/s +Step 4900/37465 | Loss: 2.3875 | LR: 4.84e-04 | 8062.9ms/step | 2.5 samples/s +Step 4910/37465 | Loss: 2.4953 | LR: 4.84e-04 | 8066.4ms/step | 2.5 samples/s +Step 4920/37465 | Loss: 2.6403 | LR: 4.84e-04 | 8063.9ms/step | 2.5 samples/s +Step 4930/37465 | Loss: 2.1881 | LR: 4.84e-04 | 8092.9ms/step | 2.5 samples/s +Step 4940/37465 | Loss: 2.3991 | LR: 4.84e-04 | 8084.3ms/step | 2.5 samples/s +Step 4950/37465 | Loss: 2.4519 | LR: 4.84e-04 | 8083.1ms/step | 2.5 samples/s +Step 4960/37465 | Loss: 2.4604 | LR: 4.84e-04 | 8082.6ms/step | 2.5 samples/s +Step 4970/37465 | Loss: 2.4633 | LR: 4.84e-04 | 8083.0ms/step | 2.5 samples/s +Step 4980/37465 | Loss: 2.3453 | LR: 4.84e-04 | 8084.9ms/step | 2.5 samples/s +Step 4990/37465 | Loss: 2.4328 | LR: 4.84e-04 | 8081.4ms/step | 2.5 samples/s +Step 5000/37465 | Loss: 2.6884 | LR: 4.84e-04 | 8084.0ms/step | 2.5 samples/s +Validation at step 5000: Loss = 2.4698, Perplexity = 11.82 +Step 5010/37465 | Loss: 2.3863 | LR: 4.84e-04 | 8754.2ms/step | 2.3 samples/s +Step 5020/37465 | Loss: 2.4304 | LR: 4.83e-04 | 8085.3ms/step | 2.5 samples/s +Step 5030/37465 | Loss: 2.5446 | LR: 4.83e-04 | 8084.4ms/step | 2.5 samples/s +Step 5040/37465 | Loss: 2.3558 | LR: 4.83e-04 | 8083.1ms/step | 2.5 samples/s +Step 5050/37465 | Loss: 2.3629 | LR: 4.83e-04 | 8085.8ms/step | 2.5 samples/s +Step 5060/37465 | Loss: 2.6030 | LR: 4.83e-04 | 8085.0ms/step | 2.5 samples/s +Step 5070/37465 | Loss: 2.6285 | LR: 4.83e-04 | 8083.1ms/step | 2.5 samples/s +Step 5080/37465 | Loss: 2.3657 | LR: 4.83e-04 | 8083.7ms/step | 2.5 samples/s +Step 5090/37465 | Loss: 2.2803 | LR: 4.83e-04 | 8086.5ms/step | 2.5 samples/s +Step 5100/37465 | Loss: 2.6280 | LR: 4.83e-04 | 8084.4ms/step | 2.5 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-5100 +Step 5110/37465 | Loss: 2.4089 | LR: 4.83e-04 | 8955.2ms/step | 2.2 samples/s +Step 5120/37465 | Loss: 2.2645 | LR: 4.83e-04 | 8074.0ms/step | 2.5 samples/s +Step 5130/37465 | Loss: 2.3024 | LR: 4.83e-04 | 8080.2ms/step | 2.5 samples/s +Step 5140/37465 | Loss: 2.3108 | LR: 4.83e-04 | 8071.6ms/step | 2.5 samples/s +Step 5150/37465 | Loss: 2.4108 | LR: 4.82e-04 | 8075.9ms/step | 2.5 samples/s +Step 5160/37465 | Loss: 2.2551 | LR: 4.82e-04 | 8082.8ms/step | 2.5 samples/s +Step 5170/37465 | Loss: 2.7241 | LR: 4.82e-04 | 8084.8ms/step | 2.5 samples/s +Step 5180/37465 | Loss: 2.3629 | LR: 4.82e-04 | 8087.8ms/step | 2.5 samples/s +Step 5190/37465 | Loss: 2.6014 | LR: 4.82e-04 | 8083.4ms/step | 2.5 samples/s +Step 5200/37465 | Loss: 2.5887 | LR: 4.82e-04 | 8085.5ms/step | 2.5 samples/s +Step 5210/37465 | Loss: 2.3217 | LR: 4.82e-04 | 8090.2ms/step | 2.5 samples/s +Step 5220/37465 | Loss: 2.3421 | LR: 4.82e-04 | 8085.4ms/step | 2.5 samples/s +Step 5230/37465 | Loss: 2.4593 | LR: 4.82e-04 | 8082.5ms/step | 2.5 samples/s +Step 5240/37465 | Loss: 2.4859 | LR: 4.82e-04 | 8081.9ms/step | 2.5 samples/s +Step 5250/37465 | Loss: 2.6048 | LR: 4.82e-04 | 8084.0ms/step | 2.5 samples/s +Step 5260/37465 | Loss: 2.5825 | LR: 4.82e-04 | 8085.2ms/step | 2.5 samples/s +Step 5270/37465 | Loss: 2.4574 | LR: 4.82e-04 | 8084.5ms/step | 2.5 samples/s +Step 5280/37465 | Loss: 2.3719 | LR: 4.81e-04 | 8085.7ms/step | 2.5 samples/s +Step 5290/37465 | Loss: 2.5576 | LR: 4.81e-04 | 8086.5ms/step | 2.5 samples/s +Step 5300/37465 | Loss: 2.6705 | LR: 4.81e-04 | 8084.8ms/step | 2.5 samples/s +Step 5310/37465 | Loss: 2.3375 | LR: 4.81e-04 | 8086.1ms/step | 2.5 samples/s +Step 5320/37465 | Loss: 2.4033 | LR: 4.81e-04 | 8087.5ms/step | 2.5 samples/s +Step 5330/37465 | Loss: 2.1951 | LR: 4.81e-04 | 8082.9ms/step | 2.5 samples/s +Step 5340/37465 | Loss: 2.4334 | LR: 4.81e-04 | 8083.9ms/step | 2.5 samples/s +Step 5350/37465 | Loss: 2.6761 | LR: 4.81e-04 | 8084.2ms/step | 2.5 samples/s +Step 5360/37465 | Loss: 2.5290 | LR: 4.81e-04 | 8082.7ms/step | 2.5 samples/s +Step 5370/37465 | Loss: 2.5438 | LR: 4.81e-04 | 8080.5ms/step | 2.5 samples/s +Step 5380/37465 | Loss: 2.4727 | LR: 4.81e-04 | 8081.9ms/step | 2.5 samples/s +Step 5390/37465 | Loss: 2.4013 | LR: 4.81e-04 | 8085.5ms/step | 2.5 samples/s +Step 5400/37465 | Loss: 2.5290 | LR: 4.80e-04 | 8079.6ms/step | 2.5 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-5400 +Step 5410/37465 | Loss: 2.4230 | LR: 4.80e-04 | 8996.0ms/step | 2.2 samples/s +Step 5420/37465 | Loss: 2.3855 | LR: 4.80e-04 | 8068.2ms/step | 2.5 samples/s +Step 5430/37465 | Loss: 2.3206 | LR: 4.80e-04 | 8073.6ms/step | 2.5 samples/s +Step 5440/37465 | Loss: 2.4543 | LR: 4.80e-04 | 8068.1ms/step | 2.5 samples/s +Step 5450/37465 | Loss: 2.3674 | LR: 4.80e-04 | 8071.0ms/step | 2.5 samples/s +Step 5460/37465 | Loss: 2.5938 | LR: 4.80e-04 | 8066.7ms/step | 2.5 samples/s +Step 5470/37465 | Loss: 2.5577 | LR: 4.80e-04 | 8071.6ms/step | 2.5 samples/s +Step 5480/37465 | Loss: 2.3778 | LR: 4.80e-04 | 8071.7ms/step | 2.5 samples/s +Step 5490/37465 | Loss: 2.6355 | LR: 4.80e-04 | 8076.2ms/step | 2.5 samples/s +Step 5500/37465 | Loss: 2.3604 | LR: 4.80e-04 | 8084.9ms/step | 2.5 samples/s +Validation at step 5500: Loss = 2.4435, Perplexity = 11.51 +Step 5510/37465 | Loss: 2.2868 | LR: 4.80e-04 | 8752.7ms/step | 2.3 samples/s +Step 5520/37465 | Loss: 2.3157 | LR: 4.79e-04 | 8081.4ms/step | 2.5 samples/s +Step 5530/37465 | Loss: 2.4186 | LR: 4.79e-04 | 8077.2ms/step | 2.5 samples/s +Step 5540/37465 | Loss: 2.2127 | LR: 4.79e-04 | 8078.9ms/step | 2.5 samples/s +Step 5550/37465 | Loss: 2.2966 | LR: 4.79e-04 | 8079.7ms/step | 2.5 samples/s +Step 5560/37465 | Loss: 2.6664 | LR: 4.79e-04 | 8083.2ms/step | 2.5 samples/s +Step 5570/37465 | Loss: 2.2826 | LR: 4.79e-04 | 8082.9ms/step | 2.5 samples/s +Step 5580/37465 | Loss: 2.4479 | LR: 4.79e-04 | 8083.3ms/step | 2.5 samples/s +Step 5590/37465 | Loss: 2.4605 | LR: 4.79e-04 | 8083.7ms/step | 2.5 samples/s +Step 5600/37465 | Loss: 2.2986 | LR: 4.79e-04 | 8081.9ms/step | 2.5 samples/s +Step 5610/37465 | Loss: 2.3985 | LR: 4.79e-04 | 8082.0ms/step | 2.5 samples/s +Step 5620/37465 | Loss: 2.4390 | LR: 4.79e-04 | 8083.8ms/step | 2.5 samples/s +Step 5630/37465 | Loss: 2.5223 | LR: 4.79e-04 | 8083.9ms/step | 2.5 samples/s +Step 5640/37465 | Loss: 2.3728 | LR: 4.78e-04 | 8082.4ms/step | 2.5 samples/s +Step 5650/37465 | Loss: 2.2795 | LR: 4.78e-04 | 8085.3ms/step | 2.5 samples/s +Step 5660/37465 | Loss: 2.4398 | LR: 4.78e-04 | 8082.7ms/step | 2.5 samples/s +Step 5670/37465 | Loss: 2.5131 | LR: 4.78e-04 | 8084.9ms/step | 2.5 samples/s +Step 5680/37465 | Loss: 2.4396 | LR: 4.78e-04 | 8082.2ms/step | 2.5 samples/s +Step 5690/37465 | Loss: 2.3987 | LR: 4.78e-04 | 8084.2ms/step | 2.5 samples/s +Step 5700/37465 | Loss: 2.5544 | LR: 4.78e-04 | 8082.0ms/step | 2.5 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-5700 +Step 5710/37465 | Loss: 2.4240 | LR: 4.78e-04 | 8972.8ms/step | 2.2 samples/s +Step 5720/37465 | Loss: 2.3987 | LR: 4.78e-04 | 8075.1ms/step | 2.5 samples/s +Step 5730/37465 | Loss: 2.4198 | LR: 4.78e-04 | 8064.6ms/step | 2.5 samples/s +Step 5740/37465 | Loss: 2.3960 | LR: 4.78e-04 | 8068.0ms/step | 2.5 samples/s +Step 5750/37465 | Loss: 2.4376 | LR: 4.77e-04 | 8066.7ms/step | 2.5 samples/s +Step 5760/37465 | Loss: 2.1489 | LR: 4.77e-04 | 8065.3ms/step | 2.5 samples/s +Step 5770/37465 | Loss: 2.4842 | LR: 4.77e-04 | 8069.4ms/step | 2.5 samples/s +W0709 03:43:10.658000 161 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 161 got signal: 15 diff --git a/epfl_final/logs/l30k365m_dense.log b/epfl_final/logs/l30k365m_dense.log new file mode 100644 index 0000000000000000000000000000000000000000..32cae746ddc77ce68291bb006e04449d28a11659 --- /dev/null +++ b/epfl_final/logs/l30k365m_dense.log @@ -0,0 +1,1436 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 11700, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 20, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 30k", + "wandb_tags": [ + "qwen3", + "365m", + "30k", + "scaleladder", + "dense" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 768] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 768] [1, 30720, 768] 768 True +├─Linear: 1-2 [1, 30720, 768] [1, 30720, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 475.65 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 58494.81 +Params size (MB): 951.29 +Estimated Total Size (MB): 59446.35 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 1 +Gradient accumulation steps: 20 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 749299 +Optimizer steps per epoch: 37464 +Total optimizer steps: 11700 (capped by max_steps; 0.31 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run jmi9928p +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035828-jmi9928p +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m dense 30k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/jmi9928p +Starting training... +Switching attention mode for step 1: dense (dense step interval) +Step 10/11700 | Loss: 12.0484 | LR: 6.67e-06 | 13293.3ms/step | 1.5 samples/s +Step 20/11700 | Loss: 11.9702 | LR: 1.33e-05 | 9042.9ms/step | 2.2 samples/s +Step 30/11700 | Loss: 11.7177 | LR: 2.00e-05 | 9041.7ms/step | 2.2 samples/s +Step 40/11700 | Loss: 11.2048 | LR: 2.67e-05 | 9025.1ms/step | 2.2 samples/s +Step 50/11700 | Loss: 10.8659 | LR: 3.33e-05 | 9007.3ms/step | 2.2 samples/s +Step 60/11700 | Loss: 10.5798 | LR: 4.00e-05 | 8999.0ms/step | 2.2 samples/s +Step 70/11700 | Loss: 10.3002 | LR: 4.67e-05 | 8990.6ms/step | 2.2 samples/s +Step 80/11700 | Loss: 10.0314 | LR: 5.33e-05 | 8986.6ms/step | 2.2 samples/s +Step 90/11700 | Loss: 9.7412 | LR: 6.00e-05 | 8980.7ms/step | 2.2 samples/s +Step 100/11700 | Loss: 9.3527 | LR: 6.67e-05 | 8990.8ms/step | 2.2 samples/s +Step 110/11700 | Loss: 8.8550 | LR: 7.33e-05 | 8994.8ms/step | 2.2 samples/s +Step 120/11700 | Loss: 8.3007 | LR: 8.00e-05 | 8988.8ms/step | 2.2 samples/s +Step 130/11700 | Loss: 7.8459 | LR: 8.67e-05 | 8996.4ms/step | 2.2 samples/s +Step 140/11700 | Loss: 7.5719 | LR: 9.33e-05 | 9003.1ms/step | 2.2 samples/s +Step 150/11700 | Loss: 7.3697 | LR: 1.00e-04 | 9012.8ms/step | 2.2 samples/s +Step 160/11700 | Loss: 7.1604 | LR: 1.07e-04 | 9020.9ms/step | 2.2 samples/s +Step 170/11700 | Loss: 7.0293 | LR: 1.13e-04 | 9025.3ms/step | 2.2 samples/s +Step 180/11700 | Loss: 6.8967 | LR: 1.20e-04 | 9037.0ms/step | 2.2 samples/s +Step 190/11700 | Loss: 6.8259 | LR: 1.27e-04 | 9042.1ms/step | 2.2 samples/s +Step 200/11700 | Loss: 6.6779 | LR: 1.33e-04 | 9045.3ms/step | 2.2 samples/s +Step 210/11700 | Loss: 6.5893 | LR: 1.40e-04 | 9045.9ms/step | 2.2 samples/s +Step 220/11700 | Loss: 6.4456 | LR: 1.47e-04 | 9050.2ms/step | 2.2 samples/s +Step 230/11700 | Loss: 6.3800 | LR: 1.53e-04 | 9067.1ms/step | 2.2 samples/s +Step 240/11700 | Loss: 6.2418 | LR: 1.60e-04 | 9062.0ms/step | 2.2 samples/s +Step 250/11700 | Loss: 6.2527 | LR: 1.67e-04 | 9079.3ms/step | 2.2 samples/s +Validation at step 250: Loss = 6.1182, Perplexity = 454.06 +Step 260/11700 | Loss: 6.1697 | LR: 1.73e-04 | 23829.3ms/step | 0.8 samples/s +Step 270/11700 | Loss: 6.0604 | LR: 1.80e-04 | 9080.0ms/step | 2.2 samples/s +Step 280/11700 | Loss: 6.0021 | LR: 1.87e-04 | 9096.9ms/step | 2.2 samples/s +Step 290/11700 | Loss: 5.8951 | LR: 1.93e-04 | 9085.4ms/step | 2.2 samples/s +Step 300/11700 | Loss: 5.8827 | LR: 2.00e-04 | 9077.2ms/step | 2.2 samples/s +Step 310/11700 | Loss: 5.8564 | LR: 2.07e-04 | 9093.0ms/step | 2.2 samples/s +Step 320/11700 | Loss: 5.7371 | LR: 2.13e-04 | 9085.5ms/step | 2.2 samples/s +Step 330/11700 | Loss: 5.6231 | LR: 2.20e-04 | 9076.8ms/step | 2.2 samples/s +Step 340/11700 | Loss: 5.5681 | LR: 2.27e-04 | 9104.3ms/step | 2.2 samples/s +Step 350/11700 | Loss: 5.4096 | LR: 2.33e-04 | 9096.5ms/step | 2.2 samples/s +Step 360/11700 | Loss: 5.4823 | LR: 2.40e-04 | 9085.0ms/step | 2.2 samples/s +Step 370/11700 | Loss: 5.5145 | LR: 2.47e-04 | 9080.0ms/step | 2.2 samples/s +Step 380/11700 | Loss: 5.4947 | LR: 2.53e-04 | 9091.2ms/step | 2.2 samples/s +Step 390/11700 | Loss: 5.4637 | LR: 2.60e-04 | 9079.5ms/step | 2.2 samples/s +Step 400/11700 | Loss: 5.3325 | LR: 2.67e-04 | 9098.3ms/step | 2.2 samples/s +Step 410/11700 | Loss: 5.3319 | LR: 2.73e-04 | 9088.6ms/step | 2.2 samples/s +Step 420/11700 | Loss: 5.2946 | LR: 2.80e-04 | 9079.4ms/step | 2.2 samples/s +Step 430/11700 | Loss: 5.2497 | LR: 2.87e-04 | 9119.7ms/step | 2.2 samples/s +Step 440/11700 | Loss: 5.2477 | LR: 2.93e-04 | 9102.9ms/step | 2.2 samples/s +Step 450/11700 | Loss: 5.1558 | LR: 3.00e-04 | 9085.8ms/step | 2.2 samples/s +Step 460/11700 | Loss: 5.1813 | LR: 3.07e-04 | 9087.6ms/step | 2.2 samples/s +Step 470/11700 | Loss: 5.1461 | LR: 3.13e-04 | 9095.0ms/step | 2.2 samples/s +Step 480/11700 | Loss: 5.1555 | LR: 3.20e-04 | 9096.5ms/step | 2.2 samples/s +Step 490/11700 | Loss: 5.0623 | LR: 3.27e-04 | 9117.5ms/step | 2.2 samples/s +Step 500/11700 | Loss: 5.0113 | LR: 3.33e-04 | 9089.6ms/step | 2.2 samples/s +Validation at step 500: Loss = 4.9550, Perplexity = 141.88 +Step 510/11700 | Loss: 4.9215 | LR: 3.40e-04 | 22160.2ms/step | 0.9 samples/s +Step 520/11700 | Loss: 4.9355 | LR: 3.47e-04 | 9102.8ms/step | 2.2 samples/s +Step 530/11700 | Loss: 4.8666 | LR: 3.53e-04 | 9110.1ms/step | 2.2 samples/s +Step 540/11700 | Loss: 4.7775 | LR: 3.60e-04 | 9102.8ms/step | 2.2 samples/s +Step 550/11700 | Loss: 4.8019 | LR: 3.67e-04 | 9075.3ms/step | 2.2 samples/s +Step 560/11700 | Loss: 4.7867 | LR: 3.73e-04 | 9104.8ms/step | 2.2 samples/s +Step 570/11700 | Loss: 4.5575 | LR: 3.80e-04 | 9093.7ms/step | 2.2 samples/s +Step 580/11700 | Loss: 4.5903 | LR: 3.87e-04 | 9092.8ms/step | 2.2 samples/s +Step 590/11700 | Loss: 4.5341 | LR: 3.93e-04 | 9100.0ms/step | 2.2 samples/s +Step 600/11700 | Loss: 4.5988 | LR: 4.00e-04 | 9099.7ms/step | 2.2 samples/s +Step 610/11700 | Loss: 4.5735 | LR: 4.07e-04 | 9095.1ms/step | 2.2 samples/s +Step 620/11700 | Loss: 4.5788 | LR: 4.13e-04 | 9107.8ms/step | 2.2 samples/s +Step 630/11700 | Loss: 4.4359 | LR: 4.20e-04 | 9095.2ms/step | 2.2 samples/s +Step 640/11700 | Loss: 4.5140 | LR: 4.27e-04 | 9102.9ms/step | 2.2 samples/s +Step 650/11700 | Loss: 4.3762 | LR: 4.33e-04 | 9091.0ms/step | 2.2 samples/s +Step 660/11700 | Loss: 4.4714 | LR: 4.40e-04 | 9092.0ms/step | 2.2 samples/s +Step 670/11700 | Loss: 4.3703 | LR: 4.47e-04 | 9097.5ms/step | 2.2 samples/s +Step 680/11700 | Loss: 4.2817 | LR: 4.53e-04 | 9076.5ms/step | 2.2 samples/s +Step 690/11700 | Loss: 4.1302 | LR: 4.60e-04 | 9092.4ms/step | 2.2 samples/s +Step 700/11700 | Loss: 4.2661 | LR: 4.67e-04 | 9110.4ms/step | 2.2 samples/s +Step 710/11700 | Loss: 4.2591 | LR: 4.73e-04 | 9083.5ms/step | 2.2 samples/s +Step 720/11700 | Loss: 4.1482 | LR: 4.80e-04 | 9090.7ms/step | 2.2 samples/s +Step 730/11700 | Loss: 4.0106 | LR: 4.87e-04 | 9111.4ms/step | 2.2 samples/s +Step 740/11700 | Loss: 4.1462 | LR: 4.93e-04 | 9101.3ms/step | 2.2 samples/s +Step 750/11700 | Loss: 4.0399 | LR: 5.00e-04 | 9088.8ms/step | 2.2 samples/s +Validation at step 750: Loss = 3.9839, Perplexity = 53.73 +Step 760/11700 | Loss: 4.0546 | LR: 5.00e-04 | 22234.9ms/step | 0.9 samples/s +Step 770/11700 | Loss: 3.8123 | LR: 5.00e-04 | 9138.5ms/step | 2.2 samples/s +Step 780/11700 | Loss: 4.0243 | LR: 5.00e-04 | 9137.2ms/step | 2.2 samples/s +Step 790/11700 | Loss: 3.7985 | LR: 5.00e-04 | 9123.2ms/step | 2.2 samples/s +Step 800/11700 | Loss: 3.7874 | LR: 5.00e-04 | 9138.4ms/step | 2.2 samples/s +Step 810/11700 | Loss: 3.8176 | LR: 5.00e-04 | 9129.4ms/step | 2.2 samples/s +Step 820/11700 | Loss: 3.7912 | LR: 5.00e-04 | 9130.9ms/step | 2.2 samples/s +Step 830/11700 | Loss: 3.7794 | LR: 5.00e-04 | 9125.9ms/step | 2.2 samples/s +Step 840/11700 | Loss: 3.7799 | LR: 5.00e-04 | 9133.8ms/step | 2.2 samples/s +Step 850/11700 | Loss: 3.8138 | LR: 5.00e-04 | 9125.5ms/step | 2.2 samples/s +Step 860/11700 | Loss: 3.6481 | LR: 5.00e-04 | 9129.5ms/step | 2.2 samples/s +Step 870/11700 | Loss: 3.6798 | LR: 5.00e-04 | 9135.2ms/step | 2.2 samples/s +Step 880/11700 | Loss: 3.7357 | LR: 5.00e-04 | 9114.8ms/step | 2.2 samples/s +Step 890/11700 | Loss: 3.5737 | LR: 5.00e-04 | 9117.9ms/step | 2.2 samples/s +Step 900/11700 | Loss: 3.5135 | LR: 5.00e-04 | 9103.1ms/step | 2.2 samples/s +Step 910/11700 | Loss: 3.6803 | LR: 5.00e-04 | 9095.5ms/step | 2.2 samples/s +Step 920/11700 | Loss: 3.5651 | LR: 5.00e-04 | 9104.5ms/step | 2.2 samples/s +Step 930/11700 | Loss: 3.5462 | LR: 5.00e-04 | 9103.9ms/step | 2.2 samples/s +Step 940/11700 | Loss: 3.6153 | LR: 5.00e-04 | 9104.7ms/step | 2.2 samples/s +Step 950/11700 | Loss: 3.4468 | LR: 5.00e-04 | 9096.4ms/step | 2.2 samples/s +Step 960/11700 | Loss: 3.5193 | LR: 5.00e-04 | 9103.9ms/step | 2.2 samples/s +Step 970/11700 | Loss: 3.5809 | LR: 5.00e-04 | 9094.9ms/step | 2.2 samples/s +Step 980/11700 | Loss: 3.2973 | LR: 4.99e-04 | 9115.3ms/step | 2.2 samples/s +Step 990/11700 | Loss: 3.5138 | LR: 4.99e-04 | 9109.0ms/step | 2.2 samples/s +Step 1000/11700 | Loss: 3.4018 | LR: 4.99e-04 | 9115.6ms/step | 2.2 samples/s +Validation at step 1000: Loss = 3.4574, Perplexity = 31.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-1000 +Step 1010/11700 | Loss: 3.5266 | LR: 4.99e-04 | 22490.1ms/step | 0.9 samples/s +Step 1020/11700 | Loss: 3.4309 | LR: 4.99e-04 | 9129.2ms/step | 2.2 samples/s +Step 1030/11700 | Loss: 3.4492 | LR: 4.99e-04 | 9101.9ms/step | 2.2 samples/s +Step 1040/11700 | Loss: 3.4735 | LR: 4.99e-04 | 9105.3ms/step | 2.2 samples/s +Step 1050/11700 | Loss: 3.3925 | LR: 4.99e-04 | 9098.1ms/step | 2.2 samples/s +Step 1060/11700 | Loss: 3.4477 | LR: 4.99e-04 | 9101.3ms/step | 2.2 samples/s +Step 1070/11700 | Loss: 3.3923 | LR: 4.99e-04 | 9093.7ms/step | 2.2 samples/s +Step 1080/11700 | Loss: 3.2509 | LR: 4.99e-04 | 9116.2ms/step | 2.2 samples/s +Step 1090/11700 | Loss: 3.3119 | LR: 4.99e-04 | 9114.6ms/step | 2.2 samples/s +Step 1100/11700 | Loss: 3.5159 | LR: 4.99e-04 | 9114.6ms/step | 2.2 samples/s +Step 1110/11700 | Loss: 3.4564 | LR: 4.99e-04 | 9119.0ms/step | 2.2 samples/s +Step 1120/11700 | Loss: 3.3510 | LR: 4.99e-04 | 9120.4ms/step | 2.2 samples/s +Step 1130/11700 | Loss: 3.3622 | LR: 4.99e-04 | 9097.7ms/step | 2.2 samples/s +Step 1140/11700 | Loss: 3.1304 | LR: 4.98e-04 | 9076.4ms/step | 2.2 samples/s +Step 1150/11700 | Loss: 3.2823 | LR: 4.98e-04 | 9121.4ms/step | 2.2 samples/s +Step 1160/11700 | Loss: 3.2942 | LR: 4.98e-04 | 9099.2ms/step | 2.2 samples/s +Step 1170/11700 | Loss: 3.2613 | LR: 4.98e-04 | 9085.6ms/step | 2.2 samples/s +Step 1180/11700 | Loss: 3.1969 | LR: 4.98e-04 | 9100.9ms/step | 2.2 samples/s +Step 1190/11700 | Loss: 3.1582 | LR: 4.98e-04 | 9122.3ms/step | 2.2 samples/s +Step 1200/11700 | Loss: 3.0894 | LR: 4.98e-04 | 9101.2ms/step | 2.2 samples/s +Step 1210/11700 | Loss: 3.4270 | LR: 4.98e-04 | 9094.2ms/step | 2.2 samples/s +Step 1220/11700 | Loss: 3.2168 | LR: 4.98e-04 | 9092.8ms/step | 2.2 samples/s +Step 1230/11700 | Loss: 3.2558 | LR: 4.98e-04 | 9083.7ms/step | 2.2 samples/s +Step 1240/11700 | Loss: 3.2018 | LR: 4.98e-04 | 9082.1ms/step | 2.2 samples/s +Step 1250/11700 | Loss: 3.1800 | LR: 4.97e-04 | 9082.1ms/step | 2.2 samples/s +Validation at step 1250: Loss = 3.2275, Perplexity = 25.22 +Step 1260/11700 | Loss: 3.2539 | LR: 4.97e-04 | 22146.9ms/step | 0.9 samples/s +Step 1270/11700 | Loss: 3.0820 | LR: 4.97e-04 | 9078.5ms/step | 2.2 samples/s +Step 1280/11700 | Loss: 3.1964 | LR: 4.97e-04 | 9078.5ms/step | 2.2 samples/s +Step 1290/11700 | Loss: 3.1911 | LR: 4.97e-04 | 9079.5ms/step | 2.2 samples/s +Step 1300/11700 | Loss: 3.1982 | LR: 4.97e-04 | 9075.3ms/step | 2.2 samples/s +Step 1310/11700 | Loss: 3.2471 | LR: 4.97e-04 | 9077.9ms/step | 2.2 samples/s +Step 1320/11700 | Loss: 3.2131 | LR: 4.97e-04 | 9078.5ms/step | 2.2 samples/s +Step 1330/11700 | Loss: 3.1315 | LR: 4.97e-04 | 9079.6ms/step | 2.2 samples/s +Step 1340/11700 | Loss: 3.1755 | LR: 4.96e-04 | 9078.4ms/step | 2.2 samples/s +Step 1350/11700 | Loss: 3.1888 | LR: 4.96e-04 | 9083.5ms/step | 2.2 samples/s +Step 1360/11700 | Loss: 3.1484 | LR: 4.96e-04 | 9089.7ms/step | 2.2 samples/s +Step 1370/11700 | Loss: 3.0922 | LR: 4.96e-04 | 9102.5ms/step | 2.2 samples/s +Step 1380/11700 | Loss: 2.9564 | LR: 4.96e-04 | 9095.3ms/step | 2.2 samples/s +Step 1390/11700 | Loss: 3.1233 | LR: 4.96e-04 | 9099.3ms/step | 2.2 samples/s +Step 1400/11700 | Loss: 3.2290 | LR: 4.96e-04 | 9094.7ms/step | 2.2 samples/s +Step 1410/11700 | Loss: 3.1625 | LR: 4.96e-04 | 9096.5ms/step | 2.2 samples/s +Step 1420/11700 | Loss: 3.0551 | LR: 4.95e-04 | 9093.0ms/step | 2.2 samples/s +Step 1430/11700 | Loss: 3.0973 | LR: 4.95e-04 | 9094.7ms/step | 2.2 samples/s +Step 1440/11700 | Loss: 3.0840 | LR: 4.95e-04 | 9095.2ms/step | 2.2 samples/s +Step 1450/11700 | Loss: 3.0051 | LR: 4.95e-04 | 9094.8ms/step | 2.2 samples/s +Step 1460/11700 | Loss: 3.0610 | LR: 4.95e-04 | 9092.2ms/step | 2.2 samples/s +Step 1470/11700 | Loss: 3.0834 | LR: 4.95e-04 | 9090.2ms/step | 2.2 samples/s +Step 1480/11700 | Loss: 3.0170 | LR: 4.95e-04 | 9091.9ms/step | 2.2 samples/s +Step 1490/11700 | Loss: 3.1768 | LR: 4.94e-04 | 9092.1ms/step | 2.2 samples/s +Step 1500/11700 | Loss: 3.0148 | LR: 4.94e-04 | 9099.8ms/step | 2.2 samples/s +Validation at step 1500: Loss = 3.0923, Perplexity = 22.03 +Step 1510/11700 | Loss: 3.0810 | LR: 4.94e-04 | 22209.7ms/step | 0.9 samples/s +Step 1520/11700 | Loss: 3.0657 | LR: 4.94e-04 | 9097.7ms/step | 2.2 samples/s +Step 1530/11700 | Loss: 3.0633 | LR: 4.94e-04 | 9094.2ms/step | 2.2 samples/s +Step 1540/11700 | Loss: 3.2202 | LR: 4.94e-04 | 9094.2ms/step | 2.2 samples/s +Step 1550/11700 | Loss: 3.0666 | LR: 4.93e-04 | 9089.4ms/step | 2.2 samples/s +Step 1560/11700 | Loss: 3.0896 | LR: 4.93e-04 | 9089.8ms/step | 2.2 samples/s +Step 1570/11700 | Loss: 3.0764 | LR: 4.93e-04 | 9095.4ms/step | 2.2 samples/s +Step 1580/11700 | Loss: 3.0389 | LR: 4.93e-04 | 9097.6ms/step | 2.2 samples/s +Step 1590/11700 | Loss: 3.1677 | LR: 4.93e-04 | 9092.2ms/step | 2.2 samples/s +Step 1600/11700 | Loss: 3.0551 | LR: 4.93e-04 | 9091.6ms/step | 2.2 samples/s +Step 1610/11700 | Loss: 3.0999 | LR: 4.92e-04 | 9087.0ms/step | 2.2 samples/s +Step 1620/11700 | Loss: 2.9752 | LR: 4.92e-04 | 9088.2ms/step | 2.2 samples/s +Step 1630/11700 | Loss: 2.8849 | LR: 4.92e-04 | 9086.6ms/step | 2.2 samples/s +Step 1640/11700 | Loss: 3.0746 | LR: 4.92e-04 | 9086.6ms/step | 2.2 samples/s +Step 1650/11700 | Loss: 3.0112 | LR: 4.92e-04 | 9087.6ms/step | 2.2 samples/s +Step 1660/11700 | Loss: 3.0404 | LR: 4.92e-04 | 9087.7ms/step | 2.2 samples/s +Step 1670/11700 | Loss: 3.1605 | LR: 4.91e-04 | 9087.4ms/step | 2.2 samples/s +Step 1680/11700 | Loss: 3.0063 | LR: 4.91e-04 | 9089.9ms/step | 2.2 samples/s +Step 1690/11700 | Loss: 2.8694 | LR: 4.91e-04 | 9092.3ms/step | 2.2 samples/s +Step 1700/11700 | Loss: 2.9316 | LR: 4.91e-04 | 9092.5ms/step | 2.2 samples/s +Step 1710/11700 | Loss: 3.0661 | LR: 4.91e-04 | 9096.3ms/step | 2.2 samples/s +Step 1720/11700 | Loss: 2.9882 | LR: 4.90e-04 | 9101.3ms/step | 2.2 samples/s +Step 1730/11700 | Loss: 3.0125 | LR: 4.90e-04 | 9093.6ms/step | 2.2 samples/s +Step 1740/11700 | Loss: 2.9136 | LR: 4.90e-04 | 9082.7ms/step | 2.2 samples/s +Step 1750/11700 | Loss: 2.8846 | LR: 4.90e-04 | 9090.7ms/step | 2.2 samples/s +Validation at step 1750: Loss = 2.9990, Perplexity = 20.07 +Step 1760/11700 | Loss: 2.9725 | LR: 4.90e-04 | 22211.9ms/step | 0.9 samples/s +Step 1770/11700 | Loss: 2.9781 | LR: 4.89e-04 | 9093.9ms/step | 2.2 samples/s +Step 1780/11700 | Loss: 2.9992 | LR: 4.89e-04 | 9096.7ms/step | 2.2 samples/s +Step 1790/11700 | Loss: 2.9722 | LR: 4.89e-04 | 9094.9ms/step | 2.2 samples/s +Step 1800/11700 | Loss: 2.9929 | LR: 4.89e-04 | 9095.2ms/step | 2.2 samples/s +Step 1810/11700 | Loss: 3.0468 | LR: 4.89e-04 | 9094.5ms/step | 2.2 samples/s +Step 1820/11700 | Loss: 2.9723 | LR: 4.88e-04 | 9095.7ms/step | 2.2 samples/s +Step 1830/11700 | Loss: 2.9482 | LR: 4.88e-04 | 9091.6ms/step | 2.2 samples/s +Step 1840/11700 | Loss: 2.9366 | LR: 4.88e-04 | 9094.6ms/step | 2.2 samples/s +Step 1850/11700 | Loss: 2.9620 | LR: 4.88e-04 | 9095.2ms/step | 2.2 samples/s +Step 1860/11700 | Loss: 2.9449 | LR: 4.87e-04 | 9103.7ms/step | 2.2 samples/s +Step 1870/11700 | Loss: 2.8820 | LR: 4.87e-04 | 9097.8ms/step | 2.2 samples/s +Step 1880/11700 | Loss: 2.8935 | LR: 4.87e-04 | 9098.9ms/step | 2.2 samples/s +Step 1890/11700 | Loss: 3.0198 | LR: 4.87e-04 | 9095.1ms/step | 2.2 samples/s +Step 1900/11700 | Loss: 2.9618 | LR: 4.87e-04 | 9094.4ms/step | 2.2 samples/s +Step 1910/11700 | Loss: 2.9861 | LR: 4.86e-04 | 9093.4ms/step | 2.2 samples/s +Step 1920/11700 | Loss: 2.7773 | LR: 4.86e-04 | 9096.8ms/step | 2.2 samples/s +Step 1930/11700 | Loss: 2.9722 | LR: 4.86e-04 | 9095.7ms/step | 2.2 samples/s +Step 1940/11700 | Loss: 2.9024 | LR: 4.86e-04 | 9097.5ms/step | 2.2 samples/s +Step 1950/11700 | Loss: 2.8189 | LR: 4.85e-04 | 9092.7ms/step | 2.2 samples/s +Step 1960/11700 | Loss: 2.8877 | LR: 4.85e-04 | 9096.5ms/step | 2.2 samples/s +Step 1970/11700 | Loss: 2.9145 | LR: 4.85e-04 | 9097.3ms/step | 2.2 samples/s +Step 1980/11700 | Loss: 2.8949 | LR: 4.85e-04 | 9092.3ms/step | 2.2 samples/s +Step 1990/11700 | Loss: 2.8350 | LR: 4.84e-04 | 9092.3ms/step | 2.2 samples/s +Step 2000/11700 | Loss: 3.0469 | LR: 4.84e-04 | 9093.2ms/step | 2.2 samples/s +Validation at step 2000: Loss = 2.9196, Perplexity = 18.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-2000 +Step 2010/11700 | Loss: 2.9461 | LR: 4.84e-04 | 22488.6ms/step | 0.9 samples/s +Step 2020/11700 | Loss: 2.9795 | LR: 4.84e-04 | 9092.4ms/step | 2.2 samples/s +Step 2030/11700 | Loss: 2.8785 | LR: 4.83e-04 | 9091.9ms/step | 2.2 samples/s +Step 2040/11700 | Loss: 2.9193 | LR: 4.83e-04 | 9089.1ms/step | 2.2 samples/s +Step 2050/11700 | Loss: 2.7876 | LR: 4.83e-04 | 9092.7ms/step | 2.2 samples/s +Step 2060/11700 | Loss: 2.8521 | LR: 4.83e-04 | 9094.2ms/step | 2.2 samples/s +Step 2070/11700 | Loss: 2.8590 | LR: 4.82e-04 | 9094.5ms/step | 2.2 samples/s +Step 2080/11700 | Loss: 2.8388 | LR: 4.82e-04 | 9091.3ms/step | 2.2 samples/s +Step 2090/11700 | Loss: 2.8704 | LR: 4.82e-04 | 9092.5ms/step | 2.2 samples/s +Step 2100/11700 | Loss: 2.9612 | LR: 4.81e-04 | 9093.1ms/step | 2.2 samples/s +Step 2110/11700 | Loss: 2.9572 | LR: 4.81e-04 | 9093.9ms/step | 2.2 samples/s +Step 2120/11700 | Loss: 2.9037 | LR: 4.81e-04 | 9086.1ms/step | 2.2 samples/s +Step 2130/11700 | Loss: 2.8931 | LR: 4.81e-04 | 9083.2ms/step | 2.2 samples/s +Step 2140/11700 | Loss: 3.0030 | LR: 4.80e-04 | 9086.1ms/step | 2.2 samples/s +Step 2150/11700 | Loss: 2.9265 | LR: 4.80e-04 | 9085.2ms/step | 2.2 samples/s +Step 2160/11700 | Loss: 2.8807 | LR: 4.80e-04 | 9086.9ms/step | 2.2 samples/s +Step 2170/11700 | Loss: 2.8106 | LR: 4.80e-04 | 9086.8ms/step | 2.2 samples/s +Step 2180/11700 | Loss: 2.8408 | LR: 4.79e-04 | 9084.0ms/step | 2.2 samples/s +Step 2190/11700 | Loss: 2.8298 | LR: 4.79e-04 | 9085.6ms/step | 2.2 samples/s +Step 2200/11700 | Loss: 2.8936 | LR: 4.79e-04 | 9092.2ms/step | 2.2 samples/s +Step 2210/11700 | Loss: 2.9368 | LR: 4.78e-04 | 9087.8ms/step | 2.2 samples/s +Step 2220/11700 | Loss: 2.7463 | LR: 4.78e-04 | 9087.6ms/step | 2.2 samples/s +Step 2230/11700 | Loss: 2.7851 | LR: 4.78e-04 | 9086.3ms/step | 2.2 samples/s +Step 2240/11700 | Loss: 2.7685 | LR: 4.78e-04 | 9083.0ms/step | 2.2 samples/s +Step 2250/11700 | Loss: 2.9539 | LR: 4.77e-04 | 9085.8ms/step | 2.2 samples/s +Validation at step 2250: Loss = 2.8584, Perplexity = 17.43 +Step 2260/11700 | Loss: 2.7345 | LR: 4.77e-04 | 22187.9ms/step | 0.9 samples/s +Step 2270/11700 | Loss: 2.9457 | LR: 4.77e-04 | 9085.0ms/step | 2.2 samples/s +Step 2280/11700 | Loss: 2.8663 | LR: 4.76e-04 | 9086.0ms/step | 2.2 samples/s +Step 2290/11700 | Loss: 2.8816 | LR: 4.76e-04 | 9086.2ms/step | 2.2 samples/s +Step 2300/11700 | Loss: 2.8199 | LR: 4.76e-04 | 9087.4ms/step | 2.2 samples/s +Step 2310/11700 | Loss: 2.8055 | LR: 4.75e-04 | 9089.1ms/step | 2.2 samples/s +Step 2320/11700 | Loss: 2.9572 | LR: 4.75e-04 | 9084.3ms/step | 2.2 samples/s +Step 2330/11700 | Loss: 2.7812 | LR: 4.75e-04 | 9086.9ms/step | 2.2 samples/s +Step 2340/11700 | Loss: 2.8155 | LR: 4.74e-04 | 9082.2ms/step | 2.2 samples/s +Step 2350/11700 | Loss: 2.7372 | LR: 4.74e-04 | 9088.3ms/step | 2.2 samples/s +Step 2360/11700 | Loss: 2.7525 | LR: 4.74e-04 | 9084.7ms/step | 2.2 samples/s +Step 2370/11700 | Loss: 2.7360 | LR: 4.73e-04 | 9089.1ms/step | 2.2 samples/s +Step 2380/11700 | Loss: 2.7325 | LR: 4.73e-04 | 9093.9ms/step | 2.2 samples/s +Step 2390/11700 | Loss: 2.8781 | LR: 4.73e-04 | 9088.8ms/step | 2.2 samples/s +Step 2400/11700 | Loss: 2.8137 | LR: 4.73e-04 | 9087.2ms/step | 2.2 samples/s +Step 2410/11700 | Loss: 2.8326 | LR: 4.72e-04 | 9086.2ms/step | 2.2 samples/s +Step 2420/11700 | Loss: 2.6931 | LR: 4.72e-04 | 9086.2ms/step | 2.2 samples/s +Step 2430/11700 | Loss: 2.8656 | LR: 4.72e-04 | 9087.9ms/step | 2.2 samples/s +Step 2440/11700 | Loss: 2.8840 | LR: 4.71e-04 | 9091.2ms/step | 2.2 samples/s +Step 2450/11700 | Loss: 2.7161 | LR: 4.71e-04 | 9123.6ms/step | 2.2 samples/s +Step 2460/11700 | Loss: 2.8401 | LR: 4.71e-04 | 9108.4ms/step | 2.2 samples/s +Step 2470/11700 | Loss: 2.8459 | LR: 4.70e-04 | 9085.3ms/step | 2.2 samples/s +Step 2480/11700 | Loss: 2.7759 | LR: 4.70e-04 | 9083.3ms/step | 2.2 samples/s +Step 2490/11700 | Loss: 2.8701 | LR: 4.69e-04 | 9080.6ms/step | 2.2 samples/s +Step 2500/11700 | Loss: 2.7407 | LR: 4.69e-04 | 9082.1ms/step | 2.2 samples/s +Validation at step 2500: Loss = 2.8055, Perplexity = 16.54 +Step 2510/11700 | Loss: 2.7124 | LR: 4.69e-04 | 22211.7ms/step | 0.9 samples/s +Step 2520/11700 | Loss: 2.8711 | LR: 4.68e-04 | 9154.5ms/step | 2.2 samples/s +Step 2530/11700 | Loss: 2.7694 | LR: 4.68e-04 | 9148.9ms/step | 2.2 samples/s +Step 2540/11700 | Loss: 2.7032 | LR: 4.68e-04 | 9155.0ms/step | 2.2 samples/s +Step 2550/11700 | Loss: 2.8745 | LR: 4.67e-04 | 9143.3ms/step | 2.2 samples/s +Step 2560/11700 | Loss: 2.8180 | LR: 4.67e-04 | 9147.7ms/step | 2.2 samples/s +Step 2570/11700 | Loss: 2.7768 | LR: 4.67e-04 | 9143.9ms/step | 2.2 samples/s +Step 2580/11700 | Loss: 2.7983 | LR: 4.66e-04 | 9148.5ms/step | 2.2 samples/s +Step 2590/11700 | Loss: 2.7380 | LR: 4.66e-04 | 9147.0ms/step | 2.2 samples/s +Step 2600/11700 | Loss: 2.7674 | LR: 4.66e-04 | 9145.7ms/step | 2.2 samples/s +Step 2610/11700 | Loss: 2.6930 | LR: 4.65e-04 | 9140.9ms/step | 2.2 samples/s +Step 2620/11700 | Loss: 2.7529 | LR: 4.65e-04 | 9141.6ms/step | 2.2 samples/s +Step 2630/11700 | Loss: 2.7013 | LR: 4.65e-04 | 9142.7ms/step | 2.2 samples/s +Step 2640/11700 | Loss: 2.8062 | LR: 4.64e-04 | 9143.4ms/step | 2.2 samples/s +Step 2650/11700 | Loss: 2.8674 | LR: 4.64e-04 | 9141.0ms/step | 2.2 samples/s +Step 2660/11700 | Loss: 2.7997 | LR: 4.63e-04 | 9141.2ms/step | 2.2 samples/s +Step 2670/11700 | Loss: 2.7375 | LR: 4.63e-04 | 9142.8ms/step | 2.2 samples/s +Step 2680/11700 | Loss: 2.7835 | LR: 4.63e-04 | 9143.7ms/step | 2.2 samples/s +Step 2690/11700 | Loss: 2.8873 | LR: 4.62e-04 | 9143.4ms/step | 2.2 samples/s +Step 2700/11700 | Loss: 2.7802 | LR: 4.62e-04 | 9143.9ms/step | 2.2 samples/s +Step 2710/11700 | Loss: 2.7743 | LR: 4.62e-04 | 9144.8ms/step | 2.2 samples/s +Step 2720/11700 | Loss: 2.7854 | LR: 4.61e-04 | 9142.1ms/step | 2.2 samples/s +Step 2730/11700 | Loss: 2.6874 | LR: 4.61e-04 | 9147.5ms/step | 2.2 samples/s +Step 2740/11700 | Loss: 2.7754 | LR: 4.60e-04 | 9144.8ms/step | 2.2 samples/s +Step 2750/11700 | Loss: 2.8248 | LR: 4.60e-04 | 9149.3ms/step | 2.2 samples/s +Validation at step 2750: Loss = 2.7661, Perplexity = 15.90 +Step 2760/11700 | Loss: 2.5761 | LR: 4.60e-04 | 22340.4ms/step | 0.9 samples/s +Step 2770/11700 | Loss: 2.7342 | LR: 4.59e-04 | 9145.3ms/step | 2.2 samples/s +Step 2780/11700 | Loss: 2.7355 | LR: 4.59e-04 | 9144.4ms/step | 2.2 samples/s +Step 2790/11700 | Loss: 2.7813 | LR: 4.58e-04 | 9145.2ms/step | 2.2 samples/s +Step 2800/11700 | Loss: 2.6912 | LR: 4.58e-04 | 9142.6ms/step | 2.2 samples/s +Step 2810/11700 | Loss: 2.7902 | LR: 4.58e-04 | 9143.8ms/step | 2.2 samples/s +Step 2820/11700 | Loss: 2.7803 | LR: 4.57e-04 | 9144.7ms/step | 2.2 samples/s +Step 2830/11700 | Loss: 2.7456 | LR: 4.57e-04 | 9146.3ms/step | 2.2 samples/s +Step 2840/11700 | Loss: 2.8269 | LR: 4.56e-04 | 9145.1ms/step | 2.2 samples/s +Step 2850/11700 | Loss: 2.8575 | LR: 4.56e-04 | 9142.7ms/step | 2.2 samples/s +Step 2860/11700 | Loss: 2.7335 | LR: 4.56e-04 | 9148.2ms/step | 2.2 samples/s +Step 2870/11700 | Loss: 2.8067 | LR: 4.55e-04 | 9147.9ms/step | 2.2 samples/s +Step 2880/11700 | Loss: 2.5400 | LR: 4.55e-04 | 9147.9ms/step | 2.2 samples/s +Step 2890/11700 | Loss: 2.7923 | LR: 4.54e-04 | 9147.3ms/step | 2.2 samples/s +Step 2900/11700 | Loss: 2.7327 | LR: 4.54e-04 | 9147.8ms/step | 2.2 samples/s +Step 2910/11700 | Loss: 2.8040 | LR: 4.54e-04 | 9147.3ms/step | 2.2 samples/s +Step 2920/11700 | Loss: 2.7339 | LR: 4.53e-04 | 9147.7ms/step | 2.2 samples/s +Step 2930/11700 | Loss: 2.8135 | LR: 4.53e-04 | 9144.5ms/step | 2.2 samples/s +Step 2940/11700 | Loss: 2.7454 | LR: 4.52e-04 | 9152.0ms/step | 2.2 samples/s +Step 2950/11700 | Loss: 2.7969 | LR: 4.52e-04 | 9152.4ms/step | 2.2 samples/s +Step 2960/11700 | Loss: 2.6901 | LR: 4.51e-04 | 9151.7ms/step | 2.2 samples/s +Step 2970/11700 | Loss: 2.6393 | LR: 4.51e-04 | 9146.9ms/step | 2.2 samples/s +Step 2980/11700 | Loss: 2.7212 | LR: 4.51e-04 | 9149.6ms/step | 2.2 samples/s +Step 2990/11700 | Loss: 2.6585 | LR: 4.50e-04 | 9149.9ms/step | 2.2 samples/s +Step 3000/11700 | Loss: 2.7389 | LR: 4.50e-04 | 9145.6ms/step | 2.2 samples/s +Validation at step 3000: Loss = 2.7428, Perplexity = 15.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-3000 +Step 3010/11700 | Loss: 2.7068 | LR: 4.49e-04 | 22618.7ms/step | 0.9 samples/s +Step 3020/11700 | Loss: 2.8515 | LR: 4.49e-04 | 9147.8ms/step | 2.2 samples/s +Step 3030/11700 | Loss: 2.6830 | LR: 4.48e-04 | 9149.6ms/step | 2.2 samples/s +Step 3040/11700 | Loss: 2.7908 | LR: 4.48e-04 | 9145.6ms/step | 2.2 samples/s +Step 3050/11700 | Loss: 2.7245 | LR: 4.48e-04 | 9145.2ms/step | 2.2 samples/s +Step 3060/11700 | Loss: 2.9043 | LR: 4.47e-04 | 9148.4ms/step | 2.2 samples/s +Step 3070/11700 | Loss: 2.6796 | LR: 4.47e-04 | 9149.0ms/step | 2.2 samples/s +Step 3080/11700 | Loss: 2.6799 | LR: 4.46e-04 | 9146.6ms/step | 2.2 samples/s +Step 3090/11700 | Loss: 2.6879 | LR: 4.46e-04 | 9148.9ms/step | 2.2 samples/s +Step 3100/11700 | Loss: 2.8214 | LR: 4.45e-04 | 9149.2ms/step | 2.2 samples/s +Step 3110/11700 | Loss: 2.6042 | LR: 4.45e-04 | 9153.4ms/step | 2.2 samples/s +Step 3120/11700 | Loss: 2.6752 | LR: 4.44e-04 | 9147.7ms/step | 2.2 samples/s +Step 3130/11700 | Loss: 2.7148 | LR: 4.44e-04 | 9155.7ms/step | 2.2 samples/s +Step 3140/11700 | Loss: 2.6663 | LR: 4.43e-04 | 9151.3ms/step | 2.2 samples/s +Step 3150/11700 | Loss: 2.6844 | LR: 4.43e-04 | 9150.6ms/step | 2.2 samples/s +Step 3160/11700 | Loss: 2.6851 | LR: 4.43e-04 | 9151.3ms/step | 2.2 samples/s +Step 3170/11700 | Loss: 2.7741 | LR: 4.42e-04 | 9141.1ms/step | 2.2 samples/s +Step 3180/11700 | Loss: 2.7398 | LR: 4.42e-04 | 9144.5ms/step | 2.2 samples/s +Step 3190/11700 | Loss: 2.6135 | LR: 4.41e-04 | 9142.9ms/step | 2.2 samples/s +Step 3200/11700 | Loss: 2.6428 | LR: 4.41e-04 | 9145.1ms/step | 2.2 samples/s +Step 3210/11700 | Loss: 2.6670 | LR: 4.40e-04 | 9142.9ms/step | 2.2 samples/s +Step 3220/11700 | Loss: 2.6909 | LR: 4.40e-04 | 9145.3ms/step | 2.2 samples/s +Step 3230/11700 | Loss: 2.6376 | LR: 4.39e-04 | 9143.7ms/step | 2.2 samples/s +Step 3240/11700 | Loss: 2.7031 | LR: 4.39e-04 | 9144.2ms/step | 2.2 samples/s +Step 3250/11700 | Loss: 2.8316 | LR: 4.38e-04 | 9144.3ms/step | 2.2 samples/s +Validation at step 3250: Loss = 2.7364, Perplexity = 15.43 +Step 3260/11700 | Loss: 2.7379 | LR: 4.38e-04 | 22347.0ms/step | 0.9 samples/s +Step 3270/11700 | Loss: 2.6646 | LR: 4.37e-04 | 9148.0ms/step | 2.2 samples/s +Step 3280/11700 | Loss: 2.7152 | LR: 4.37e-04 | 9146.4ms/step | 2.2 samples/s +Step 3290/11700 | Loss: 2.6564 | LR: 4.37e-04 | 9143.2ms/step | 2.2 samples/s +Step 3300/11700 | Loss: 2.7192 | LR: 4.36e-04 | 9143.9ms/step | 2.2 samples/s +Step 3310/11700 | Loss: 2.7195 | LR: 4.36e-04 | 9146.9ms/step | 2.2 samples/s +Step 3320/11700 | Loss: 2.6384 | LR: 4.35e-04 | 9147.9ms/step | 2.2 samples/s +Step 3330/11700 | Loss: 2.6238 | LR: 4.35e-04 | 9154.1ms/step | 2.2 samples/s +Step 3340/11700 | Loss: 2.7555 | LR: 4.34e-04 | 9159.8ms/step | 2.2 samples/s +Step 3350/11700 | Loss: 2.6588 | LR: 4.34e-04 | 9152.0ms/step | 2.2 samples/s +Step 3360/11700 | Loss: 2.7379 | LR: 4.33e-04 | 9174.9ms/step | 2.2 samples/s +Step 3370/11700 | Loss: 2.6215 | LR: 4.33e-04 | 9187.1ms/step | 2.2 samples/s +Step 3380/11700 | Loss: 2.6871 | LR: 4.32e-04 | 9213.6ms/step | 2.2 samples/s +Step 3390/11700 | Loss: 2.6531 | LR: 4.32e-04 | 9187.0ms/step | 2.2 samples/s +Step 3400/11700 | Loss: 2.7664 | LR: 4.31e-04 | 9210.0ms/step | 2.2 samples/s +Step 3410/11700 | Loss: 2.7072 | LR: 4.31e-04 | 9206.2ms/step | 2.2 samples/s +Step 3420/11700 | Loss: 2.5366 | LR: 4.30e-04 | 9188.1ms/step | 2.2 samples/s +Step 3430/11700 | Loss: 2.7227 | LR: 4.30e-04 | 9207.2ms/step | 2.2 samples/s +Step 3440/11700 | Loss: 2.5776 | LR: 4.29e-04 | 9213.5ms/step | 2.2 samples/s +Step 3450/11700 | Loss: 2.6877 | LR: 4.29e-04 | 9200.0ms/step | 2.2 samples/s +Step 3460/11700 | Loss: 2.6685 | LR: 4.28e-04 | 9192.9ms/step | 2.2 samples/s +Step 3470/11700 | Loss: 2.5592 | LR: 4.28e-04 | 9214.6ms/step | 2.2 samples/s +Step 3480/11700 | Loss: 2.4814 | LR: 4.27e-04 | 9206.7ms/step | 2.2 samples/s +Step 3490/11700 | Loss: 2.8105 | LR: 4.27e-04 | 9208.5ms/step | 2.2 samples/s +Step 3500/11700 | Loss: 2.7242 | LR: 4.26e-04 | 9217.6ms/step | 2.2 samples/s +Validation at step 3500: Loss = 2.6718, Perplexity = 14.47 +Step 3510/11700 | Loss: 2.7039 | LR: 4.26e-04 | 22433.8ms/step | 0.9 samples/s +Step 3520/11700 | Loss: 2.6942 | LR: 4.25e-04 | 9137.8ms/step | 2.2 samples/s +Step 3530/11700 | Loss: 2.7028 | LR: 4.25e-04 | 9149.7ms/step | 2.2 samples/s +Step 3540/11700 | Loss: 2.7097 | LR: 4.24e-04 | 9145.6ms/step | 2.2 samples/s +Step 3550/11700 | Loss: 2.6274 | LR: 4.24e-04 | 9150.6ms/step | 2.2 samples/s +Step 3560/11700 | Loss: 2.8224 | LR: 4.23e-04 | 9144.4ms/step | 2.2 samples/s +Step 3570/11700 | Loss: 2.6953 | LR: 4.23e-04 | 9149.1ms/step | 2.2 samples/s +Step 3580/11700 | Loss: 2.7066 | LR: 4.22e-04 | 9155.1ms/step | 2.2 samples/s +Step 3590/11700 | Loss: 2.6058 | LR: 4.22e-04 | 9154.7ms/step | 2.2 samples/s +Step 3600/11700 | Loss: 2.6026 | LR: 4.21e-04 | 9154.7ms/step | 2.2 samples/s +Step 3610/11700 | Loss: 2.7301 | LR: 4.20e-04 | 9152.7ms/step | 2.2 samples/s +Step 3620/11700 | Loss: 2.5953 | LR: 4.20e-04 | 9172.3ms/step | 2.2 samples/s +Step 3630/11700 | Loss: 2.6029 | LR: 4.19e-04 | 9178.6ms/step | 2.2 samples/s +Step 3640/11700 | Loss: 2.5528 | LR: 4.19e-04 | 9183.8ms/step | 2.2 samples/s +Step 3650/11700 | Loss: 2.6825 | LR: 4.18e-04 | 9179.0ms/step | 2.2 samples/s +Step 3660/11700 | Loss: 2.6443 | LR: 4.18e-04 | 9175.6ms/step | 2.2 samples/s +Step 3670/11700 | Loss: 2.6032 | LR: 4.17e-04 | 9181.3ms/step | 2.2 samples/s +Step 3680/11700 | Loss: 2.6991 | LR: 4.17e-04 | 9183.1ms/step | 2.2 samples/s +Step 3690/11700 | Loss: 2.6377 | LR: 4.16e-04 | 9180.6ms/step | 2.2 samples/s +Step 3700/11700 | Loss: 2.6789 | LR: 4.16e-04 | 9180.5ms/step | 2.2 samples/s +Step 3710/11700 | Loss: 2.5817 | LR: 4.15e-04 | 9185.4ms/step | 2.2 samples/s +Step 3720/11700 | Loss: 2.5925 | LR: 4.15e-04 | 9181.9ms/step | 2.2 samples/s +Step 3730/11700 | Loss: 2.6544 | LR: 4.14e-04 | 9174.1ms/step | 2.2 samples/s +Step 3740/11700 | Loss: 2.6027 | LR: 4.14e-04 | 9181.6ms/step | 2.2 samples/s +Step 3750/11700 | Loss: 2.6788 | LR: 4.13e-04 | 9183.7ms/step | 2.2 samples/s +Validation at step 3750: Loss = 2.6482, Perplexity = 14.13 +Step 3760/11700 | Loss: 2.6879 | LR: 4.12e-04 | 22419.5ms/step | 0.9 samples/s +Step 3770/11700 | Loss: 2.6824 | LR: 4.12e-04 | 9180.1ms/step | 2.2 samples/s +Step 3780/11700 | Loss: 2.6346 | LR: 4.11e-04 | 9174.3ms/step | 2.2 samples/s +Step 3790/11700 | Loss: 2.7748 | LR: 4.11e-04 | 9174.6ms/step | 2.2 samples/s +Step 3800/11700 | Loss: 2.6658 | LR: 4.10e-04 | 9179.2ms/step | 2.2 samples/s +Step 3810/11700 | Loss: 2.6197 | LR: 4.10e-04 | 9177.4ms/step | 2.2 samples/s +Step 3820/11700 | Loss: 2.6065 | LR: 4.09e-04 | 9183.6ms/step | 2.2 samples/s +Step 3830/11700 | Loss: 2.6302 | LR: 4.09e-04 | 9186.6ms/step | 2.2 samples/s +Step 3840/11700 | Loss: 2.5601 | LR: 4.08e-04 | 9185.5ms/step | 2.2 samples/s +Step 3850/11700 | Loss: 2.6774 | LR: 4.07e-04 | 9178.6ms/step | 2.2 samples/s +Step 3860/11700 | Loss: 2.5934 | LR: 4.07e-04 | 9179.5ms/step | 2.2 samples/s +Step 3870/11700 | Loss: 2.6261 | LR: 4.06e-04 | 9183.3ms/step | 2.2 samples/s +Step 3880/11700 | Loss: 2.5092 | LR: 4.06e-04 | 9190.1ms/step | 2.2 samples/s +Step 3890/11700 | Loss: 2.6865 | LR: 4.05e-04 | 9181.2ms/step | 2.2 samples/s +Step 3900/11700 | Loss: 2.5557 | LR: 4.05e-04 | 9181.2ms/step | 2.2 samples/s +Step 3910/11700 | Loss: 2.6462 | LR: 4.04e-04 | 9187.3ms/step | 2.2 samples/s +Step 3920/11700 | Loss: 2.5508 | LR: 4.04e-04 | 9187.3ms/step | 2.2 samples/s +Step 3930/11700 | Loss: 2.6112 | LR: 4.03e-04 | 9190.3ms/step | 2.2 samples/s +Step 3940/11700 | Loss: 2.7508 | LR: 4.02e-04 | 9187.2ms/step | 2.2 samples/s +Step 3950/11700 | Loss: 2.5759 | LR: 4.02e-04 | 9179.6ms/step | 2.2 samples/s +Step 3960/11700 | Loss: 2.6340 | LR: 4.01e-04 | 9179.3ms/step | 2.2 samples/s +Step 3970/11700 | Loss: 2.6121 | LR: 4.01e-04 | 9179.7ms/step | 2.2 samples/s +Step 3980/11700 | Loss: 2.6706 | LR: 4.00e-04 | 9182.9ms/step | 2.2 samples/s +Step 3990/11700 | Loss: 2.6512 | LR: 4.00e-04 | 9177.5ms/step | 2.2 samples/s +Step 4000/11700 | Loss: 2.7034 | LR: 3.99e-04 | 9185.0ms/step | 2.2 samples/s +Validation at step 4000: Loss = 2.6267, Perplexity = 13.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-4000 +Step 4010/11700 | Loss: 2.5210 | LR: 3.98e-04 | 22710.5ms/step | 0.9 samples/s +Step 4020/11700 | Loss: 2.6185 | LR: 3.98e-04 | 9184.9ms/step | 2.2 samples/s +Step 4030/11700 | Loss: 2.5127 | LR: 3.97e-04 | 9188.5ms/step | 2.2 samples/s +Step 4040/11700 | Loss: 2.6520 | LR: 3.97e-04 | 9191.1ms/step | 2.2 samples/s +Step 4050/11700 | Loss: 2.6454 | LR: 3.96e-04 | 9178.5ms/step | 2.2 samples/s +Step 4060/11700 | Loss: 2.5226 | LR: 3.95e-04 | 9177.4ms/step | 2.2 samples/s +Step 4070/11700 | Loss: 2.7078 | LR: 3.95e-04 | 9189.6ms/step | 2.2 samples/s +Step 4080/11700 | Loss: 2.5944 | LR: 3.94e-04 | 9182.0ms/step | 2.2 samples/s +Step 4090/11700 | Loss: 2.5680 | LR: 3.94e-04 | 9186.4ms/step | 2.2 samples/s +Step 4100/11700 | Loss: 2.6090 | LR: 3.93e-04 | 9181.1ms/step | 2.2 samples/s +Step 4110/11700 | Loss: 2.5944 | LR: 3.93e-04 | 9175.6ms/step | 2.2 samples/s +Step 4120/11700 | Loss: 2.6656 | LR: 3.92e-04 | 9187.2ms/step | 2.2 samples/s +Step 4130/11700 | Loss: 2.5277 | LR: 3.91e-04 | 9163.1ms/step | 2.2 samples/s +Step 4140/11700 | Loss: 2.5843 | LR: 3.91e-04 | 9182.8ms/step | 2.2 samples/s +Step 4150/11700 | Loss: 2.5183 | LR: 3.90e-04 | 9182.2ms/step | 2.2 samples/s +Step 4160/11700 | Loss: 2.5944 | LR: 3.90e-04 | 9178.9ms/step | 2.2 samples/s +Step 4170/11700 | Loss: 2.6527 | LR: 3.89e-04 | 9181.1ms/step | 2.2 samples/s +Step 4180/11700 | Loss: 2.5931 | LR: 3.88e-04 | 9181.5ms/step | 2.2 samples/s +Step 4190/11700 | Loss: 2.6259 | LR: 3.88e-04 | 9182.0ms/step | 2.2 samples/s +Step 4200/11700 | Loss: 2.5414 | LR: 3.87e-04 | 9147.4ms/step | 2.2 samples/s +Step 4210/11700 | Loss: 2.6445 | LR: 3.87e-04 | 9161.8ms/step | 2.2 samples/s +Step 4220/11700 | Loss: 2.6078 | LR: 3.86e-04 | 9176.1ms/step | 2.2 samples/s +Step 4230/11700 | Loss: 2.5248 | LR: 3.85e-04 | 9182.6ms/step | 2.2 samples/s +Step 4240/11700 | Loss: 2.6356 | LR: 3.85e-04 | 9105.1ms/step | 2.2 samples/s +Step 4250/11700 | Loss: 2.7374 | LR: 3.84e-04 | 9091.7ms/step | 2.2 samples/s +Validation at step 4250: Loss = 2.6084, Perplexity = 13.58 +Step 4260/11700 | Loss: 2.7040 | LR: 3.84e-04 | 22210.4ms/step | 0.9 samples/s +Step 4270/11700 | Loss: 2.5536 | LR: 3.83e-04 | 9097.8ms/step | 2.2 samples/s +Step 4280/11700 | Loss: 2.5853 | LR: 3.82e-04 | 9098.5ms/step | 2.2 samples/s +Step 4290/11700 | Loss: 2.5905 | LR: 3.82e-04 | 9109.1ms/step | 2.2 samples/s +Step 4300/11700 | Loss: 2.6446 | LR: 3.81e-04 | 9094.8ms/step | 2.2 samples/s +Step 4310/11700 | Loss: 2.5708 | LR: 3.81e-04 | 9090.3ms/step | 2.2 samples/s +Step 4320/11700 | Loss: 2.5501 | LR: 3.80e-04 | 9091.2ms/step | 2.2 samples/s +Step 4330/11700 | Loss: 2.5576 | LR: 3.79e-04 | 9088.6ms/step | 2.2 samples/s +Step 4340/11700 | Loss: 2.6276 | LR: 3.79e-04 | 9091.7ms/step | 2.2 samples/s +Step 4350/11700 | Loss: 2.7014 | LR: 3.78e-04 | 9097.6ms/step | 2.2 samples/s +Step 4360/11700 | Loss: 2.5363 | LR: 3.77e-04 | 9104.7ms/step | 2.2 samples/s +Step 4370/11700 | Loss: 2.5719 | LR: 3.77e-04 | 9099.0ms/step | 2.2 samples/s +Step 4380/11700 | Loss: 2.5311 | LR: 3.76e-04 | 9094.4ms/step | 2.2 samples/s +Step 4390/11700 | Loss: 2.6035 | LR: 3.76e-04 | 9098.4ms/step | 2.2 samples/s +Step 4400/11700 | Loss: 2.5941 | LR: 3.75e-04 | 9099.8ms/step | 2.2 samples/s +Step 4410/11700 | Loss: 2.6133 | LR: 3.74e-04 | 9100.1ms/step | 2.2 samples/s +Step 4420/11700 | Loss: 2.6019 | LR: 3.74e-04 | 9100.1ms/step | 2.2 samples/s +Step 4430/11700 | Loss: 2.6283 | LR: 3.73e-04 | 9095.8ms/step | 2.2 samples/s +Step 4440/11700 | Loss: 2.6294 | LR: 3.73e-04 | 9094.4ms/step | 2.2 samples/s +Step 4450/11700 | Loss: 2.5654 | LR: 3.72e-04 | 9087.1ms/step | 2.2 samples/s +Step 4460/11700 | Loss: 2.6786 | LR: 3.71e-04 | 9089.1ms/step | 2.2 samples/s +Step 4470/11700 | Loss: 2.4859 | LR: 3.71e-04 | 9087.2ms/step | 2.2 samples/s +Step 4480/11700 | Loss: 2.5625 | LR: 3.70e-04 | 9090.8ms/step | 2.2 samples/s +Step 4490/11700 | Loss: 2.5761 | LR: 3.69e-04 | 9086.7ms/step | 2.2 samples/s +Step 4500/11700 | Loss: 2.5913 | LR: 3.69e-04 | 9086.2ms/step | 2.2 samples/s +Validation at step 4500: Loss = 2.5905, Perplexity = 13.34 +Step 4510/11700 | Loss: 2.4838 | LR: 3.68e-04 | 22193.3ms/step | 0.9 samples/s +Step 4520/11700 | Loss: 2.6065 | LR: 3.67e-04 | 9088.9ms/step | 2.2 samples/s +Step 4530/11700 | Loss: 2.7042 | LR: 3.67e-04 | 9088.6ms/step | 2.2 samples/s +Step 4540/11700 | Loss: 2.6265 | LR: 3.66e-04 | 9097.4ms/step | 2.2 samples/s +Step 4550/11700 | Loss: 2.5595 | LR: 3.66e-04 | 9085.1ms/step | 2.2 samples/s +Step 4560/11700 | Loss: 2.5309 | LR: 3.65e-04 | 9085.8ms/step | 2.2 samples/s +Step 4570/11700 | Loss: 2.5565 | LR: 3.64e-04 | 9091.1ms/step | 2.2 samples/s +Step 4580/11700 | Loss: 2.5183 | LR: 3.64e-04 | 9087.7ms/step | 2.2 samples/s +Step 4590/11700 | Loss: 2.6746 | LR: 3.63e-04 | 9088.0ms/step | 2.2 samples/s +Step 4600/11700 | Loss: 2.4194 | LR: 3.62e-04 | 9093.4ms/step | 2.2 samples/s +Step 4610/11700 | Loss: 2.4734 | LR: 3.62e-04 | 9099.5ms/step | 2.2 samples/s +Step 4620/11700 | Loss: 2.5818 | LR: 3.61e-04 | 9086.5ms/step | 2.2 samples/s +Step 4630/11700 | Loss: 2.5878 | LR: 3.60e-04 | 9101.8ms/step | 2.2 samples/s +Step 4640/11700 | Loss: 2.5371 | LR: 3.60e-04 | 9086.3ms/step | 2.2 samples/s +Step 4650/11700 | Loss: 2.5379 | LR: 3.59e-04 | 9087.5ms/step | 2.2 samples/s +Step 4660/11700 | Loss: 2.6097 | LR: 3.59e-04 | 9089.8ms/step | 2.2 samples/s +Step 4670/11700 | Loss: 2.5240 | LR: 3.58e-04 | 9087.5ms/step | 2.2 samples/s +Step 4680/11700 | Loss: 2.6091 | LR: 3.57e-04 | 9085.7ms/step | 2.2 samples/s +Step 4690/11700 | Loss: 2.5086 | LR: 3.57e-04 | 9087.1ms/step | 2.2 samples/s +Step 4700/11700 | Loss: 2.5090 | LR: 3.56e-04 | 9083.1ms/step | 2.2 samples/s +Step 4710/11700 | Loss: 2.6718 | LR: 3.55e-04 | 9086.3ms/step | 2.2 samples/s +Step 4720/11700 | Loss: 2.5350 | LR: 3.55e-04 | 9108.2ms/step | 2.2 samples/s +Step 4730/11700 | Loss: 2.5965 | LR: 3.54e-04 | 9115.2ms/step | 2.2 samples/s +Step 4740/11700 | Loss: 2.6134 | LR: 3.53e-04 | 9115.1ms/step | 2.2 samples/s +Step 4750/11700 | Loss: 2.5538 | LR: 3.53e-04 | 9116.7ms/step | 2.2 samples/s +Validation at step 4750: Loss = 2.5700, Perplexity = 13.07 +Step 4760/11700 | Loss: 2.6443 | LR: 3.52e-04 | 22301.6ms/step | 0.9 samples/s +Step 4770/11700 | Loss: 2.6017 | LR: 3.51e-04 | 9123.8ms/step | 2.2 samples/s +Step 4780/11700 | Loss: 2.5689 | LR: 3.51e-04 | 9125.8ms/step | 2.2 samples/s +Step 4790/11700 | Loss: 2.5420 | LR: 3.50e-04 | 9125.8ms/step | 2.2 samples/s +Step 4800/11700 | Loss: 2.4844 | LR: 3.49e-04 | 9123.6ms/step | 2.2 samples/s +Step 4810/11700 | Loss: 2.4562 | LR: 3.49e-04 | 9124.6ms/step | 2.2 samples/s +Step 4820/11700 | Loss: 2.5596 | LR: 3.48e-04 | 9124.3ms/step | 2.2 samples/s +Step 4830/11700 | Loss: 2.5792 | LR: 3.47e-04 | 9124.6ms/step | 2.2 samples/s +Step 4840/11700 | Loss: 2.5768 | LR: 3.47e-04 | 9121.6ms/step | 2.2 samples/s +Step 4850/11700 | Loss: 2.5850 | LR: 3.46e-04 | 9122.8ms/step | 2.2 samples/s +Step 4860/11700 | Loss: 2.5276 | LR: 3.45e-04 | 9122.8ms/step | 2.2 samples/s +Step 4870/11700 | Loss: 2.6563 | LR: 3.45e-04 | 9123.2ms/step | 2.2 samples/s +Step 4880/11700 | Loss: 2.5872 | LR: 3.44e-04 | 9129.5ms/step | 2.2 samples/s +Step 4890/11700 | Loss: 2.6480 | LR: 3.43e-04 | 9125.1ms/step | 2.2 samples/s +Step 4900/11700 | Loss: 2.5856 | LR: 3.43e-04 | 9121.0ms/step | 2.2 samples/s +Step 4910/11700 | Loss: 2.5337 | LR: 3.42e-04 | 9121.0ms/step | 2.2 samples/s +Step 4920/11700 | Loss: 2.5076 | LR: 3.41e-04 | 9123.2ms/step | 2.2 samples/s +Step 4930/11700 | Loss: 2.4733 | LR: 3.41e-04 | 9127.2ms/step | 2.2 samples/s +Step 4940/11700 | Loss: 2.5986 | LR: 3.40e-04 | 9123.7ms/step | 2.2 samples/s +Step 4950/11700 | Loss: 2.4187 | LR: 3.39e-04 | 9107.5ms/step | 2.2 samples/s +Step 4960/11700 | Loss: 2.5819 | LR: 3.39e-04 | 9120.8ms/step | 2.2 samples/s +Step 4970/11700 | Loss: 2.5678 | LR: 3.38e-04 | 9122.9ms/step | 2.2 samples/s +Step 4980/11700 | Loss: 2.6756 | LR: 3.37e-04 | 9126.2ms/step | 2.2 samples/s +Step 4990/11700 | Loss: 2.4945 | LR: 3.37e-04 | 9124.5ms/step | 2.2 samples/s +Step 5000/11700 | Loss: 2.6561 | LR: 3.36e-04 | 9124.5ms/step | 2.2 samples/s +Validation at step 5000: Loss = 2.5532, Perplexity = 12.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-5000 +Step 5010/11700 | Loss: 2.6522 | LR: 3.35e-04 | 22609.3ms/step | 0.9 samples/s +Step 5020/11700 | Loss: 2.6356 | LR: 3.35e-04 | 9127.6ms/step | 2.2 samples/s +Step 5030/11700 | Loss: 2.6305 | LR: 3.34e-04 | 9133.8ms/step | 2.2 samples/s +Step 5040/11700 | Loss: 2.4645 | LR: 3.33e-04 | 9131.4ms/step | 2.2 samples/s +Step 5050/11700 | Loss: 2.4033 | LR: 3.33e-04 | 9139.6ms/step | 2.2 samples/s +Step 5060/11700 | Loss: 2.5739 | LR: 3.32e-04 | 9145.8ms/step | 2.2 samples/s +Step 5070/11700 | Loss: 2.5333 | LR: 3.31e-04 | 9141.6ms/step | 2.2 samples/s +Step 5080/11700 | Loss: 2.6661 | LR: 3.31e-04 | 9135.2ms/step | 2.2 samples/s +Step 5090/11700 | Loss: 2.5662 | LR: 3.30e-04 | 9131.1ms/step | 2.2 samples/s +Step 5100/11700 | Loss: 2.6633 | LR: 3.29e-04 | 9124.5ms/step | 2.2 samples/s +Step 5110/11700 | Loss: 2.5641 | LR: 3.29e-04 | 9111.3ms/step | 2.2 samples/s +Step 5120/11700 | Loss: 2.5140 | LR: 3.28e-04 | 9112.7ms/step | 2.2 samples/s +Step 5130/11700 | Loss: 2.4283 | LR: 3.27e-04 | 9114.7ms/step | 2.2 samples/s +Step 5140/11700 | Loss: 2.4814 | LR: 3.27e-04 | 9121.6ms/step | 2.2 samples/s +Step 5150/11700 | Loss: 2.5493 | LR: 3.26e-04 | 9142.0ms/step | 2.2 samples/s +Step 5160/11700 | Loss: 2.4954 | LR: 3.25e-04 | 9138.3ms/step | 2.2 samples/s +Step 5170/11700 | Loss: 2.5028 | LR: 3.25e-04 | 9138.4ms/step | 2.2 samples/s +Step 5180/11700 | Loss: 2.4840 | LR: 3.24e-04 | 9144.3ms/step | 2.2 samples/s +Step 5190/11700 | Loss: 2.5557 | LR: 3.23e-04 | 9142.2ms/step | 2.2 samples/s +Step 5200/11700 | Loss: 2.3914 | LR: 3.22e-04 | 9146.8ms/step | 2.2 samples/s +Step 5210/11700 | Loss: 2.5091 | LR: 3.22e-04 | 9137.4ms/step | 2.2 samples/s +Step 5220/11700 | Loss: 2.5639 | LR: 3.21e-04 | 9147.2ms/step | 2.2 samples/s +Step 5230/11700 | Loss: 2.6396 | LR: 3.20e-04 | 9147.1ms/step | 2.2 samples/s +Step 5240/11700 | Loss: 2.5145 | LR: 3.20e-04 | 9140.8ms/step | 2.2 samples/s +Step 5250/11700 | Loss: 2.5530 | LR: 3.19e-04 | 9134.5ms/step | 2.2 samples/s +Validation at step 5250: Loss = 2.5364, Perplexity = 12.63 +Step 5260/11700 | Loss: 2.5480 | LR: 3.18e-04 | 22340.4ms/step | 0.9 samples/s +Step 5270/11700 | Loss: 2.5259 | LR: 3.18e-04 | 9141.2ms/step | 2.2 samples/s +Step 5280/11700 | Loss: 2.4820 | LR: 3.17e-04 | 9136.9ms/step | 2.2 samples/s +Step 5290/11700 | Loss: 2.4899 | LR: 3.16e-04 | 9135.1ms/step | 2.2 samples/s +Step 5300/11700 | Loss: 2.5074 | LR: 3.16e-04 | 9145.1ms/step | 2.2 samples/s +Step 5310/11700 | Loss: 2.4973 | LR: 3.15e-04 | 9143.2ms/step | 2.2 samples/s +Step 5320/11700 | Loss: 2.4731 | LR: 3.14e-04 | 9142.1ms/step | 2.2 samples/s +Step 5330/11700 | Loss: 2.4606 | LR: 3.13e-04 | 9144.4ms/step | 2.2 samples/s +Step 5340/11700 | Loss: 2.4969 | LR: 3.13e-04 | 9141.1ms/step | 2.2 samples/s +Step 5350/11700 | Loss: 2.4974 | LR: 3.12e-04 | 9138.0ms/step | 2.2 samples/s +Step 5360/11700 | Loss: 2.4239 | LR: 3.11e-04 | 9143.3ms/step | 2.2 samples/s +Step 5370/11700 | Loss: 2.5292 | LR: 3.11e-04 | 9148.0ms/step | 2.2 samples/s +Step 5380/11700 | Loss: 2.4242 | LR: 3.10e-04 | 9147.3ms/step | 2.2 samples/s +Step 5390/11700 | Loss: 2.5637 | LR: 3.09e-04 | 9145.0ms/step | 2.2 samples/s +Step 5400/11700 | Loss: 2.4731 | LR: 3.09e-04 | 9145.7ms/step | 2.2 samples/s +Step 5410/11700 | Loss: 2.4578 | LR: 3.08e-04 | 9145.7ms/step | 2.2 samples/s +Step 5420/11700 | Loss: 2.5254 | LR: 3.07e-04 | 9148.2ms/step | 2.2 samples/s +Step 5430/11700 | Loss: 2.5997 | LR: 3.07e-04 | 9145.5ms/step | 2.2 samples/s +Step 5440/11700 | Loss: 2.6039 | LR: 3.06e-04 | 9141.0ms/step | 2.2 samples/s +Step 5450/11700 | Loss: 2.4865 | LR: 3.05e-04 | 9143.9ms/step | 2.2 samples/s +Step 5460/11700 | Loss: 2.5275 | LR: 3.04e-04 | 9144.4ms/step | 2.2 samples/s +Step 5470/11700 | Loss: 2.3889 | LR: 3.04e-04 | 9143.7ms/step | 2.2 samples/s +Step 5480/11700 | Loss: 2.4927 | LR: 3.03e-04 | 9146.4ms/step | 2.2 samples/s +Step 5490/11700 | Loss: 2.6596 | LR: 3.02e-04 | 9146.4ms/step | 2.2 samples/s +Step 5500/11700 | Loss: 2.5612 | LR: 3.02e-04 | 9144.6ms/step | 2.2 samples/s +Validation at step 5500: Loss = 2.5212, Perplexity = 12.44 +Step 5510/11700 | Loss: 2.4003 | LR: 3.01e-04 | 22347.8ms/step | 0.9 samples/s +Step 5520/11700 | Loss: 2.3806 | LR: 3.00e-04 | 9147.3ms/step | 2.2 samples/s +Step 5530/11700 | Loss: 2.5384 | LR: 3.00e-04 | 9140.6ms/step | 2.2 samples/s +Step 5540/11700 | Loss: 2.5183 | LR: 2.99e-04 | 9144.0ms/step | 2.2 samples/s +Step 5550/11700 | Loss: 2.5236 | LR: 2.98e-04 | 9146.6ms/step | 2.2 samples/s +Step 5560/11700 | Loss: 2.5641 | LR: 2.97e-04 | 9142.3ms/step | 2.2 samples/s +Step 5570/11700 | Loss: 2.4370 | LR: 2.97e-04 | 9144.8ms/step | 2.2 samples/s +Step 5580/11700 | Loss: 2.5312 | LR: 2.96e-04 | 9146.5ms/step | 2.2 samples/s +Step 5590/11700 | Loss: 2.5564 | LR: 2.95e-04 | 9148.6ms/step | 2.2 samples/s +Step 5600/11700 | Loss: 2.5006 | LR: 2.95e-04 | 9145.2ms/step | 2.2 samples/s +Step 5610/11700 | Loss: 2.3596 | LR: 2.94e-04 | 9146.3ms/step | 2.2 samples/s +Step 5620/11700 | Loss: 2.5198 | LR: 2.93e-04 | 9140.5ms/step | 2.2 samples/s +Step 5630/11700 | Loss: 2.4093 | LR: 2.92e-04 | 9142.6ms/step | 2.2 samples/s +Step 5640/11700 | Loss: 2.5402 | LR: 2.92e-04 | 9146.3ms/step | 2.2 samples/s +Step 5650/11700 | Loss: 2.5929 | LR: 2.91e-04 | 9143.9ms/step | 2.2 samples/s +Step 5660/11700 | Loss: 2.4986 | LR: 2.90e-04 | 9147.2ms/step | 2.2 samples/s +Step 5670/11700 | Loss: 2.4853 | LR: 2.90e-04 | 9146.6ms/step | 2.2 samples/s +Step 5680/11700 | Loss: 2.5581 | LR: 2.89e-04 | 9144.0ms/step | 2.2 samples/s +Step 5690/11700 | Loss: 2.4697 | LR: 2.88e-04 | 9145.1ms/step | 2.2 samples/s +Step 5700/11700 | Loss: 2.4927 | LR: 2.88e-04 | 9145.0ms/step | 2.2 samples/s +Step 5710/11700 | Loss: 2.4671 | LR: 2.87e-04 | 9140.2ms/step | 2.2 samples/s +Step 5720/11700 | Loss: 2.4447 | LR: 2.86e-04 | 9144.8ms/step | 2.2 samples/s +Step 5730/11700 | Loss: 2.5575 | LR: 2.85e-04 | 9148.4ms/step | 2.2 samples/s +Step 5740/11700 | Loss: 2.4266 | LR: 2.85e-04 | 9149.9ms/step | 2.2 samples/s +Step 5750/11700 | Loss: 2.4603 | LR: 2.84e-04 | 9164.8ms/step | 2.2 samples/s +Validation at step 5750: Loss = 2.5089, Perplexity = 12.29 +Step 5760/11700 | Loss: 2.4971 | LR: 2.83e-04 | 22359.4ms/step | 0.9 samples/s +Step 5770/11700 | Loss: 2.6273 | LR: 2.83e-04 | 9163.2ms/step | 2.2 samples/s +Step 5780/11700 | Loss: 2.4672 | LR: 2.82e-04 | 9157.2ms/step | 2.2 samples/s +Step 5790/11700 | Loss: 2.5947 | LR: 2.81e-04 | 9157.8ms/step | 2.2 samples/s +Step 5800/11700 | Loss: 2.5658 | LR: 2.80e-04 | 9143.3ms/step | 2.2 samples/s +Step 5810/11700 | Loss: 2.5349 | LR: 2.80e-04 | 9155.1ms/step | 2.2 samples/s +Step 5820/11700 | Loss: 2.4280 | LR: 2.79e-04 | 9134.7ms/step | 2.2 samples/s +Step 5830/11700 | Loss: 2.4763 | LR: 2.78e-04 | 9144.2ms/step | 2.2 samples/s +Step 5840/11700 | Loss: 2.4669 | LR: 2.78e-04 | 9142.8ms/step | 2.2 samples/s +Step 5850/11700 | Loss: 2.4744 | LR: 2.77e-04 | 9143.0ms/step | 2.2 samples/s +Step 5860/11700 | Loss: 2.5493 | LR: 2.76e-04 | 9145.0ms/step | 2.2 samples/s +Step 5870/11700 | Loss: 2.5632 | LR: 2.75e-04 | 9136.9ms/step | 2.2 samples/s +Step 5880/11700 | Loss: 2.5382 | LR: 2.75e-04 | 9144.8ms/step | 2.2 samples/s +Step 5890/11700 | Loss: 2.4765 | LR: 2.74e-04 | 9143.4ms/step | 2.2 samples/s +Step 5900/11700 | Loss: 2.4887 | LR: 2.73e-04 | 9145.4ms/step | 2.2 samples/s +Step 5910/11700 | Loss: 2.5337 | LR: 2.73e-04 | 9143.6ms/step | 2.2 samples/s +Step 5920/11700 | Loss: 2.4420 | LR: 2.72e-04 | 9144.9ms/step | 2.2 samples/s +Step 5930/11700 | Loss: 2.4724 | LR: 2.71e-04 | 9139.7ms/step | 2.2 samples/s +Step 5940/11700 | Loss: 2.3999 | LR: 2.70e-04 | 9136.9ms/step | 2.2 samples/s +Step 5950/11700 | Loss: 2.5692 | LR: 2.70e-04 | 9145.2ms/step | 2.2 samples/s +Step 5960/11700 | Loss: 2.5918 | LR: 2.69e-04 | 9144.4ms/step | 2.2 samples/s +Step 5970/11700 | Loss: 2.4833 | LR: 2.68e-04 | 9144.8ms/step | 2.2 samples/s +Step 5980/11700 | Loss: 2.5034 | LR: 2.68e-04 | 9141.5ms/step | 2.2 samples/s +Step 5990/11700 | Loss: 2.4954 | LR: 2.67e-04 | 9147.2ms/step | 2.2 samples/s +Step 6000/11700 | Loss: 2.4905 | LR: 2.66e-04 | 9143.5ms/step | 2.2 samples/s +Validation at step 6000: Loss = 2.4989, Perplexity = 12.17 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-6000 +Step 6010/11700 | Loss: 2.5441 | LR: 2.65e-04 | 22572.9ms/step | 0.9 samples/s +Step 6020/11700 | Loss: 2.4072 | LR: 2.65e-04 | 9107.4ms/step | 2.2 samples/s +Step 6030/11700 | Loss: 2.4658 | LR: 2.64e-04 | 9102.4ms/step | 2.2 samples/s +Step 6040/11700 | Loss: 2.4316 | LR: 2.63e-04 | 9107.2ms/step | 2.2 samples/s +Step 6050/11700 | Loss: 2.4332 | LR: 2.63e-04 | 9109.8ms/step | 2.2 samples/s +Step 6060/11700 | Loss: 2.5352 | LR: 2.62e-04 | 9103.8ms/step | 2.2 samples/s +Step 6070/11700 | Loss: 2.5268 | LR: 2.61e-04 | 9086.7ms/step | 2.2 samples/s +Step 6080/11700 | Loss: 2.5599 | LR: 2.60e-04 | 9090.2ms/step | 2.2 samples/s +Step 6090/11700 | Loss: 2.4253 | LR: 2.60e-04 | 9092.1ms/step | 2.2 samples/s +Step 6100/11700 | Loss: 2.4976 | LR: 2.59e-04 | 9087.8ms/step | 2.2 samples/s +Step 6110/11700 | Loss: 2.4302 | LR: 2.58e-04 | 9111.5ms/step | 2.2 samples/s +Step 6120/11700 | Loss: 2.5551 | LR: 2.58e-04 | 9103.2ms/step | 2.2 samples/s +Step 6130/11700 | Loss: 2.4852 | LR: 2.57e-04 | 9088.5ms/step | 2.2 samples/s +Step 6140/11700 | Loss: 2.4975 | LR: 2.56e-04 | 9093.7ms/step | 2.2 samples/s +Step 6150/11700 | Loss: 2.4709 | LR: 2.55e-04 | 9098.2ms/step | 2.2 samples/s +Step 6160/11700 | Loss: 2.4090 | LR: 2.55e-04 | 9091.3ms/step | 2.2 samples/s +Step 6170/11700 | Loss: 2.4188 | LR: 2.54e-04 | 9086.8ms/step | 2.2 samples/s +Step 6180/11700 | Loss: 2.3879 | LR: 2.53e-04 | 9091.1ms/step | 2.2 samples/s +Step 6190/11700 | Loss: 2.4477 | LR: 2.53e-04 | 9104.0ms/step | 2.2 samples/s +Step 6200/11700 | Loss: 2.5750 | LR: 2.52e-04 | 9091.9ms/step | 2.2 samples/s +Step 6210/11700 | Loss: 2.5572 | LR: 2.51e-04 | 9087.3ms/step | 2.2 samples/s +Step 6220/11700 | Loss: 2.5046 | LR: 2.50e-04 | 9079.4ms/step | 2.2 samples/s +Step 6230/11700 | Loss: 2.6126 | LR: 2.50e-04 | 9095.4ms/step | 2.2 samples/s +Step 6240/11700 | Loss: 2.4705 | LR: 2.49e-04 | 9097.9ms/step | 2.2 samples/s +Step 6250/11700 | Loss: 2.5093 | LR: 2.48e-04 | 9086.6ms/step | 2.2 samples/s +Validation at step 6250: Loss = 2.4877, Perplexity = 12.03 +Step 6260/11700 | Loss: 2.4028 | LR: 2.47e-04 | 22194.2ms/step | 0.9 samples/s +Step 6270/11700 | Loss: 2.4997 | LR: 2.47e-04 | 9095.5ms/step | 2.2 samples/s +Step 6280/11700 | Loss: 2.4763 | LR: 2.46e-04 | 9088.6ms/step | 2.2 samples/s +Step 6290/11700 | Loss: 2.4564 | LR: 2.45e-04 | 9086.5ms/step | 2.2 samples/s +Step 6300/11700 | Loss: 2.4645 | LR: 2.45e-04 | 9086.9ms/step | 2.2 samples/s +Step 6310/11700 | Loss: 2.4618 | LR: 2.44e-04 | 9089.0ms/step | 2.2 samples/s +Step 6320/11700 | Loss: 2.5257 | LR: 2.43e-04 | 9090.5ms/step | 2.2 samples/s +Step 6330/11700 | Loss: 2.5899 | LR: 2.42e-04 | 9089.1ms/step | 2.2 samples/s +Step 6340/11700 | Loss: 2.4999 | LR: 2.42e-04 | 9087.3ms/step | 2.2 samples/s +Step 6350/11700 | Loss: 2.5105 | LR: 2.41e-04 | 9092.8ms/step | 2.2 samples/s +Step 6360/11700 | Loss: 2.4084 | LR: 2.40e-04 | 9090.1ms/step | 2.2 samples/s +Step 6370/11700 | Loss: 2.3882 | LR: 2.40e-04 | 9094.9ms/step | 2.2 samples/s +Step 6380/11700 | Loss: 2.3609 | LR: 2.39e-04 | 9087.4ms/step | 2.2 samples/s +Step 6390/11700 | Loss: 2.4928 | LR: 2.38e-04 | 9089.4ms/step | 2.2 samples/s +Step 6400/11700 | Loss: 2.4704 | LR: 2.37e-04 | 9081.8ms/step | 2.2 samples/s +Step 6410/11700 | Loss: 2.3977 | LR: 2.37e-04 | 9085.0ms/step | 2.2 samples/s +Step 6420/11700 | Loss: 2.3963 | LR: 2.36e-04 | 9081.8ms/step | 2.2 samples/s +Step 6430/11700 | Loss: 2.2914 | LR: 2.35e-04 | 9085.5ms/step | 2.2 samples/s +Step 6440/11700 | Loss: 2.4633 | LR: 2.35e-04 | 9084.8ms/step | 2.2 samples/s +Step 6450/11700 | Loss: 2.4043 | LR: 2.34e-04 | 9086.9ms/step | 2.2 samples/s +Step 6460/11700 | Loss: 2.5353 | LR: 2.33e-04 | 9084.0ms/step | 2.2 samples/s +Step 6470/11700 | Loss: 2.3994 | LR: 2.32e-04 | 9086.7ms/step | 2.2 samples/s +Step 6480/11700 | Loss: 2.6502 | LR: 2.32e-04 | 9086.2ms/step | 2.2 samples/s +Step 6490/11700 | Loss: 2.4557 | LR: 2.31e-04 | 9086.7ms/step | 2.2 samples/s +Step 6500/11700 | Loss: 2.4896 | LR: 2.30e-04 | 9089.1ms/step | 2.2 samples/s +Validation at step 6500: Loss = 2.4769, Perplexity = 11.90 +Step 6510/11700 | Loss: 2.4695 | LR: 2.30e-04 | 22226.5ms/step | 0.9 samples/s +Step 6520/11700 | Loss: 2.4668 | LR: 2.29e-04 | 9092.0ms/step | 2.2 samples/s +Step 6530/11700 | Loss: 2.3864 | LR: 2.28e-04 | 9086.2ms/step | 2.2 samples/s +Step 6540/11700 | Loss: 2.5386 | LR: 2.27e-04 | 9084.5ms/step | 2.2 samples/s +Step 6550/11700 | Loss: 2.4390 | LR: 2.27e-04 | 9086.9ms/step | 2.2 samples/s +Step 6560/11700 | Loss: 2.3897 | LR: 2.26e-04 | 9087.6ms/step | 2.2 samples/s +Step 6570/11700 | Loss: 2.3455 | LR: 2.25e-04 | 9090.0ms/step | 2.2 samples/s +Step 6580/11700 | Loss: 2.5081 | LR: 2.25e-04 | 9086.5ms/step | 2.2 samples/s +Step 6590/11700 | Loss: 2.4877 | LR: 2.24e-04 | 9086.6ms/step | 2.2 samples/s +Step 6600/11700 | Loss: 2.4995 | LR: 2.23e-04 | 9085.1ms/step | 2.2 samples/s +Step 6610/11700 | Loss: 2.5723 | LR: 2.22e-04 | 9084.4ms/step | 2.2 samples/s +Step 6620/11700 | Loss: 2.3717 | LR: 2.22e-04 | 9084.0ms/step | 2.2 samples/s +Step 6630/11700 | Loss: 2.4787 | LR: 2.21e-04 | 9088.2ms/step | 2.2 samples/s +Step 6640/11700 | Loss: 2.4856 | LR: 2.20e-04 | 9083.6ms/step | 2.2 samples/s +Step 6650/11700 | Loss: 2.4182 | LR: 2.20e-04 | 9088.5ms/step | 2.2 samples/s +Step 6660/11700 | Loss: 2.4429 | LR: 2.19e-04 | 9083.3ms/step | 2.2 samples/s +Step 6670/11700 | Loss: 2.4375 | LR: 2.18e-04 | 9089.8ms/step | 2.2 samples/s +Step 6680/11700 | Loss: 2.4325 | LR: 2.17e-04 | 9084.9ms/step | 2.2 samples/s +Step 6690/11700 | Loss: 2.5237 | LR: 2.17e-04 | 9086.3ms/step | 2.2 samples/s +Step 6700/11700 | Loss: 2.4622 | LR: 2.16e-04 | 9082.0ms/step | 2.2 samples/s +Step 6710/11700 | Loss: 2.4756 | LR: 2.15e-04 | 9102.0ms/step | 2.2 samples/s +Step 6720/11700 | Loss: 2.3583 | LR: 2.15e-04 | 9086.0ms/step | 2.2 samples/s +Step 6730/11700 | Loss: 2.4169 | LR: 2.14e-04 | 9088.4ms/step | 2.2 samples/s +Step 6740/11700 | Loss: 2.4360 | LR: 2.13e-04 | 9090.0ms/step | 2.2 samples/s +Step 6750/11700 | Loss: 2.4182 | LR: 2.12e-04 | 9089.1ms/step | 2.2 samples/s +Validation at step 6750: Loss = 2.4716, Perplexity = 11.84 +Step 6760/11700 | Loss: 2.4926 | LR: 2.12e-04 | 22201.2ms/step | 0.9 samples/s +Step 6770/11700 | Loss: 2.4568 | LR: 2.11e-04 | 9086.8ms/step | 2.2 samples/s +Step 6780/11700 | Loss: 2.3963 | LR: 2.10e-04 | 9087.4ms/step | 2.2 samples/s +Step 6790/11700 | Loss: 2.4010 | LR: 2.10e-04 | 9095.0ms/step | 2.2 samples/s +Step 6800/11700 | Loss: 2.4824 | LR: 2.09e-04 | 9089.1ms/step | 2.2 samples/s +Step 6810/11700 | Loss: 2.6517 | LR: 2.08e-04 | 9086.0ms/step | 2.2 samples/s +Step 6820/11700 | Loss: 2.4906 | LR: 2.08e-04 | 9081.7ms/step | 2.2 samples/s +Step 6830/11700 | Loss: 2.4017 | LR: 2.07e-04 | 9085.8ms/step | 2.2 samples/s +Step 6840/11700 | Loss: 2.4221 | LR: 2.06e-04 | 9093.4ms/step | 2.2 samples/s +Step 6850/11700 | Loss: 2.4960 | LR: 2.05e-04 | 9090.2ms/step | 2.2 samples/s +Step 6860/11700 | Loss: 2.4352 | LR: 2.05e-04 | 9089.5ms/step | 2.2 samples/s +Step 6870/11700 | Loss: 2.4228 | LR: 2.04e-04 | 9081.3ms/step | 2.2 samples/s +Step 6880/11700 | Loss: 2.4238 | LR: 2.03e-04 | 9085.0ms/step | 2.2 samples/s +Step 6890/11700 | Loss: 2.4464 | LR: 2.03e-04 | 9087.8ms/step | 2.2 samples/s +Step 6900/11700 | Loss: 2.5689 | LR: 2.02e-04 | 9085.3ms/step | 2.2 samples/s +Step 6910/11700 | Loss: 2.5364 | LR: 2.01e-04 | 9084.3ms/step | 2.2 samples/s +Step 6920/11700 | Loss: 2.3927 | LR: 2.00e-04 | 9083.8ms/step | 2.2 samples/s +Step 6930/11700 | Loss: 2.3413 | LR: 2.00e-04 | 9086.5ms/step | 2.2 samples/s +Step 6940/11700 | Loss: 2.5533 | LR: 1.99e-04 | 9087.6ms/step | 2.2 samples/s +Step 6950/11700 | Loss: 2.5008 | LR: 1.98e-04 | 9086.5ms/step | 2.2 samples/s +Step 6960/11700 | Loss: 2.4483 | LR: 1.98e-04 | 9089.1ms/step | 2.2 samples/s +Step 6970/11700 | Loss: 2.2961 | LR: 1.97e-04 | 9089.6ms/step | 2.2 samples/s +Step 6980/11700 | Loss: 2.4580 | LR: 1.96e-04 | 9082.8ms/step | 2.2 samples/s +Step 6990/11700 | Loss: 2.5262 | LR: 1.96e-04 | 9084.6ms/step | 2.2 samples/s +Step 7000/11700 | Loss: 2.4571 | LR: 1.95e-04 | 9089.4ms/step | 2.2 samples/s +Validation at step 7000: Loss = 2.4603, Perplexity = 11.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-7000 +Step 7010/11700 | Loss: 2.5595 | LR: 1.94e-04 | 22535.3ms/step | 0.9 samples/s +Step 7020/11700 | Loss: 2.4402 | LR: 1.93e-04 | 9139.3ms/step | 2.2 samples/s +Step 7030/11700 | Loss: 2.5336 | LR: 1.93e-04 | 9140.2ms/step | 2.2 samples/s +Step 7040/11700 | Loss: 2.5102 | LR: 1.92e-04 | 9143.1ms/step | 2.2 samples/s +Step 7050/11700 | Loss: 2.3820 | LR: 1.91e-04 | 9141.9ms/step | 2.2 samples/s +Step 7060/11700 | Loss: 2.5037 | LR: 1.91e-04 | 9143.0ms/step | 2.2 samples/s +Step 7070/11700 | Loss: 2.4297 | LR: 1.90e-04 | 9143.6ms/step | 2.2 samples/s +Step 7080/11700 | Loss: 2.4468 | LR: 1.89e-04 | 9144.4ms/step | 2.2 samples/s +Step 7090/11700 | Loss: 2.4500 | LR: 1.89e-04 | 9145.8ms/step | 2.2 samples/s +Step 7100/11700 | Loss: 2.4864 | LR: 1.88e-04 | 9142.5ms/step | 2.2 samples/s +Step 7110/11700 | Loss: 2.4266 | LR: 1.87e-04 | 9142.3ms/step | 2.2 samples/s +Step 7120/11700 | Loss: 2.3727 | LR: 1.87e-04 | 9138.2ms/step | 2.2 samples/s +Step 7130/11700 | Loss: 2.3702 | LR: 1.86e-04 | 9145.9ms/step | 2.2 samples/s +Step 7140/11700 | Loss: 2.4294 | LR: 1.85e-04 | 9144.2ms/step | 2.2 samples/s +Step 7150/11700 | Loss: 2.5114 | LR: 1.84e-04 | 9143.2ms/step | 2.2 samples/s +Step 7160/11700 | Loss: 2.4243 | LR: 1.84e-04 | 9146.2ms/step | 2.2 samples/s +Step 7170/11700 | Loss: 2.4631 | LR: 1.83e-04 | 9135.0ms/step | 2.2 samples/s +Step 7180/11700 | Loss: 2.3695 | LR: 1.82e-04 | 9140.4ms/step | 2.2 samples/s +Step 7190/11700 | Loss: 2.3734 | LR: 1.82e-04 | 9140.6ms/step | 2.2 samples/s +Step 7200/11700 | Loss: 2.3384 | LR: 1.81e-04 | 9139.8ms/step | 2.2 samples/s +Step 7210/11700 | Loss: 2.2830 | LR: 1.80e-04 | 9138.4ms/step | 2.2 samples/s +Step 7220/11700 | Loss: 2.4199 | LR: 1.80e-04 | 9143.8ms/step | 2.2 samples/s +Step 7230/11700 | Loss: 2.4641 | LR: 1.79e-04 | 9141.6ms/step | 2.2 samples/s +Step 7240/11700 | Loss: 2.4233 | LR: 1.78e-04 | 9142.7ms/step | 2.2 samples/s +Step 7250/11700 | Loss: 2.2937 | LR: 1.78e-04 | 9145.0ms/step | 2.2 samples/s +Validation at step 7250: Loss = 2.4528, Perplexity = 11.62 +Step 7260/11700 | Loss: 2.4109 | LR: 1.77e-04 | 22333.6ms/step | 0.9 samples/s +Step 7270/11700 | Loss: 2.4652 | LR: 1.76e-04 | 9145.3ms/step | 2.2 samples/s +Step 7280/11700 | Loss: 2.4718 | LR: 1.75e-04 | 9142.2ms/step | 2.2 samples/s +Step 7290/11700 | Loss: 2.4664 | LR: 1.75e-04 | 9140.0ms/step | 2.2 samples/s +Step 7300/11700 | Loss: 2.3871 | LR: 1.74e-04 | 9141.7ms/step | 2.2 samples/s +Step 7310/11700 | Loss: 2.5113 | LR: 1.73e-04 | 9137.1ms/step | 2.2 samples/s +Step 7320/11700 | Loss: 2.4106 | LR: 1.73e-04 | 9143.0ms/step | 2.2 samples/s +Step 7330/11700 | Loss: 2.4799 | LR: 1.72e-04 | 9146.5ms/step | 2.2 samples/s +Step 7340/11700 | Loss: 2.4191 | LR: 1.71e-04 | 9143.0ms/step | 2.2 samples/s +Step 7350/11700 | Loss: 2.4018 | LR: 1.71e-04 | 9146.7ms/step | 2.2 samples/s +Step 7360/11700 | Loss: 2.3563 | LR: 1.70e-04 | 9144.3ms/step | 2.2 samples/s +Step 7370/11700 | Loss: 2.4135 | LR: 1.69e-04 | 9142.6ms/step | 2.2 samples/s +Step 7380/11700 | Loss: 2.4531 | LR: 1.69e-04 | 9144.6ms/step | 2.2 samples/s +Step 7390/11700 | Loss: 2.4200 | LR: 1.68e-04 | 9147.0ms/step | 2.2 samples/s +Step 7400/11700 | Loss: 2.4536 | LR: 1.67e-04 | 9144.0ms/step | 2.2 samples/s +Step 7410/11700 | Loss: 2.3756 | LR: 1.67e-04 | 9149.2ms/step | 2.2 samples/s +Step 7420/11700 | Loss: 2.3599 | LR: 1.66e-04 | 9141.9ms/step | 2.2 samples/s +Step 7430/11700 | Loss: 2.3726 | LR: 1.65e-04 | 9147.2ms/step | 2.2 samples/s +Step 7440/11700 | Loss: 2.4754 | LR: 1.65e-04 | 9143.8ms/step | 2.2 samples/s +Step 7450/11700 | Loss: 2.4553 | LR: 1.64e-04 | 9147.7ms/step | 2.2 samples/s +Step 7460/11700 | Loss: 2.4626 | LR: 1.63e-04 | 9143.4ms/step | 2.2 samples/s +Step 7470/11700 | Loss: 2.4152 | LR: 1.63e-04 | 9140.5ms/step | 2.2 samples/s +Step 7480/11700 | Loss: 2.5465 | LR: 1.62e-04 | 9142.6ms/step | 2.2 samples/s +Step 7490/11700 | Loss: 2.3791 | LR: 1.61e-04 | 9146.4ms/step | 2.2 samples/s +Step 7500/11700 | Loss: 2.3715 | LR: 1.61e-04 | 9145.2ms/step | 2.2 samples/s +Validation at step 7500: Loss = 2.4463, Perplexity = 11.55 +Step 7510/11700 | Loss: 2.4556 | LR: 1.60e-04 | 22336.7ms/step | 0.9 samples/s +Step 7520/11700 | Loss: 2.4793 | LR: 1.59e-04 | 9139.9ms/step | 2.2 samples/s +Step 7530/11700 | Loss: 2.3835 | LR: 1.59e-04 | 9139.9ms/step | 2.2 samples/s +Step 7540/11700 | Loss: 2.4624 | LR: 1.58e-04 | 9129.6ms/step | 2.2 samples/s +Step 7550/11700 | Loss: 2.4081 | LR: 1.57e-04 | 9143.2ms/step | 2.2 samples/s +Step 7560/11700 | Loss: 2.4680 | LR: 1.57e-04 | 9143.4ms/step | 2.2 samples/s +Step 7570/11700 | Loss: 2.4275 | LR: 1.56e-04 | 9142.9ms/step | 2.2 samples/s +Step 7580/11700 | Loss: 2.4140 | LR: 1.55e-04 | 9146.5ms/step | 2.2 samples/s +Step 7590/11700 | Loss: 2.3957 | LR: 1.55e-04 | 9144.1ms/step | 2.2 samples/s +Step 7600/11700 | Loss: 2.4526 | LR: 1.54e-04 | 9143.8ms/step | 2.2 samples/s +Step 7610/11700 | Loss: 2.5479 | LR: 1.53e-04 | 9148.2ms/step | 2.2 samples/s +Step 7620/11700 | Loss: 2.5143 | LR: 1.53e-04 | 9145.1ms/step | 2.2 samples/s +Step 7630/11700 | Loss: 2.3889 | LR: 1.52e-04 | 9143.9ms/step | 2.2 samples/s +Step 7640/11700 | Loss: 2.4230 | LR: 1.51e-04 | 9144.7ms/step | 2.2 samples/s +Step 7650/11700 | Loss: 2.4599 | LR: 1.51e-04 | 9138.4ms/step | 2.2 samples/s +Step 7660/11700 | Loss: 2.4075 | LR: 1.50e-04 | 9142.3ms/step | 2.2 samples/s +Step 7670/11700 | Loss: 2.3434 | LR: 1.49e-04 | 9145.0ms/step | 2.2 samples/s +Step 7680/11700 | Loss: 2.4047 | LR: 1.49e-04 | 9139.5ms/step | 2.2 samples/s +Step 7690/11700 | Loss: 2.4854 | LR: 1.48e-04 | 9107.0ms/step | 2.2 samples/s +Step 7700/11700 | Loss: 2.5507 | LR: 1.47e-04 | 9125.5ms/step | 2.2 samples/s +Step 7710/11700 | Loss: 2.3507 | LR: 1.47e-04 | 9109.7ms/step | 2.2 samples/s +Step 7720/11700 | Loss: 2.4551 | LR: 1.46e-04 | 9118.9ms/step | 2.2 samples/s +Step 7730/11700 | Loss: 2.3479 | LR: 1.45e-04 | 9115.3ms/step | 2.2 samples/s +Step 7740/11700 | Loss: 2.5090 | LR: 1.45e-04 | 9105.5ms/step | 2.2 samples/s +Step 7750/11700 | Loss: 2.4241 | LR: 1.44e-04 | 9106.0ms/step | 2.2 samples/s +Validation at step 7750: Loss = 2.4400, Perplexity = 11.47 +Step 7760/11700 | Loss: 2.5415 | LR: 1.43e-04 | 22356.0ms/step | 0.9 samples/s +Step 7770/11700 | Loss: 2.4044 | LR: 1.43e-04 | 9160.6ms/step | 2.2 samples/s +Step 7780/11700 | Loss: 2.4987 | LR: 1.42e-04 | 9138.5ms/step | 2.2 samples/s +Step 7790/11700 | Loss: 2.4003 | LR: 1.41e-04 | 9124.7ms/step | 2.2 samples/s +Step 7800/11700 | Loss: 2.4857 | LR: 1.41e-04 | 9172.1ms/step | 2.2 samples/s +Step 7810/11700 | Loss: 2.3267 | LR: 1.40e-04 | 9160.9ms/step | 2.2 samples/s +Step 7820/11700 | Loss: 2.3706 | LR: 1.40e-04 | 9172.9ms/step | 2.2 samples/s +Step 7830/11700 | Loss: 2.3749 | LR: 1.39e-04 | 9172.6ms/step | 2.2 samples/s +Step 7840/11700 | Loss: 2.4368 | LR: 1.38e-04 | 9177.7ms/step | 2.2 samples/s +Step 7850/11700 | Loss: 2.4554 | LR: 1.38e-04 | 9170.9ms/step | 2.2 samples/s +Step 7860/11700 | Loss: 2.4551 | LR: 1.37e-04 | 9152.4ms/step | 2.2 samples/s +Step 7870/11700 | Loss: 2.4937 | LR: 1.36e-04 | 9161.1ms/step | 2.2 samples/s +Step 7880/11700 | Loss: 2.4503 | LR: 1.36e-04 | 9175.5ms/step | 2.2 samples/s +Step 7890/11700 | Loss: 2.4065 | LR: 1.35e-04 | 9181.0ms/step | 2.2 samples/s +Step 7900/11700 | Loss: 2.5095 | LR: 1.34e-04 | 9174.3ms/step | 2.2 samples/s +Step 7910/11700 | Loss: 2.2876 | LR: 1.34e-04 | 9173.1ms/step | 2.2 samples/s +Step 7920/11700 | Loss: 2.3911 | LR: 1.33e-04 | 9172.1ms/step | 2.2 samples/s +Step 7930/11700 | Loss: 2.3992 | LR: 1.33e-04 | 9181.6ms/step | 2.2 samples/s +Step 7940/11700 | Loss: 2.4527 | LR: 1.32e-04 | 9180.0ms/step | 2.2 samples/s +Step 7950/11700 | Loss: 2.4753 | LR: 1.31e-04 | 9176.4ms/step | 2.2 samples/s +Step 7960/11700 | Loss: 2.3989 | LR: 1.31e-04 | 9182.5ms/step | 2.2 samples/s +Step 7970/11700 | Loss: 2.3474 | LR: 1.30e-04 | 9164.9ms/step | 2.2 samples/s +Step 7980/11700 | Loss: 2.5024 | LR: 1.29e-04 | 9163.4ms/step | 2.2 samples/s +Step 7990/11700 | Loss: 2.4774 | LR: 1.29e-04 | 9167.9ms/step | 2.2 samples/s +Step 8000/11700 | Loss: 2.3614 | LR: 1.28e-04 | 9170.9ms/step | 2.2 samples/s +Validation at step 8000: Loss = 2.4351, Perplexity = 11.42 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-8000 +Step 8010/11700 | Loss: 2.4495 | LR: 1.27e-04 | 22663.5ms/step | 0.9 samples/s +Step 8020/11700 | Loss: 2.3236 | LR: 1.27e-04 | 9170.4ms/step | 2.2 samples/s +Step 8030/11700 | Loss: 2.4147 | LR: 1.26e-04 | 9171.3ms/step | 2.2 samples/s +Step 8040/11700 | Loss: 2.4230 | LR: 1.26e-04 | 9167.4ms/step | 2.2 samples/s +Step 8050/11700 | Loss: 2.3836 | LR: 1.25e-04 | 9180.0ms/step | 2.2 samples/s +Step 8060/11700 | Loss: 2.4107 | LR: 1.24e-04 | 9176.3ms/step | 2.2 samples/s +Step 8070/11700 | Loss: 2.4668 | LR: 1.24e-04 | 9162.6ms/step | 2.2 samples/s +Step 8080/11700 | Loss: 2.4645 | LR: 1.23e-04 | 9166.3ms/step | 2.2 samples/s +Step 8090/11700 | Loss: 2.3952 | LR: 1.23e-04 | 9166.6ms/step | 2.2 samples/s +Step 8100/11700 | Loss: 2.4315 | LR: 1.22e-04 | 9136.1ms/step | 2.2 samples/s +Step 8110/11700 | Loss: 2.3796 | LR: 1.21e-04 | 9153.0ms/step | 2.2 samples/s +Step 8120/11700 | Loss: 2.3576 | LR: 1.21e-04 | 9156.1ms/step | 2.2 samples/s +Step 8130/11700 | Loss: 2.3386 | LR: 1.20e-04 | 9161.0ms/step | 2.2 samples/s +Step 8140/11700 | Loss: 2.3917 | LR: 1.19e-04 | 9164.5ms/step | 2.2 samples/s +Step 8150/11700 | Loss: 2.2752 | LR: 1.19e-04 | 9172.2ms/step | 2.2 samples/s +Step 8160/11700 | Loss: 2.5058 | LR: 1.18e-04 | 9167.1ms/step | 2.2 samples/s +Step 8170/11700 | Loss: 2.4014 | LR: 1.18e-04 | 9172.2ms/step | 2.2 samples/s +Step 8180/11700 | Loss: 2.4522 | LR: 1.17e-04 | 9165.4ms/step | 2.2 samples/s +Step 8190/11700 | Loss: 2.3676 | LR: 1.16e-04 | 9131.8ms/step | 2.2 samples/s +Step 8200/11700 | Loss: 2.4183 | LR: 1.16e-04 | 9163.8ms/step | 2.2 samples/s +Step 8210/11700 | Loss: 2.4511 | LR: 1.15e-04 | 9170.2ms/step | 2.2 samples/s +Step 8220/11700 | Loss: 2.4512 | LR: 1.15e-04 | 9172.0ms/step | 2.2 samples/s +Step 8230/11700 | Loss: 2.4338 | LR: 1.14e-04 | 9117.3ms/step | 2.2 samples/s +Step 8240/11700 | Loss: 2.4048 | LR: 1.13e-04 | 9151.1ms/step | 2.2 samples/s +Step 8250/11700 | Loss: 2.4346 | LR: 1.13e-04 | 9141.2ms/step | 2.2 samples/s +Validation at step 8250: Loss = 2.4308, Perplexity = 11.37 +Step 8260/11700 | Loss: 2.3707 | LR: 1.12e-04 | 22385.4ms/step | 0.9 samples/s +Step 8270/11700 | Loss: 2.3795 | LR: 1.12e-04 | 9169.9ms/step | 2.2 samples/s +Step 8280/11700 | Loss: 2.5287 | LR: 1.11e-04 | 9174.3ms/step | 2.2 samples/s +Step 8290/11700 | Loss: 2.3900 | LR: 1.10e-04 | 9175.0ms/step | 2.2 samples/s +Step 8300/11700 | Loss: 2.4435 | LR: 1.10e-04 | 9165.9ms/step | 2.2 samples/s +Step 8310/11700 | Loss: 2.3756 | LR: 1.09e-04 | 9161.2ms/step | 2.2 samples/s +Step 8320/11700 | Loss: 2.4607 | LR: 1.09e-04 | 9165.4ms/step | 2.2 samples/s +Step 8330/11700 | Loss: 2.4689 | LR: 1.08e-04 | 9158.3ms/step | 2.2 samples/s +Step 8340/11700 | Loss: 2.4337 | LR: 1.07e-04 | 9161.7ms/step | 2.2 samples/s +Step 8350/11700 | Loss: 2.4452 | LR: 1.07e-04 | 9169.4ms/step | 2.2 samples/s +Step 8360/11700 | Loss: 2.3883 | LR: 1.06e-04 | 9162.0ms/step | 2.2 samples/s +Step 8370/11700 | Loss: 2.3772 | LR: 1.06e-04 | 9147.3ms/step | 2.2 samples/s +Step 8380/11700 | Loss: 2.4115 | LR: 1.05e-04 | 9101.6ms/step | 2.2 samples/s +Step 8390/11700 | Loss: 2.4160 | LR: 1.05e-04 | 9149.6ms/step | 2.2 samples/s +Step 8400/11700 | Loss: 2.4773 | LR: 1.04e-04 | 9163.5ms/step | 2.2 samples/s +Step 8410/11700 | Loss: 2.6004 | LR: 1.03e-04 | 9140.4ms/step | 2.2 samples/s +Step 8420/11700 | Loss: 2.3565 | LR: 1.03e-04 | 9154.5ms/step | 2.2 samples/s +Step 8430/11700 | Loss: 2.4777 | LR: 1.02e-04 | 9162.9ms/step | 2.2 samples/s +Step 8440/11700 | Loss: 2.4763 | LR: 1.02e-04 | 9172.2ms/step | 2.2 samples/s +Step 8450/11700 | Loss: 2.4180 | LR: 1.01e-04 | 9155.9ms/step | 2.2 samples/s +Step 8460/11700 | Loss: 2.4077 | LR: 1.00e-04 | 9165.0ms/step | 2.2 samples/s +Step 8470/11700 | Loss: 2.4033 | LR: 9.99e-05 | 9169.0ms/step | 2.2 samples/s +Step 8480/11700 | Loss: 2.4570 | LR: 9.93e-05 | 9164.0ms/step | 2.2 samples/s +Step 8490/11700 | Loss: 2.4096 | LR: 9.87e-05 | 9155.6ms/step | 2.2 samples/s +Step 8500/11700 | Loss: 2.2719 | LR: 9.82e-05 | 9153.6ms/step | 2.2 samples/s +Validation at step 8500: Loss = 2.4269, Perplexity = 11.32 +Step 8510/11700 | Loss: 2.3755 | LR: 9.76e-05 | 22344.3ms/step | 0.9 samples/s +Step 8520/11700 | Loss: 2.4218 | LR: 9.70e-05 | 9159.8ms/step | 2.2 samples/s +Step 8530/11700 | Loss: 2.4472 | LR: 9.65e-05 | 9162.9ms/step | 2.2 samples/s +Step 8540/11700 | Loss: 2.4183 | LR: 9.59e-05 | 9156.1ms/step | 2.2 samples/s +Step 8550/11700 | Loss: 2.3086 | LR: 9.53e-05 | 9129.7ms/step | 2.2 samples/s +Step 8560/11700 | Loss: 2.4185 | LR: 9.48e-05 | 9157.5ms/step | 2.2 samples/s +Step 8570/11700 | Loss: 2.4111 | LR: 9.42e-05 | 9164.6ms/step | 2.2 samples/s +Step 8580/11700 | Loss: 2.3969 | LR: 9.36e-05 | 9161.1ms/step | 2.2 samples/s +Step 8590/11700 | Loss: 2.3657 | LR: 9.31e-05 | 9170.4ms/step | 2.2 samples/s +Step 8600/11700 | Loss: 2.4370 | LR: 9.25e-05 | 9162.9ms/step | 2.2 samples/s +Step 8610/11700 | Loss: 2.4129 | LR: 9.20e-05 | 9166.8ms/step | 2.2 samples/s +Step 8620/11700 | Loss: 2.4666 | LR: 9.14e-05 | 9162.5ms/step | 2.2 samples/s +Step 8630/11700 | Loss: 2.4403 | LR: 9.09e-05 | 9173.1ms/step | 2.2 samples/s +Step 8640/11700 | Loss: 2.3765 | LR: 9.03e-05 | 9163.9ms/step | 2.2 samples/s +Step 8650/11700 | Loss: 2.3332 | LR: 8.98e-05 | 9159.5ms/step | 2.2 samples/s +Step 8660/11700 | Loss: 2.3980 | LR: 8.92e-05 | 9172.3ms/step | 2.2 samples/s +Step 8670/11700 | Loss: 2.4891 | LR: 8.87e-05 | 9160.7ms/step | 2.2 samples/s +Step 8680/11700 | Loss: 2.3668 | LR: 8.81e-05 | 9165.0ms/step | 2.2 samples/s +Step 8690/11700 | Loss: 2.4908 | LR: 8.76e-05 | 9165.1ms/step | 2.2 samples/s +Step 8700/11700 | Loss: 2.4151 | LR: 8.70e-05 | 9170.5ms/step | 2.2 samples/s +Step 8710/11700 | Loss: 2.2804 | LR: 8.65e-05 | 9159.9ms/step | 2.2 samples/s +Step 8720/11700 | Loss: 2.3564 | LR: 8.59e-05 | 9171.8ms/step | 2.2 samples/s +Step 8730/11700 | Loss: 2.4057 | LR: 8.54e-05 | 9165.4ms/step | 2.2 samples/s +Step 8740/11700 | Loss: 2.4338 | LR: 8.49e-05 | 9168.4ms/step | 2.2 samples/s +Step 8750/11700 | Loss: 2.4397 | LR: 8.43e-05 | 9172.2ms/step | 2.2 samples/s +Validation at step 8750: Loss = 2.4242, Perplexity = 11.29 +Step 8760/11700 | Loss: 2.3489 | LR: 8.38e-05 | 22404.7ms/step | 0.9 samples/s +Step 8770/11700 | Loss: 2.4060 | LR: 8.33e-05 | 9175.1ms/step | 2.2 samples/s +Step 8780/11700 | Loss: 2.4045 | LR: 8.27e-05 | 9181.0ms/step | 2.2 samples/s +Step 8790/11700 | Loss: 2.4425 | LR: 8.22e-05 | 9164.4ms/step | 2.2 samples/s +Step 8800/11700 | Loss: 2.4719 | LR: 8.17e-05 | 9135.8ms/step | 2.2 samples/s +Step 8810/11700 | Loss: 2.3267 | LR: 8.11e-05 | 9139.0ms/step | 2.2 samples/s +Step 8820/11700 | Loss: 2.3196 | LR: 8.06e-05 | 9139.1ms/step | 2.2 samples/s +Step 8830/11700 | Loss: 2.3807 | LR: 8.01e-05 | 9151.4ms/step | 2.2 samples/s +Step 8840/11700 | Loss: 2.4610 | LR: 7.95e-05 | 9164.5ms/step | 2.2 samples/s +Step 8850/11700 | Loss: 2.4271 | LR: 7.90e-05 | 9162.8ms/step | 2.2 samples/s +Step 8860/11700 | Loss: 2.3090 | LR: 7.85e-05 | 9172.6ms/step | 2.2 samples/s +Step 8870/11700 | Loss: 2.3629 | LR: 7.80e-05 | 9172.7ms/step | 2.2 samples/s +Step 8880/11700 | Loss: 2.4479 | LR: 7.75e-05 | 9170.8ms/step | 2.2 samples/s +Step 8890/11700 | Loss: 2.4114 | LR: 7.69e-05 | 9169.6ms/step | 2.2 samples/s +Step 8900/11700 | Loss: 2.3883 | LR: 7.64e-05 | 9167.5ms/step | 2.2 samples/s +Step 8910/11700 | Loss: 2.4879 | LR: 7.59e-05 | 9171.3ms/step | 2.2 samples/s +Step 8920/11700 | Loss: 2.4656 | LR: 7.54e-05 | 9169.8ms/step | 2.2 samples/s +Step 8930/11700 | Loss: 2.3480 | LR: 7.49e-05 | 9165.0ms/step | 2.2 samples/s +Step 8940/11700 | Loss: 2.4487 | LR: 7.44e-05 | 9157.1ms/step | 2.2 samples/s +Step 8950/11700 | Loss: 2.3798 | LR: 7.39e-05 | 9159.3ms/step | 2.2 samples/s +Step 8960/11700 | Loss: 2.3885 | LR: 7.34e-05 | 9160.4ms/step | 2.2 samples/s +Step 8970/11700 | Loss: 2.3963 | LR: 7.28e-05 | 9151.3ms/step | 2.2 samples/s +Step 8980/11700 | Loss: 2.3837 | LR: 7.23e-05 | 9131.0ms/step | 2.2 samples/s +Step 8990/11700 | Loss: 2.4221 | LR: 7.18e-05 | 9162.3ms/step | 2.2 samples/s +Step 9000/11700 | Loss: 2.4007 | LR: 7.13e-05 | 9160.8ms/step | 2.2 samples/s +Validation at step 9000: Loss = 2.4219, Perplexity = 11.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-9000 +Step 9010/11700 | Loss: 2.4891 | LR: 7.08e-05 | 22653.2ms/step | 0.9 samples/s +Step 9020/11700 | Loss: 2.4024 | LR: 7.03e-05 | 9162.6ms/step | 2.2 samples/s +Step 9030/11700 | Loss: 2.3673 | LR: 6.98e-05 | 9109.2ms/step | 2.2 samples/s +Step 9040/11700 | Loss: 2.3385 | LR: 6.93e-05 | 9152.2ms/step | 2.2 samples/s +Step 9050/11700 | Loss: 2.4325 | LR: 6.88e-05 | 9169.0ms/step | 2.2 samples/s +Step 9060/11700 | Loss: 2.4489 | LR: 6.83e-05 | 9169.6ms/step | 2.2 samples/s +Step 9070/11700 | Loss: 2.4624 | LR: 6.79e-05 | 9154.7ms/step | 2.2 samples/s +Step 9080/11700 | Loss: 2.3576 | LR: 6.74e-05 | 9161.5ms/step | 2.2 samples/s +Step 9090/11700 | Loss: 2.3443 | LR: 6.69e-05 | 9164.2ms/step | 2.2 samples/s +Step 9100/11700 | Loss: 2.4335 | LR: 6.64e-05 | 9164.0ms/step | 2.2 samples/s +Step 9110/11700 | Loss: 2.4318 | LR: 6.59e-05 | 9128.5ms/step | 2.2 samples/s +Step 9120/11700 | Loss: 2.4627 | LR: 6.54e-05 | 9101.7ms/step | 2.2 samples/s +Step 9130/11700 | Loss: 2.3290 | LR: 6.49e-05 | 9144.0ms/step | 2.2 samples/s +Step 9140/11700 | Loss: 2.3076 | LR: 6.45e-05 | 9167.6ms/step | 2.2 samples/s +Step 9150/11700 | Loss: 2.4196 | LR: 6.40e-05 | 9174.0ms/step | 2.2 samples/s +Step 9160/11700 | Loss: 2.5077 | LR: 6.35e-05 | 9161.6ms/step | 2.2 samples/s +Step 9170/11700 | Loss: 2.4575 | LR: 6.30e-05 | 9132.4ms/step | 2.2 samples/s +Step 9180/11700 | Loss: 2.3932 | LR: 6.25e-05 | 9134.0ms/step | 2.2 samples/s +Step 9190/11700 | Loss: 2.4162 | LR: 6.21e-05 | 9152.2ms/step | 2.2 samples/s +Step 9200/11700 | Loss: 2.2983 | LR: 6.16e-05 | 9161.5ms/step | 2.2 samples/s +Step 9210/11700 | Loss: 2.4039 | LR: 6.11e-05 | 9155.3ms/step | 2.2 samples/s +Step 9220/11700 | Loss: 2.4864 | LR: 6.07e-05 | 9158.9ms/step | 2.2 samples/s +Step 9230/11700 | Loss: 2.3932 | LR: 6.02e-05 | 9157.0ms/step | 2.2 samples/s +Step 9240/11700 | Loss: 2.3804 | LR: 5.97e-05 | 9154.8ms/step | 2.2 samples/s +Step 9250/11700 | Loss: 2.4109 | LR: 5.93e-05 | 9155.3ms/step | 2.2 samples/s +Validation at step 9250: Loss = 2.4202, Perplexity = 11.25 +Step 9260/11700 | Loss: 2.3340 | LR: 5.88e-05 | 22387.4ms/step | 0.9 samples/s +Step 9270/11700 | Loss: 2.4286 | LR: 5.83e-05 | 9163.9ms/step | 2.2 samples/s +Step 9280/11700 | Loss: 2.2828 | LR: 5.79e-05 | 9165.9ms/step | 2.2 samples/s +Step 9290/11700 | Loss: 2.3716 | LR: 5.74e-05 | 9163.2ms/step | 2.2 samples/s +Step 9300/11700 | Loss: 2.4253 | LR: 5.70e-05 | 9170.2ms/step | 2.2 samples/s +Step 9310/11700 | Loss: 2.3197 | LR: 5.65e-05 | 9154.8ms/step | 2.2 samples/s +Step 9320/11700 | Loss: 2.3636 | LR: 5.61e-05 | 9174.8ms/step | 2.2 samples/s +Step 9330/11700 | Loss: 2.3599 | LR: 5.56e-05 | 9161.4ms/step | 2.2 samples/s +Step 9340/11700 | Loss: 2.4703 | LR: 5.52e-05 | 9163.2ms/step | 2.2 samples/s +Step 9350/11700 | Loss: 2.3582 | LR: 5.47e-05 | 9165.9ms/step | 2.2 samples/s +Step 9360/11700 | Loss: 2.3917 | LR: 5.43e-05 | 9134.7ms/step | 2.2 samples/s +Step 9370/11700 | Loss: 2.4085 | LR: 5.38e-05 | 9151.9ms/step | 2.2 samples/s +Step 9380/11700 | Loss: 2.5106 | LR: 5.34e-05 | 9160.6ms/step | 2.2 samples/s +Step 9390/11700 | Loss: 2.4178 | LR: 5.29e-05 | 9159.9ms/step | 2.2 samples/s +Step 9400/11700 | Loss: 2.3949 | LR: 5.25e-05 | 9163.4ms/step | 2.2 samples/s +Step 9410/11700 | Loss: 2.4115 | LR: 5.20e-05 | 9163.4ms/step | 2.2 samples/s +Step 9420/11700 | Loss: 2.4736 | LR: 5.16e-05 | 9167.9ms/step | 2.2 samples/s +Step 9430/11700 | Loss: 2.2829 | LR: 5.12e-05 | 9157.2ms/step | 2.2 samples/s +Step 9440/11700 | Loss: 2.5008 | LR: 5.07e-05 | 9169.4ms/step | 2.2 samples/s +Step 9450/11700 | Loss: 2.4437 | LR: 5.03e-05 | 9171.1ms/step | 2.2 samples/s +Step 9460/11700 | Loss: 2.4427 | LR: 4.99e-05 | 9164.0ms/step | 2.2 samples/s +Step 9470/11700 | Loss: 2.4177 | LR: 4.94e-05 | 9160.0ms/step | 2.2 samples/s +Step 9480/11700 | Loss: 2.3766 | LR: 4.90e-05 | 9124.3ms/step | 2.2 samples/s +Step 9490/11700 | Loss: 2.5288 | LR: 4.86e-05 | 9091.2ms/step | 2.2 samples/s +Step 9500/11700 | Loss: 2.3329 | LR: 4.82e-05 | 9083.6ms/step | 2.2 samples/s +Validation at step 9500: Loss = 2.4189, Perplexity = 11.23 +Step 9510/11700 | Loss: 2.4208 | LR: 4.77e-05 | 22171.4ms/step | 0.9 samples/s +Step 9520/11700 | Loss: 2.3744 | LR: 4.73e-05 | 9083.4ms/step | 2.2 samples/s +Step 9530/11700 | Loss: 2.3762 | LR: 4.69e-05 | 9081.0ms/step | 2.2 samples/s +Step 9540/11700 | Loss: 2.3385 | LR: 4.65e-05 | 9079.3ms/step | 2.2 samples/s +Step 9550/11700 | Loss: 2.5458 | LR: 4.61e-05 | 9077.4ms/step | 2.2 samples/s +Step 9560/11700 | Loss: 2.3236 | LR: 4.57e-05 | 9077.5ms/step | 2.2 samples/s +Step 9570/11700 | Loss: 2.3622 | LR: 4.52e-05 | 9096.5ms/step | 2.2 samples/s +Step 9580/11700 | Loss: 2.4830 | LR: 4.48e-05 | 9101.4ms/step | 2.2 samples/s +Step 9590/11700 | Loss: 2.4136 | LR: 4.44e-05 | 9096.7ms/step | 2.2 samples/s +Step 9600/11700 | Loss: 2.4221 | LR: 4.40e-05 | 9100.0ms/step | 2.2 samples/s +Step 9610/11700 | Loss: 2.4217 | LR: 4.36e-05 | 9098.0ms/step | 2.2 samples/s +Step 9620/11700 | Loss: 2.4346 | LR: 4.32e-05 | 9096.1ms/step | 2.2 samples/s +Step 9630/11700 | Loss: 2.3820 | LR: 4.28e-05 | 9099.6ms/step | 2.2 samples/s +Step 9640/11700 | Loss: 2.4332 | LR: 4.24e-05 | 9100.8ms/step | 2.2 samples/s +Step 9650/11700 | Loss: 2.5114 | LR: 4.20e-05 | 9093.8ms/step | 2.2 samples/s +Step 9660/11700 | Loss: 2.5194 | LR: 4.16e-05 | 9091.2ms/step | 2.2 samples/s +Step 9670/11700 | Loss: 2.4134 | LR: 4.12e-05 | 9089.9ms/step | 2.2 samples/s +Step 9680/11700 | Loss: 2.2336 | LR: 4.08e-05 | 9093.9ms/step | 2.2 samples/s +Step 9690/11700 | Loss: 2.4744 | LR: 4.04e-05 | 9099.4ms/step | 2.2 samples/s +Step 9700/11700 | Loss: 2.3434 | LR: 4.00e-05 | 9099.8ms/step | 2.2 samples/s +Step 9710/11700 | Loss: 2.3949 | LR: 3.97e-05 | 9098.3ms/step | 2.2 samples/s +Step 9720/11700 | Loss: 2.4678 | LR: 3.93e-05 | 9099.1ms/step | 2.2 samples/s +Step 9730/11700 | Loss: 2.3112 | LR: 3.89e-05 | 9103.3ms/step | 2.2 samples/s +Step 9740/11700 | Loss: 2.4822 | LR: 3.85e-05 | 9099.7ms/step | 2.2 samples/s +Step 9750/11700 | Loss: 2.4073 | LR: 3.81e-05 | 9098.1ms/step | 2.2 samples/s +Validation at step 9750: Loss = 2.4181, Perplexity = 11.22 +Step 9760/11700 | Loss: 2.3913 | LR: 3.77e-05 | 22216.4ms/step | 0.9 samples/s +Step 9770/11700 | Loss: 2.3342 | LR: 3.74e-05 | 9100.0ms/step | 2.2 samples/s +Step 9780/11700 | Loss: 2.5310 | LR: 3.70e-05 | 9106.5ms/step | 2.2 samples/s +Step 9790/11700 | Loss: 2.3876 | LR: 3.66e-05 | 9098.7ms/step | 2.2 samples/s +Step 9800/11700 | Loss: 2.3358 | LR: 3.62e-05 | 9102.9ms/step | 2.2 samples/s +Step 9810/11700 | Loss: 2.4528 | LR: 3.59e-05 | 9103.0ms/step | 2.2 samples/s +Step 9820/11700 | Loss: 2.3030 | LR: 3.55e-05 | 9102.3ms/step | 2.2 samples/s +Step 9830/11700 | Loss: 2.3220 | LR: 3.51e-05 | 9098.1ms/step | 2.2 samples/s +Step 9840/11700 | Loss: 2.3566 | LR: 3.48e-05 | 9097.1ms/step | 2.2 samples/s +Step 9850/11700 | Loss: 2.4318 | LR: 3.44e-05 | 9095.0ms/step | 2.2 samples/s +Step 9860/11700 | Loss: 2.4119 | LR: 3.40e-05 | 9098.3ms/step | 2.2 samples/s +Step 9870/11700 | Loss: 2.3591 | LR: 3.37e-05 | 9093.9ms/step | 2.2 samples/s +Step 9880/11700 | Loss: 2.4002 | LR: 3.33e-05 | 9091.3ms/step | 2.2 samples/s +Step 9890/11700 | Loss: 2.3055 | LR: 3.30e-05 | 9094.7ms/step | 2.2 samples/s +Step 9900/11700 | Loss: 2.4387 | LR: 3.26e-05 | 9094.9ms/step | 2.2 samples/s +Step 9910/11700 | Loss: 2.4232 | LR: 3.22e-05 | 9093.2ms/step | 2.2 samples/s +Step 9920/11700 | Loss: 2.3867 | LR: 3.19e-05 | 9092.9ms/step | 2.2 samples/s +Step 9930/11700 | Loss: 2.4695 | LR: 3.15e-05 | 9083.8ms/step | 2.2 samples/s +Step 9940/11700 | Loss: 2.2801 | LR: 3.12e-05 | 9077.9ms/step | 2.2 samples/s +Step 9950/11700 | Loss: 2.3903 | LR: 3.09e-05 | 9083.8ms/step | 2.2 samples/s +Step 9960/11700 | Loss: 2.3374 | LR: 3.05e-05 | 9088.2ms/step | 2.2 samples/s +Step 9970/11700 | Loss: 2.4123 | LR: 3.02e-05 | 9099.7ms/step | 2.2 samples/s +Step 9980/11700 | Loss: 2.3435 | LR: 2.98e-05 | 9094.8ms/step | 2.2 samples/s +Step 9990/11700 | Loss: 2.2962 | LR: 2.95e-05 | 9103.3ms/step | 2.2 samples/s +Step 10000/11700 | Loss: 2.4109 | LR: 2.92e-05 | 9106.0ms/step | 2.2 samples/s +Validation at step 10000: Loss = 2.4175, Perplexity = 11.22 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-10000 +Step 10010/11700 | Loss: 2.5097 | LR: 2.88e-05 | 22548.5ms/step | 0.9 samples/s +Step 10020/11700 | Loss: 2.4932 | LR: 2.85e-05 | 9106.1ms/step | 2.2 samples/s +Step 10030/11700 | Loss: 2.3928 | LR: 2.82e-05 | 9105.3ms/step | 2.2 samples/s +Step 10040/11700 | Loss: 2.4463 | LR: 2.78e-05 | 9099.3ms/step | 2.2 samples/s +Step 10050/11700 | Loss: 2.4821 | LR: 2.75e-05 | 9103.3ms/step | 2.2 samples/s +Step 10060/11700 | Loss: 2.3571 | LR: 2.72e-05 | 9102.9ms/step | 2.2 samples/s +Step 10070/11700 | Loss: 2.4759 | LR: 2.68e-05 | 9102.8ms/step | 2.2 samples/s +Step 10080/11700 | Loss: 2.3518 | LR: 2.65e-05 | 9100.5ms/step | 2.2 samples/s +Step 10090/11700 | Loss: 2.4206 | LR: 2.62e-05 | 9102.1ms/step | 2.2 samples/s +Step 10100/11700 | Loss: 2.3821 | LR: 2.59e-05 | 9102.8ms/step | 2.2 samples/s +Step 10110/11700 | Loss: 2.4557 | LR: 2.56e-05 | 9097.5ms/step | 2.2 samples/s +Step 10120/11700 | Loss: 2.3226 | LR: 2.52e-05 | 9100.8ms/step | 2.2 samples/s +Step 10130/11700 | Loss: 2.4919 | LR: 2.49e-05 | 9102.8ms/step | 2.2 samples/s +Step 10140/11700 | Loss: 2.5247 | LR: 2.46e-05 | 9104.9ms/step | 2.2 samples/s +Step 10150/11700 | Loss: 2.4309 | LR: 2.43e-05 | 9100.1ms/step | 2.2 samples/s +Step 10160/11700 | Loss: 2.4178 | LR: 2.40e-05 | 9100.3ms/step | 2.2 samples/s +Step 10170/11700 | Loss: 2.4020 | LR: 2.37e-05 | 9104.2ms/step | 2.2 samples/s +Step 10180/11700 | Loss: 2.4233 | LR: 2.34e-05 | 9107.6ms/step | 2.2 samples/s +Step 10190/11700 | Loss: 2.3305 | LR: 2.31e-05 | 9101.2ms/step | 2.2 samples/s +Step 10200/11700 | Loss: 2.3575 | LR: 2.28e-05 | 9103.3ms/step | 2.2 samples/s +Step 10210/11700 | Loss: 2.4528 | LR: 2.25e-05 | 9101.5ms/step | 2.2 samples/s +Step 10220/11700 | Loss: 2.4093 | LR: 2.22e-05 | 9099.6ms/step | 2.2 samples/s +Step 10230/11700 | Loss: 2.5148 | LR: 2.19e-05 | 9105.7ms/step | 2.2 samples/s +Step 10240/11700 | Loss: 2.5010 | LR: 2.16e-05 | 9106.7ms/step | 2.2 samples/s +Step 10250/11700 | Loss: 2.4495 | LR: 2.13e-05 | 9101.9ms/step | 2.2 samples/s +Validation at step 10250: Loss = 2.4171, Perplexity = 11.21 +Step 10260/11700 | Loss: 2.4596 | LR: 2.10e-05 | 22198.9ms/step | 0.9 samples/s +Step 10270/11700 | Loss: 2.3600 | LR: 2.07e-05 | 9086.5ms/step | 2.2 samples/s +Step 10280/11700 | Loss: 2.4585 | LR: 2.05e-05 | 9123.9ms/step | 2.2 samples/s +Step 10290/11700 | Loss: 2.4423 | LR: 2.02e-05 | 9139.1ms/step | 2.2 samples/s +Step 10300/11700 | Loss: 2.2947 | LR: 1.99e-05 | 9116.9ms/step | 2.2 samples/s +Step 10310/11700 | Loss: 2.4039 | LR: 1.96e-05 | 9134.9ms/step | 2.2 samples/s +Step 10320/11700 | Loss: 2.3497 | LR: 1.93e-05 | 9112.7ms/step | 2.2 samples/s +Step 10330/11700 | Loss: 2.3420 | LR: 1.91e-05 | 9080.9ms/step | 2.2 samples/s +Step 10340/11700 | Loss: 2.5107 | LR: 1.88e-05 | 9092.8ms/step | 2.2 samples/s +Step 10350/11700 | Loss: 2.4409 | LR: 1.85e-05 | 9096.1ms/step | 2.2 samples/s +Step 10360/11700 | Loss: 2.4368 | LR: 1.82e-05 | 9141.3ms/step | 2.2 samples/s +Step 10370/11700 | Loss: 2.4174 | LR: 1.80e-05 | 9124.1ms/step | 2.2 samples/s +Step 10380/11700 | Loss: 2.2527 | LR: 1.77e-05 | 9139.7ms/step | 2.2 samples/s +Step 10390/11700 | Loss: 2.5497 | LR: 1.75e-05 | 9136.6ms/step | 2.2 samples/s +Step 10400/11700 | Loss: 2.4588 | LR: 1.72e-05 | 9118.6ms/step | 2.2 samples/s +Step 10410/11700 | Loss: 2.4077 | LR: 1.69e-05 | 9116.5ms/step | 2.2 samples/s +Step 10420/11700 | Loss: 2.3855 | LR: 1.67e-05 | 9127.3ms/step | 2.2 samples/s +Step 10430/11700 | Loss: 2.3808 | LR: 1.64e-05 | 9119.5ms/step | 2.2 samples/s +Step 10440/11700 | Loss: 2.3537 | LR: 1.62e-05 | 9131.4ms/step | 2.2 samples/s +Step 10450/11700 | Loss: 2.4298 | LR: 1.59e-05 | 9135.3ms/step | 2.2 samples/s +Step 10460/11700 | Loss: 2.4546 | LR: 1.57e-05 | 9150.5ms/step | 2.2 samples/s +Step 10470/11700 | Loss: 2.3395 | LR: 1.54e-05 | 9126.9ms/step | 2.2 samples/s +Step 10480/11700 | Loss: 2.3809 | LR: 1.52e-05 | 9132.6ms/step | 2.2 samples/s +Step 10490/11700 | Loss: 2.3225 | LR: 1.49e-05 | 9135.3ms/step | 2.2 samples/s +Step 10500/11700 | Loss: 2.4105 | LR: 1.47e-05 | 9121.3ms/step | 2.2 samples/s +Validation at step 10500: Loss = 2.4170, Perplexity = 11.21 +Step 10510/11700 | Loss: 2.5349 | LR: 1.44e-05 | 22222.6ms/step | 0.9 samples/s +Step 10520/11700 | Loss: 2.4357 | LR: 1.42e-05 | 9102.4ms/step | 2.2 samples/s +Step 10530/11700 | Loss: 2.4227 | LR: 1.40e-05 | 9107.3ms/step | 2.2 samples/s +Step 10540/11700 | Loss: 2.2726 | LR: 1.37e-05 | 9111.3ms/step | 2.2 samples/s +Step 10550/11700 | Loss: 2.4487 | LR: 1.35e-05 | 9109.7ms/step | 2.2 samples/s +Step 10560/11700 | Loss: 2.5486 | LR: 1.33e-05 | 9103.0ms/step | 2.2 samples/s +Step 10570/11700 | Loss: 2.3045 | LR: 1.30e-05 | 9108.7ms/step | 2.2 samples/s +Step 10580/11700 | Loss: 2.4801 | LR: 1.28e-05 | 9117.5ms/step | 2.2 samples/s +Step 10590/11700 | Loss: 2.3417 | LR: 1.26e-05 | 9126.4ms/step | 2.2 samples/s +Step 10600/11700 | Loss: 2.4317 | LR: 1.23e-05 | 9124.0ms/step | 2.2 samples/s +Step 10610/11700 | Loss: 2.4008 | LR: 1.21e-05 | 9121.9ms/step | 2.2 samples/s +Step 10620/11700 | Loss: 2.3191 | LR: 1.19e-05 | 9113.9ms/step | 2.2 samples/s +Step 10630/11700 | Loss: 2.3881 | LR: 1.17e-05 | 9120.8ms/step | 2.2 samples/s +Step 10640/11700 | Loss: 2.3732 | LR: 1.15e-05 | 9137.7ms/step | 2.2 samples/s +Step 10650/11700 | Loss: 2.4272 | LR: 1.13e-05 | 9141.0ms/step | 2.2 samples/s +Step 10660/11700 | Loss: 2.3779 | LR: 1.10e-05 | 9106.6ms/step | 2.2 samples/s +Step 10670/11700 | Loss: 2.3638 | LR: 1.08e-05 | 9103.7ms/step | 2.2 samples/s +Step 10680/11700 | Loss: 2.4410 | LR: 1.06e-05 | 9121.0ms/step | 2.2 samples/s +Step 10690/11700 | Loss: 2.4300 | LR: 1.04e-05 | 9104.0ms/step | 2.2 samples/s +Step 10700/11700 | Loss: 2.3986 | LR: 1.02e-05 | 9121.9ms/step | 2.2 samples/s +Step 10710/11700 | Loss: 2.4302 | LR: 1.00e-05 | 9112.8ms/step | 2.2 samples/s +Step 10720/11700 | Loss: 2.3663 | LR: 9.82e-06 | 9108.8ms/step | 2.2 samples/s +Step 10730/11700 | Loss: 2.4699 | LR: 9.62e-06 | 9133.5ms/step | 2.2 samples/s +Step 10740/11700 | Loss: 2.2946 | LR: 9.42e-06 | 9148.5ms/step | 2.2 samples/s +Step 10750/11700 | Loss: 2.3235 | LR: 9.23e-06 | 9152.1ms/step | 2.2 samples/s +Validation at step 10750: Loss = 2.4169, Perplexity = 11.21 +Step 10760/11700 | Loss: 2.4929 | LR: 9.04e-06 | 22361.3ms/step | 0.9 samples/s +Step 10770/11700 | Loss: 2.4670 | LR: 8.85e-06 | 9158.1ms/step | 2.2 samples/s +Step 10780/11700 | Loss: 2.4953 | LR: 8.66e-06 | 9165.7ms/step | 2.2 samples/s +Step 10790/11700 | Loss: 2.4054 | LR: 8.47e-06 | 9159.9ms/step | 2.2 samples/s +Step 10800/11700 | Loss: 2.5014 | LR: 8.29e-06 | 9151.5ms/step | 2.2 samples/s +Step 10810/11700 | Loss: 2.3980 | LR: 8.11e-06 | 9149.6ms/step | 2.2 samples/s +Step 10820/11700 | Loss: 2.4854 | LR: 7.93e-06 | 9148.7ms/step | 2.2 samples/s +Step 10830/11700 | Loss: 2.3843 | LR: 7.75e-06 | 9151.3ms/step | 2.2 samples/s +Step 10840/11700 | Loss: 2.3658 | LR: 7.57e-06 | 9150.1ms/step | 2.2 samples/s +Step 10850/11700 | Loss: 2.3163 | LR: 7.40e-06 | 9134.2ms/step | 2.2 samples/s +Step 10860/11700 | Loss: 2.3183 | LR: 7.22e-06 | 9140.0ms/step | 2.2 samples/s +Step 10870/11700 | Loss: 2.3455 | LR: 7.05e-06 | 9145.6ms/step | 2.2 samples/s +Step 10880/11700 | Loss: 2.3252 | LR: 6.89e-06 | 9148.4ms/step | 2.2 samples/s +Step 10890/11700 | Loss: 2.3133 | LR: 6.72e-06 | 9144.5ms/step | 2.2 samples/s +Step 10900/11700 | Loss: 2.4186 | LR: 6.56e-06 | 9151.7ms/step | 2.2 samples/s +Step 10910/11700 | Loss: 2.4580 | LR: 6.39e-06 | 9152.0ms/step | 2.2 samples/s +Step 10920/11700 | Loss: 2.3813 | LR: 6.23e-06 | 9152.0ms/step | 2.2 samples/s +Step 10930/11700 | Loss: 2.3034 | LR: 6.08e-06 | 9139.7ms/step | 2.2 samples/s +Step 10940/11700 | Loss: 2.3248 | LR: 5.92e-06 | 9138.4ms/step | 2.2 samples/s +Step 10950/11700 | Loss: 2.3327 | LR: 5.77e-06 | 9149.5ms/step | 2.2 samples/s +Step 10960/11700 | Loss: 2.4807 | LR: 5.61e-06 | 9130.0ms/step | 2.2 samples/s +Step 10970/11700 | Loss: 2.4386 | LR: 5.46e-06 | 9108.1ms/step | 2.2 samples/s +Step 10980/11700 | Loss: 2.4359 | LR: 5.31e-06 | 9124.5ms/step | 2.2 samples/s +Step 10990/11700 | Loss: 2.3741 | LR: 5.17e-06 | 9128.7ms/step | 2.2 samples/s +Step 11000/11700 | Loss: 2.3525 | LR: 5.02e-06 | 9133.9ms/step | 2.2 samples/s +Validation at step 11000: Loss = 2.4168, Perplexity = 11.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11000 +Step 11010/11700 | Loss: 2.3427 | LR: 4.88e-06 | 22607.8ms/step | 0.9 samples/s +Step 11020/11700 | Loss: 2.4104 | LR: 4.74e-06 | 9131.2ms/step | 2.2 samples/s +Step 11030/11700 | Loss: 2.4457 | LR: 4.60e-06 | 9135.8ms/step | 2.2 samples/s +Step 11040/11700 | Loss: 2.4375 | LR: 4.47e-06 | 9135.8ms/step | 2.2 samples/s +Step 11050/11700 | Loss: 2.4598 | LR: 4.33e-06 | 9131.4ms/step | 2.2 samples/s +Step 11060/11700 | Loss: 2.3929 | LR: 4.20e-06 | 9131.5ms/step | 2.2 samples/s +Step 11070/11700 | Loss: 2.3549 | LR: 4.07e-06 | 9129.3ms/step | 2.2 samples/s +Step 11080/11700 | Loss: 2.4145 | LR: 3.94e-06 | 9122.4ms/step | 2.2 samples/s +Step 11090/11700 | Loss: 2.3997 | LR: 3.82e-06 | 9150.4ms/step | 2.2 samples/s +Step 11100/11700 | Loss: 2.4456 | LR: 3.69e-06 | 9149.3ms/step | 2.2 samples/s +Step 11110/11700 | Loss: 2.3905 | LR: 3.57e-06 | 9144.8ms/step | 2.2 samples/s +Step 11120/11700 | Loss: 2.4358 | LR: 3.45e-06 | 9146.0ms/step | 2.2 samples/s +Step 11130/11700 | Loss: 2.4798 | LR: 3.34e-06 | 9148.4ms/step | 2.2 samples/s +Step 11140/11700 | Loss: 2.3864 | LR: 3.22e-06 | 9144.8ms/step | 2.2 samples/s +Step 11150/11700 | Loss: 2.4082 | LR: 3.11e-06 | 9144.2ms/step | 2.2 samples/s +Step 11160/11700 | Loss: 2.3546 | LR: 2.99e-06 | 9146.1ms/step | 2.2 samples/s +Step 11170/11700 | Loss: 2.4663 | LR: 2.88e-06 | 9143.8ms/step | 2.2 samples/s +Step 11180/11700 | Loss: 2.3766 | LR: 2.78e-06 | 9140.5ms/step | 2.2 samples/s +Step 11190/11700 | Loss: 2.4169 | LR: 2.67e-06 | 9141.8ms/step | 2.2 samples/s +Step 11200/11700 | Loss: 2.4129 | LR: 2.57e-06 | 9143.4ms/step | 2.2 samples/s +Step 11210/11700 | Loss: 2.4614 | LR: 2.47e-06 | 9144.1ms/step | 2.2 samples/s +Step 11220/11700 | Loss: 2.3528 | LR: 2.37e-06 | 9143.7ms/step | 2.2 samples/s +Step 11230/11700 | Loss: 2.2803 | LR: 2.27e-06 | 9141.3ms/step | 2.2 samples/s +Step 11240/11700 | Loss: 2.3476 | LR: 2.17e-06 | 9144.4ms/step | 2.2 samples/s +Step 11250/11700 | Loss: 2.3648 | LR: 2.08e-06 | 9144.3ms/step | 2.2 samples/s +Validation at step 11250: Loss = 2.4168, Perplexity = 11.21 +Step 11260/11700 | Loss: 2.3975 | LR: 1.99e-06 | 22340.4ms/step | 0.9 samples/s +Step 11270/11700 | Loss: 2.4421 | LR: 1.90e-06 | 9141.8ms/step | 2.2 samples/s +Step 11280/11700 | Loss: 2.3819 | LR: 1.81e-06 | 9143.4ms/step | 2.2 samples/s +Step 11290/11700 | Loss: 2.4519 | LR: 1.73e-06 | 9145.6ms/step | 2.2 samples/s +Step 11300/11700 | Loss: 2.4193 | LR: 1.64e-06 | 9146.9ms/step | 2.2 samples/s +Step 11310/11700 | Loss: 2.3875 | LR: 1.56e-06 | 9145.4ms/step | 2.2 samples/s +Step 11320/11700 | Loss: 2.4187 | LR: 1.48e-06 | 9143.3ms/step | 2.2 samples/s +Step 11330/11700 | Loss: 2.3440 | LR: 1.41e-06 | 9143.8ms/step | 2.2 samples/s +Step 11340/11700 | Loss: 2.3200 | LR: 1.33e-06 | 9143.5ms/step | 2.2 samples/s +Step 11350/11700 | Loss: 2.4451 | LR: 1.26e-06 | 9141.5ms/step | 2.2 samples/s +Step 11360/11700 | Loss: 2.4379 | LR: 1.19e-06 | 9145.6ms/step | 2.2 samples/s +Step 11370/11700 | Loss: 2.3974 | LR: 1.12e-06 | 9143.7ms/step | 2.2 samples/s +Step 11380/11700 | Loss: 2.3833 | LR: 1.05e-06 | 9141.1ms/step | 2.2 samples/s +Step 11390/11700 | Loss: 2.4030 | LR: 9.88e-07 | 9144.8ms/step | 2.2 samples/s +Step 11400/11700 | Loss: 2.4292 | LR: 9.25e-07 | 9144.1ms/step | 2.2 samples/s +Step 11410/11700 | Loss: 2.4034 | LR: 8.65e-07 | 9146.5ms/step | 2.2 samples/s +Step 11420/11700 | Loss: 2.2530 | LR: 8.06e-07 | 9145.4ms/step | 2.2 samples/s +Step 11430/11700 | Loss: 2.3557 | LR: 7.50e-07 | 9142.0ms/step | 2.2 samples/s +Step 11440/11700 | Loss: 2.4461 | LR: 6.95e-07 | 9145.1ms/step | 2.2 samples/s +Step 11450/11700 | Loss: 2.2934 | LR: 6.43e-07 | 9147.1ms/step | 2.2 samples/s +Step 11460/11700 | Loss: 2.4179 | LR: 5.92e-07 | 9143.5ms/step | 2.2 samples/s +Step 11470/11700 | Loss: 2.4641 | LR: 5.44e-07 | 9144.8ms/step | 2.2 samples/s +Step 11480/11700 | Loss: 2.3326 | LR: 4.98e-07 | 9142.3ms/step | 2.2 samples/s +Step 11490/11700 | Loss: 2.4337 | LR: 4.54e-07 | 9144.3ms/step | 2.2 samples/s +Step 11500/11700 | Loss: 2.3676 | LR: 4.11e-07 | 9149.6ms/step | 2.2 samples/s +Validation at step 11500: Loss = 2.4168, Perplexity = 11.21 +Step 11510/11700 | Loss: 2.3525 | LR: 3.71e-07 | 22351.6ms/step | 0.9 samples/s +Step 11520/11700 | Loss: 2.4273 | LR: 3.33e-07 | 9148.2ms/step | 2.2 samples/s +Step 11530/11700 | Loss: 2.3713 | LR: 2.97e-07 | 9147.8ms/step | 2.2 samples/s +Step 11540/11700 | Loss: 2.3495 | LR: 2.63e-07 | 9147.0ms/step | 2.2 samples/s +Step 11550/11700 | Loss: 2.4695 | LR: 2.31e-07 | 9145.7ms/step | 2.2 samples/s +Step 11560/11700 | Loss: 2.3688 | LR: 2.02e-07 | 9145.8ms/step | 2.2 samples/s +Step 11570/11700 | Loss: 2.4825 | LR: 1.74e-07 | 9144.8ms/step | 2.2 samples/s +Step 11580/11700 | Loss: 2.3171 | LR: 1.48e-07 | 9145.7ms/step | 2.2 samples/s +Step 11590/11700 | Loss: 2.4345 | LR: 1.24e-07 | 9147.6ms/step | 2.2 samples/s +Step 11600/11700 | Loss: 2.4364 | LR: 1.03e-07 | 9145.1ms/step | 2.2 samples/s +Step 11610/11700 | Loss: 2.4370 | LR: 8.33e-08 | 9146.4ms/step | 2.2 samples/s +Step 11620/11700 | Loss: 2.3535 | LR: 6.58e-08 | 9146.2ms/step | 2.2 samples/s +Step 11630/11700 | Loss: 2.3021 | LR: 5.04e-08 | 9143.1ms/step | 2.2 samples/s +Step 11640/11700 | Loss: 2.4361 | LR: 3.70e-08 | 9145.8ms/step | 2.2 samples/s +Step 11650/11700 | Loss: 2.4875 | LR: 2.57e-08 | 9145.5ms/step | 2.2 samples/s +Step 11660/11700 | Loss: 2.2632 | LR: 1.65e-08 | 9147.1ms/step | 2.2 samples/s +Step 11670/11700 | Loss: 2.4166 | LR: 9.26e-09 | 9149.2ms/step | 2.2 samples/s +Step 11680/11700 | Loss: 2.4306 | LR: 4.12e-09 | 9147.5ms/step | 2.2 samples/s +Step 11690/11700 | Loss: 2.3321 | LR: 1.03e-09 | 9147.6ms/step | 2.2 samples/s +Step 11700/11700 | Loss: 2.3975 | LR: 0.00e+00 | 9147.1ms/step | 2.2 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11700 +wandb: updating run metadata +wandb: updating run metadata; uploading history steps 1168-1169, summary, console lines 1228-1229 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log +wandb: uploading data +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▆▄▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ███▁██▁███████████████████████▁█████████ +wandb: speed/steps_per_sec █████████▁██████████████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.41683 +wandb: eval/perplexity 11.21025 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 9147.05243 +wandb: speed/samples_per_sec 2.1865 +wandb: speed/steps_per_sec 0.10932 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m dense 30k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/jmi9928p +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260707_035828-jmi9928p/logs +Training complete! diff --git a/epfl_final/logs/l30k365m_fixed_s1.log b/epfl_final/logs/l30k365m_fixed_s1.log new file mode 100644 index 0000000000000000000000000000000000000000..9c1ecb999ed61a3acb64005cb66f11a24e777618 --- /dev/null +++ b/epfl_final/logs/l30k365m_fixed_s1.log @@ -0,0 +1,1441 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 11700, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 20, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m fixed s1 30k", + "wandb_tags": [ + "qwen3", + "365m", + "30k", + "scaleladder", + "fixed", + "seed1" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s1", + "sparse_mask_seed": 1, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 768] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 768] [1, 30720, 768] 768 True +├─Linear: 1-2 [1, 30720, 768] [1, 30720, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 475.65 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 58494.81 +Params size (MB): 951.29 +Estimated Total Size (MB): 59446.35 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 1 +Gradient accumulation steps: 20 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 749299 +Optimizer steps per epoch: 37464 +Total optimizer steps: 11700 (capped by max_steps; 0.31 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035814-kxni9nyw +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m fixed s1 30k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/kxni9nyw +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/11700 | Loss: 12.0481 | LR: 6.67e-06 | 12242.1ms/step | 1.6 samples/s +Step 20/11700 | Loss: 11.9639 | LR: 1.33e-05 | 7668.7ms/step | 2.6 samples/s +Step 30/11700 | Loss: 11.6999 | LR: 2.00e-05 | 7668.2ms/step | 2.6 samples/s +Step 40/11700 | Loss: 11.1855 | LR: 2.67e-05 | 7651.9ms/step | 2.6 samples/s +Step 50/11700 | Loss: 10.8574 | LR: 3.33e-05 | 7638.5ms/step | 2.6 samples/s +Step 60/11700 | Loss: 10.5730 | LR: 4.00e-05 | 7630.7ms/step | 2.6 samples/s +Step 70/11700 | Loss: 10.2935 | LR: 4.67e-05 | 7624.0ms/step | 2.6 samples/s +Step 80/11700 | Loss: 10.0250 | LR: 5.33e-05 | 7617.0ms/step | 2.6 samples/s +Step 90/11700 | Loss: 9.7335 | LR: 6.00e-05 | 7618.1ms/step | 2.6 samples/s +Step 100/11700 | Loss: 9.3537 | LR: 6.67e-05 | 7627.8ms/step | 2.6 samples/s +Step 110/11700 | Loss: 8.8777 | LR: 7.33e-05 | 7624.6ms/step | 2.6 samples/s +Step 120/11700 | Loss: 8.3232 | LR: 8.00e-05 | 7627.7ms/step | 2.6 samples/s +Step 130/11700 | Loss: 7.8608 | LR: 8.67e-05 | 7631.8ms/step | 2.6 samples/s +Step 140/11700 | Loss: 7.5848 | LR: 9.33e-05 | 7638.4ms/step | 2.6 samples/s +Step 150/11700 | Loss: 7.3759 | LR: 1.00e-04 | 7643.7ms/step | 2.6 samples/s +Step 160/11700 | Loss: 7.1625 | LR: 1.07e-04 | 7649.8ms/step | 2.6 samples/s +Step 170/11700 | Loss: 7.0453 | LR: 1.13e-04 | 7657.7ms/step | 2.6 samples/s +Step 180/11700 | Loss: 6.9226 | LR: 1.20e-04 | 7665.7ms/step | 2.6 samples/s +Step 190/11700 | Loss: 6.8475 | LR: 1.27e-04 | 7669.9ms/step | 2.6 samples/s +Step 200/11700 | Loss: 6.6835 | LR: 1.33e-04 | 7672.2ms/step | 2.6 samples/s +Step 210/11700 | Loss: 6.5853 | LR: 1.40e-04 | 7676.6ms/step | 2.6 samples/s +Step 220/11700 | Loss: 6.4342 | LR: 1.47e-04 | 7680.1ms/step | 2.6 samples/s +Step 230/11700 | Loss: 6.3811 | LR: 1.53e-04 | 7684.1ms/step | 2.6 samples/s +Step 240/11700 | Loss: 6.2366 | LR: 1.60e-04 | 7677.1ms/step | 2.6 samples/s +Step 250/11700 | Loss: 6.2542 | LR: 1.67e-04 | 7681.6ms/step | 2.6 samples/s +Validation at step 250: Loss = 6.1219, Perplexity = 455.74 +Step 260/11700 | Loss: 6.1660 | LR: 1.73e-04 | 21093.3ms/step | 0.9 samples/s +Step 270/11700 | Loss: 6.0283 | LR: 1.80e-04 | 7705.5ms/step | 2.6 samples/s +Step 280/11700 | Loss: 5.9767 | LR: 1.87e-04 | 7713.7ms/step | 2.6 samples/s +Step 290/11700 | Loss: 5.8484 | LR: 1.93e-04 | 7708.6ms/step | 2.6 samples/s +Step 300/11700 | Loss: 5.8329 | LR: 2.00e-04 | 7726.5ms/step | 2.6 samples/s +Step 310/11700 | Loss: 5.8043 | LR: 2.07e-04 | 7704.7ms/step | 2.6 samples/s +Step 320/11700 | Loss: 5.7098 | LR: 2.13e-04 | 7706.5ms/step | 2.6 samples/s +Step 330/11700 | Loss: 5.5965 | LR: 2.20e-04 | 7716.5ms/step | 2.6 samples/s +Step 340/11700 | Loss: 5.5504 | LR: 2.27e-04 | 7719.7ms/step | 2.6 samples/s +Step 350/11700 | Loss: 5.3912 | LR: 2.33e-04 | 7701.3ms/step | 2.6 samples/s +Step 360/11700 | Loss: 5.4623 | LR: 2.40e-04 | 7716.7ms/step | 2.6 samples/s +Step 370/11700 | Loss: 5.4942 | LR: 2.47e-04 | 7719.4ms/step | 2.6 samples/s +Step 380/11700 | Loss: 5.4840 | LR: 2.53e-04 | 7714.6ms/step | 2.6 samples/s +Step 390/11700 | Loss: 5.4318 | LR: 2.60e-04 | 7718.0ms/step | 2.6 samples/s +Step 400/11700 | Loss: 5.2920 | LR: 2.67e-04 | 7725.3ms/step | 2.6 samples/s +Step 410/11700 | Loss: 5.3159 | LR: 2.73e-04 | 7729.6ms/step | 2.6 samples/s +Step 420/11700 | Loss: 5.2723 | LR: 2.80e-04 | 7736.8ms/step | 2.6 samples/s +Step 430/11700 | Loss: 5.2204 | LR: 2.87e-04 | 7708.9ms/step | 2.6 samples/s +Step 440/11700 | Loss: 5.2186 | LR: 2.93e-04 | 7718.0ms/step | 2.6 samples/s +Step 450/11700 | Loss: 5.1471 | LR: 3.00e-04 | 7730.3ms/step | 2.6 samples/s +Step 460/11700 | Loss: 5.1613 | LR: 3.07e-04 | 7720.2ms/step | 2.6 samples/s +Step 470/11700 | Loss: 5.1233 | LR: 3.13e-04 | 7744.1ms/step | 2.6 samples/s +Step 480/11700 | Loss: 5.1200 | LR: 3.20e-04 | 7735.6ms/step | 2.6 samples/s +Step 490/11700 | Loss: 5.0295 | LR: 3.27e-04 | 7726.2ms/step | 2.6 samples/s +Step 500/11700 | Loss: 4.9870 | LR: 3.33e-04 | 7715.6ms/step | 2.6 samples/s +Validation at step 500: Loss = 4.9378, Perplexity = 139.46 +Step 510/11700 | Loss: 4.9054 | LR: 3.40e-04 | 19236.1ms/step | 1.0 samples/s +Step 520/11700 | Loss: 4.9075 | LR: 3.47e-04 | 7717.4ms/step | 2.6 samples/s +Step 530/11700 | Loss: 4.8455 | LR: 3.53e-04 | 7716.2ms/step | 2.6 samples/s +Step 540/11700 | Loss: 4.7478 | LR: 3.60e-04 | 7743.2ms/step | 2.6 samples/s +Step 550/11700 | Loss: 4.7653 | LR: 3.67e-04 | 7741.2ms/step | 2.6 samples/s +Step 560/11700 | Loss: 4.7563 | LR: 3.73e-04 | 7749.7ms/step | 2.6 samples/s +Step 570/11700 | Loss: 4.5215 | LR: 3.80e-04 | 7742.4ms/step | 2.6 samples/s +Step 580/11700 | Loss: 4.5547 | LR: 3.87e-04 | 7725.1ms/step | 2.6 samples/s +Step 590/11700 | Loss: 4.5127 | LR: 3.93e-04 | 7738.4ms/step | 2.6 samples/s +Step 600/11700 | Loss: 4.5684 | LR: 4.00e-04 | 7736.3ms/step | 2.6 samples/s +Step 610/11700 | Loss: 4.5378 | LR: 4.07e-04 | 7729.6ms/step | 2.6 samples/s +Step 620/11700 | Loss: 4.5248 | LR: 4.13e-04 | 7740.9ms/step | 2.6 samples/s +Step 630/11700 | Loss: 4.3725 | LR: 4.20e-04 | 7747.3ms/step | 2.6 samples/s +Step 640/11700 | Loss: 4.4640 | LR: 4.27e-04 | 7743.0ms/step | 2.6 samples/s +Step 650/11700 | Loss: 4.3467 | LR: 4.33e-04 | 7715.0ms/step | 2.6 samples/s +Step 660/11700 | Loss: 4.4468 | LR: 4.40e-04 | 7735.0ms/step | 2.6 samples/s +Step 670/11700 | Loss: 4.3328 | LR: 4.47e-04 | 7720.5ms/step | 2.6 samples/s +Step 680/11700 | Loss: 4.2429 | LR: 4.53e-04 | 7732.2ms/step | 2.6 samples/s +Step 690/11700 | Loss: 4.0877 | LR: 4.60e-04 | 7736.0ms/step | 2.6 samples/s +Step 700/11700 | Loss: 4.2427 | LR: 4.67e-04 | 7745.8ms/step | 2.6 samples/s +Step 710/11700 | Loss: 4.2394 | LR: 4.73e-04 | 7742.0ms/step | 2.6 samples/s +Step 720/11700 | Loss: 4.1194 | LR: 4.80e-04 | 7745.0ms/step | 2.6 samples/s +Step 730/11700 | Loss: 3.9965 | LR: 4.87e-04 | 7748.7ms/step | 2.6 samples/s +Step 740/11700 | Loss: 4.1403 | LR: 4.93e-04 | 7740.3ms/step | 2.6 samples/s +Step 750/11700 | Loss: 4.0257 | LR: 5.00e-04 | 7754.3ms/step | 2.6 samples/s +Validation at step 750: Loss = 3.9729, Perplexity = 53.14 +Step 760/11700 | Loss: 4.0513 | LR: 5.00e-04 | 19286.1ms/step | 1.0 samples/s +Step 770/11700 | Loss: 3.8243 | LR: 5.00e-04 | 7745.6ms/step | 2.6 samples/s +Step 780/11700 | Loss: 4.0187 | LR: 5.00e-04 | 7741.7ms/step | 2.6 samples/s +Step 790/11700 | Loss: 3.7950 | LR: 5.00e-04 | 7726.2ms/step | 2.6 samples/s +Step 800/11700 | Loss: 3.7916 | LR: 5.00e-04 | 7734.4ms/step | 2.6 samples/s +Step 810/11700 | Loss: 3.8328 | LR: 5.00e-04 | 7745.5ms/step | 2.6 samples/s +Step 820/11700 | Loss: 3.8211 | LR: 5.00e-04 | 7748.2ms/step | 2.6 samples/s +Step 830/11700 | Loss: 3.8070 | LR: 5.00e-04 | 7740.3ms/step | 2.6 samples/s +Step 840/11700 | Loss: 3.8021 | LR: 5.00e-04 | 7757.2ms/step | 2.6 samples/s +Step 850/11700 | Loss: 3.8368 | LR: 5.00e-04 | 7760.7ms/step | 2.6 samples/s +Step 860/11700 | Loss: 3.6708 | LR: 5.00e-04 | 7753.3ms/step | 2.6 samples/s +Step 870/11700 | Loss: 3.7118 | LR: 5.00e-04 | 7737.4ms/step | 2.6 samples/s +Step 880/11700 | Loss: 3.7644 | LR: 5.00e-04 | 7749.7ms/step | 2.6 samples/s +Step 890/11700 | Loss: 3.6054 | LR: 5.00e-04 | 7765.9ms/step | 2.6 samples/s +Step 900/11700 | Loss: 3.5428 | LR: 5.00e-04 | 7751.2ms/step | 2.6 samples/s +Step 910/11700 | Loss: 3.7092 | LR: 5.00e-04 | 7761.1ms/step | 2.6 samples/s +Step 920/11700 | Loss: 3.5774 | LR: 5.00e-04 | 7749.7ms/step | 2.6 samples/s +Step 930/11700 | Loss: 3.5651 | LR: 5.00e-04 | 7759.4ms/step | 2.6 samples/s +Step 940/11700 | Loss: 3.6396 | LR: 5.00e-04 | 7762.3ms/step | 2.6 samples/s +Step 950/11700 | Loss: 3.4800 | LR: 5.00e-04 | 7750.7ms/step | 2.6 samples/s +Step 960/11700 | Loss: 3.5553 | LR: 5.00e-04 | 7738.4ms/step | 2.6 samples/s +Step 970/11700 | Loss: 3.6108 | LR: 5.00e-04 | 7747.6ms/step | 2.6 samples/s +Step 980/11700 | Loss: 3.3437 | LR: 4.99e-04 | 7744.6ms/step | 2.6 samples/s +Step 990/11700 | Loss: 3.5572 | LR: 4.99e-04 | 7755.6ms/step | 2.6 samples/s +Step 1000/11700 | Loss: 3.4443 | LR: 4.99e-04 | 7740.1ms/step | 2.6 samples/s +Validation at step 1000: Loss = 3.4954, Perplexity = 32.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-1000 +Step 1010/11700 | Loss: 3.5579 | LR: 4.99e-04 | 19617.8ms/step | 1.0 samples/s +Step 1020/11700 | Loss: 3.4608 | LR: 4.99e-04 | 7758.4ms/step | 2.6 samples/s +Step 1030/11700 | Loss: 3.4865 | LR: 4.99e-04 | 7740.5ms/step | 2.6 samples/s +Step 1040/11700 | Loss: 3.5065 | LR: 4.99e-04 | 7758.5ms/step | 2.6 samples/s +Step 1050/11700 | Loss: 3.4233 | LR: 4.99e-04 | 7757.3ms/step | 2.6 samples/s +Step 1060/11700 | Loss: 3.4798 | LR: 4.99e-04 | 7735.3ms/step | 2.6 samples/s +Step 1070/11700 | Loss: 3.4378 | LR: 4.99e-04 | 7746.1ms/step | 2.6 samples/s +Step 1080/11700 | Loss: 3.2996 | LR: 4.99e-04 | 7759.9ms/step | 2.6 samples/s +Step 1090/11700 | Loss: 3.3530 | LR: 4.99e-04 | 7751.9ms/step | 2.6 samples/s +Step 1100/11700 | Loss: 3.5537 | LR: 4.99e-04 | 7762.2ms/step | 2.6 samples/s +Step 1110/11700 | Loss: 3.5042 | LR: 4.99e-04 | 7749.6ms/step | 2.6 samples/s +Step 1120/11700 | Loss: 3.3953 | LR: 4.99e-04 | 7741.2ms/step | 2.6 samples/s +Step 1130/11700 | Loss: 3.3990 | LR: 4.99e-04 | 7749.6ms/step | 2.6 samples/s +Step 1140/11700 | Loss: 3.1649 | LR: 4.98e-04 | 7749.7ms/step | 2.6 samples/s +Step 1150/11700 | Loss: 3.3282 | LR: 4.98e-04 | 7750.3ms/step | 2.6 samples/s +Step 1160/11700 | Loss: 3.3397 | LR: 4.98e-04 | 7749.2ms/step | 2.6 samples/s +Step 1170/11700 | Loss: 3.3018 | LR: 4.98e-04 | 7732.0ms/step | 2.6 samples/s +Step 1180/11700 | Loss: 3.2404 | LR: 4.98e-04 | 7740.3ms/step | 2.6 samples/s +Step 1190/11700 | Loss: 3.2017 | LR: 4.98e-04 | 7745.0ms/step | 2.6 samples/s +Step 1200/11700 | Loss: 3.1345 | LR: 4.98e-04 | 7747.7ms/step | 2.6 samples/s +Step 1210/11700 | Loss: 3.4671 | LR: 4.98e-04 | 7761.1ms/step | 2.6 samples/s +Step 1220/11700 | Loss: 3.2681 | LR: 4.98e-04 | 7749.0ms/step | 2.6 samples/s +Step 1230/11700 | Loss: 3.3052 | LR: 4.98e-04 | 7753.3ms/step | 2.6 samples/s +Step 1240/11700 | Loss: 3.2441 | LR: 4.98e-04 | 7734.4ms/step | 2.6 samples/s +Step 1250/11700 | Loss: 3.2245 | LR: 4.97e-04 | 7748.7ms/step | 2.6 samples/s +Validation at step 1250: Loss = 3.2738, Perplexity = 26.41 +Step 1260/11700 | Loss: 3.3045 | LR: 4.97e-04 | 19296.8ms/step | 1.0 samples/s +Step 1270/11700 | Loss: 3.1380 | LR: 4.97e-04 | 7749.9ms/step | 2.6 samples/s +Step 1280/11700 | Loss: 3.2447 | LR: 4.97e-04 | 7743.7ms/step | 2.6 samples/s +Step 1290/11700 | Loss: 3.2474 | LR: 4.97e-04 | 7757.3ms/step | 2.6 samples/s +Step 1300/11700 | Loss: 3.2476 | LR: 4.97e-04 | 7755.0ms/step | 2.6 samples/s +Step 1310/11700 | Loss: 3.2854 | LR: 4.97e-04 | 7757.5ms/step | 2.6 samples/s +Step 1320/11700 | Loss: 3.2521 | LR: 4.97e-04 | 7743.1ms/step | 2.6 samples/s +Step 1330/11700 | Loss: 3.1758 | LR: 4.97e-04 | 7739.0ms/step | 2.6 samples/s +Step 1340/11700 | Loss: 3.2265 | LR: 4.96e-04 | 7731.2ms/step | 2.6 samples/s +Step 1350/11700 | Loss: 3.2369 | LR: 4.96e-04 | 7758.0ms/step | 2.6 samples/s +Step 1360/11700 | Loss: 3.1989 | LR: 4.96e-04 | 7745.4ms/step | 2.6 samples/s +Step 1370/11700 | Loss: 3.1353 | LR: 4.96e-04 | 7730.8ms/step | 2.6 samples/s +Step 1380/11700 | Loss: 3.0074 | LR: 4.96e-04 | 7735.7ms/step | 2.6 samples/s +Step 1390/11700 | Loss: 3.1887 | LR: 4.96e-04 | 7744.3ms/step | 2.6 samples/s +Step 1400/11700 | Loss: 3.3018 | LR: 4.96e-04 | 7753.0ms/step | 2.6 samples/s +Step 1410/11700 | Loss: 3.2127 | LR: 4.96e-04 | 7738.8ms/step | 2.6 samples/s +Step 1420/11700 | Loss: 3.1101 | LR: 4.95e-04 | 7740.1ms/step | 2.6 samples/s +Step 1430/11700 | Loss: 3.1519 | LR: 4.95e-04 | 7733.4ms/step | 2.6 samples/s +Step 1440/11700 | Loss: 3.1423 | LR: 4.95e-04 | 7722.7ms/step | 2.6 samples/s +Step 1450/11700 | Loss: 3.0653 | LR: 4.95e-04 | 7721.9ms/step | 2.6 samples/s +Step 1460/11700 | Loss: 3.1189 | LR: 4.95e-04 | 7718.2ms/step | 2.6 samples/s +Step 1470/11700 | Loss: 3.1297 | LR: 4.95e-04 | 7717.5ms/step | 2.6 samples/s +Step 1480/11700 | Loss: 3.0623 | LR: 4.95e-04 | 7716.4ms/step | 2.6 samples/s +Step 1490/11700 | Loss: 3.2100 | LR: 4.94e-04 | 7718.4ms/step | 2.6 samples/s +Step 1500/11700 | Loss: 3.0466 | LR: 4.94e-04 | 7718.7ms/step | 2.6 samples/s +Validation at step 1500: Loss = 3.1303, Perplexity = 22.88 +Step 1510/11700 | Loss: 3.1255 | LR: 4.94e-04 | 19214.2ms/step | 1.0 samples/s +Step 1520/11700 | Loss: 3.1178 | LR: 4.94e-04 | 7717.9ms/step | 2.6 samples/s +Step 1530/11700 | Loss: 3.1247 | LR: 4.94e-04 | 7714.6ms/step | 2.6 samples/s +Step 1540/11700 | Loss: 3.2733 | LR: 4.94e-04 | 7715.6ms/step | 2.6 samples/s +Step 1550/11700 | Loss: 3.1211 | LR: 4.93e-04 | 7713.9ms/step | 2.6 samples/s +Step 1560/11700 | Loss: 3.1404 | LR: 4.93e-04 | 7717.1ms/step | 2.6 samples/s +Step 1570/11700 | Loss: 3.1272 | LR: 4.93e-04 | 7715.1ms/step | 2.6 samples/s +Step 1580/11700 | Loss: 3.0854 | LR: 4.93e-04 | 7718.1ms/step | 2.6 samples/s +Step 1590/11700 | Loss: 3.2201 | LR: 4.93e-04 | 7722.6ms/step | 2.6 samples/s +Step 1600/11700 | Loss: 3.1082 | LR: 4.93e-04 | 7726.9ms/step | 2.6 samples/s +Step 1610/11700 | Loss: 3.1493 | LR: 4.92e-04 | 7727.9ms/step | 2.6 samples/s +Step 1620/11700 | Loss: 3.0263 | LR: 4.92e-04 | 7726.7ms/step | 2.6 samples/s +Step 1630/11700 | Loss: 2.9366 | LR: 4.92e-04 | 7729.3ms/step | 2.6 samples/s +Step 1640/11700 | Loss: 3.1180 | LR: 4.92e-04 | 7727.9ms/step | 2.6 samples/s +Step 1650/11700 | Loss: 3.0492 | LR: 4.92e-04 | 7727.2ms/step | 2.6 samples/s +Step 1660/11700 | Loss: 3.0831 | LR: 4.92e-04 | 7724.6ms/step | 2.6 samples/s +Step 1670/11700 | Loss: 3.2078 | LR: 4.91e-04 | 7726.1ms/step | 2.6 samples/s +Step 1680/11700 | Loss: 3.0567 | LR: 4.91e-04 | 7725.0ms/step | 2.6 samples/s +Step 1690/11700 | Loss: 2.9181 | LR: 4.91e-04 | 7727.8ms/step | 2.6 samples/s +Step 1700/11700 | Loss: 2.9730 | LR: 4.91e-04 | 7729.6ms/step | 2.6 samples/s +Step 1710/11700 | Loss: 3.1066 | LR: 4.91e-04 | 7725.5ms/step | 2.6 samples/s +Step 1720/11700 | Loss: 3.0302 | LR: 4.90e-04 | 7724.3ms/step | 2.6 samples/s +Step 1730/11700 | Loss: 3.0550 | LR: 4.90e-04 | 7717.7ms/step | 2.6 samples/s +Step 1740/11700 | Loss: 2.9553 | LR: 4.90e-04 | 7724.0ms/step | 2.6 samples/s +Step 1750/11700 | Loss: 2.9241 | LR: 4.90e-04 | 7726.0ms/step | 2.6 samples/s +Validation at step 1750: Loss = 3.0454, Perplexity = 21.02 +Step 1760/11700 | Loss: 3.0235 | LR: 4.90e-04 | 19232.2ms/step | 1.0 samples/s +Step 1770/11700 | Loss: 3.0231 | LR: 4.89e-04 | 7716.8ms/step | 2.6 samples/s +Step 1780/11700 | Loss: 3.0465 | LR: 4.89e-04 | 7716.9ms/step | 2.6 samples/s +Step 1790/11700 | Loss: 3.0168 | LR: 4.89e-04 | 7717.2ms/step | 2.6 samples/s +Step 1800/11700 | Loss: 3.0394 | LR: 4.89e-04 | 7716.7ms/step | 2.6 samples/s +Step 1810/11700 | Loss: 3.1019 | LR: 4.89e-04 | 7714.4ms/step | 2.6 samples/s +Step 1820/11700 | Loss: 3.0397 | LR: 4.88e-04 | 7711.3ms/step | 2.6 samples/s +Step 1830/11700 | Loss: 3.0036 | LR: 4.88e-04 | 7714.1ms/step | 2.6 samples/s +Step 1840/11700 | Loss: 2.9925 | LR: 4.88e-04 | 7717.9ms/step | 2.6 samples/s +Step 1850/11700 | Loss: 3.0180 | LR: 4.88e-04 | 7717.8ms/step | 2.6 samples/s +Step 1860/11700 | Loss: 2.9950 | LR: 4.87e-04 | 7724.4ms/step | 2.6 samples/s +Step 1870/11700 | Loss: 2.9243 | LR: 4.87e-04 | 7720.2ms/step | 2.6 samples/s +Step 1880/11700 | Loss: 2.9448 | LR: 4.87e-04 | 7716.2ms/step | 2.6 samples/s +Step 1890/11700 | Loss: 3.0652 | LR: 4.87e-04 | 7712.1ms/step | 2.6 samples/s +Step 1900/11700 | Loss: 3.0094 | LR: 4.87e-04 | 7711.9ms/step | 2.6 samples/s +Step 1910/11700 | Loss: 3.0336 | LR: 4.86e-04 | 7705.0ms/step | 2.6 samples/s +Step 1920/11700 | Loss: 2.8256 | LR: 4.86e-04 | 7707.2ms/step | 2.6 samples/s +Step 1930/11700 | Loss: 3.0109 | LR: 4.86e-04 | 7705.3ms/step | 2.6 samples/s +Step 1940/11700 | Loss: 2.9488 | LR: 4.86e-04 | 7702.9ms/step | 2.6 samples/s +Step 1950/11700 | Loss: 2.8663 | LR: 4.85e-04 | 7701.3ms/step | 2.6 samples/s +Step 1960/11700 | Loss: 2.9363 | LR: 4.85e-04 | 7701.2ms/step | 2.6 samples/s +Step 1970/11700 | Loss: 2.9601 | LR: 4.85e-04 | 7704.9ms/step | 2.6 samples/s +Step 1980/11700 | Loss: 2.9415 | LR: 4.85e-04 | 7711.0ms/step | 2.6 samples/s +Step 1990/11700 | Loss: 2.8845 | LR: 4.84e-04 | 7718.8ms/step | 2.6 samples/s +Step 2000/11700 | Loss: 3.0976 | LR: 4.84e-04 | 7738.4ms/step | 2.6 samples/s +Validation at step 2000: Loss = 2.9675, Perplexity = 19.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-2000 +Step 2010/11700 | Loss: 2.9858 | LR: 4.84e-04 | 19609.6ms/step | 1.0 samples/s +Step 2020/11700 | Loss: 3.0210 | LR: 4.84e-04 | 7712.6ms/step | 2.6 samples/s +Step 2030/11700 | Loss: 2.9242 | LR: 4.83e-04 | 7711.4ms/step | 2.6 samples/s +Step 2040/11700 | Loss: 2.9689 | LR: 4.83e-04 | 7717.7ms/step | 2.6 samples/s +Step 2050/11700 | Loss: 2.8396 | LR: 4.83e-04 | 7719.3ms/step | 2.6 samples/s +Step 2060/11700 | Loss: 2.9108 | LR: 4.83e-04 | 7716.6ms/step | 2.6 samples/s +Step 2070/11700 | Loss: 2.9181 | LR: 4.82e-04 | 7714.4ms/step | 2.6 samples/s +Step 2080/11700 | Loss: 2.8970 | LR: 4.82e-04 | 7714.0ms/step | 2.6 samples/s +Step 2090/11700 | Loss: 2.9291 | LR: 4.82e-04 | 7717.8ms/step | 2.6 samples/s +Step 2100/11700 | Loss: 3.0159 | LR: 4.81e-04 | 7719.0ms/step | 2.6 samples/s +Step 2110/11700 | Loss: 3.0049 | LR: 4.81e-04 | 7718.3ms/step | 2.6 samples/s +Step 2120/11700 | Loss: 2.9440 | LR: 4.81e-04 | 7718.5ms/step | 2.6 samples/s +Step 2130/11700 | Loss: 2.9360 | LR: 4.81e-04 | 7717.0ms/step | 2.6 samples/s +Step 2140/11700 | Loss: 3.0475 | LR: 4.80e-04 | 7714.6ms/step | 2.6 samples/s +Step 2150/11700 | Loss: 2.9680 | LR: 4.80e-04 | 7715.5ms/step | 2.6 samples/s +Step 2160/11700 | Loss: 2.9293 | LR: 4.80e-04 | 7714.0ms/step | 2.6 samples/s +Step 2170/11700 | Loss: 2.8703 | LR: 4.80e-04 | 7714.4ms/step | 2.6 samples/s +Step 2180/11700 | Loss: 2.8978 | LR: 4.79e-04 | 7718.9ms/step | 2.6 samples/s +Step 2190/11700 | Loss: 2.8808 | LR: 4.79e-04 | 7722.0ms/step | 2.6 samples/s +Step 2200/11700 | Loss: 2.9465 | LR: 4.79e-04 | 7717.3ms/step | 2.6 samples/s +Step 2210/11700 | Loss: 2.9855 | LR: 4.78e-04 | 7717.1ms/step | 2.6 samples/s +Step 2220/11700 | Loss: 2.7951 | LR: 4.78e-04 | 7714.0ms/step | 2.6 samples/s +Step 2230/11700 | Loss: 2.8323 | LR: 4.78e-04 | 7714.4ms/step | 2.6 samples/s +Step 2240/11700 | Loss: 2.8158 | LR: 4.78e-04 | 7713.5ms/step | 2.6 samples/s +Step 2250/11700 | Loss: 3.0016 | LR: 4.77e-04 | 7714.4ms/step | 2.6 samples/s +Validation at step 2250: Loss = 2.9025, Perplexity = 18.22 +Step 2260/11700 | Loss: 2.7785 | LR: 4.77e-04 | 19205.7ms/step | 1.0 samples/s +Step 2270/11700 | Loss: 2.9910 | LR: 4.77e-04 | 7716.3ms/step | 2.6 samples/s +Step 2280/11700 | Loss: 2.9136 | LR: 4.76e-04 | 7716.6ms/step | 2.6 samples/s +Step 2290/11700 | Loss: 2.9289 | LR: 4.76e-04 | 7717.3ms/step | 2.6 samples/s +Step 2300/11700 | Loss: 2.8717 | LR: 4.76e-04 | 7718.3ms/step | 2.6 samples/s +Step 2310/11700 | Loss: 2.8606 | LR: 4.75e-04 | 7718.4ms/step | 2.6 samples/s +Step 2320/11700 | Loss: 3.0052 | LR: 4.75e-04 | 7714.6ms/step | 2.6 samples/s +Step 2330/11700 | Loss: 2.8291 | LR: 4.75e-04 | 7713.8ms/step | 2.6 samples/s +Step 2340/11700 | Loss: 2.8628 | LR: 4.74e-04 | 7716.4ms/step | 2.6 samples/s +Step 2350/11700 | Loss: 2.7823 | LR: 4.74e-04 | 7715.5ms/step | 2.6 samples/s +Step 2360/11700 | Loss: 2.8004 | LR: 4.74e-04 | 7717.4ms/step | 2.6 samples/s +Step 2370/11700 | Loss: 2.7834 | LR: 4.73e-04 | 7715.2ms/step | 2.6 samples/s +Step 2380/11700 | Loss: 2.7784 | LR: 4.73e-04 | 7717.2ms/step | 2.6 samples/s +Step 2390/11700 | Loss: 2.9225 | LR: 4.73e-04 | 7714.4ms/step | 2.6 samples/s +Step 2400/11700 | Loss: 2.8597 | LR: 4.73e-04 | 7714.1ms/step | 2.6 samples/s +Step 2410/11700 | Loss: 2.8766 | LR: 4.72e-04 | 7714.1ms/step | 2.6 samples/s +Step 2420/11700 | Loss: 2.7377 | LR: 4.72e-04 | 7714.1ms/step | 2.6 samples/s +Step 2430/11700 | Loss: 2.9091 | LR: 4.72e-04 | 7716.4ms/step | 2.6 samples/s +Step 2440/11700 | Loss: 2.9300 | LR: 4.71e-04 | 7714.6ms/step | 2.6 samples/s +Step 2450/11700 | Loss: 2.7659 | LR: 4.71e-04 | 7713.3ms/step | 2.6 samples/s +Step 2460/11700 | Loss: 2.8842 | LR: 4.71e-04 | 7714.5ms/step | 2.6 samples/s +Step 2470/11700 | Loss: 2.8932 | LR: 4.70e-04 | 7712.5ms/step | 2.6 samples/s +Step 2480/11700 | Loss: 2.8211 | LR: 4.70e-04 | 7715.6ms/step | 2.6 samples/s +Step 2490/11700 | Loss: 2.9127 | LR: 4.69e-04 | 7716.2ms/step | 2.6 samples/s +Step 2500/11700 | Loss: 2.7902 | LR: 4.69e-04 | 7709.2ms/step | 2.6 samples/s +Validation at step 2500: Loss = 2.8526, Perplexity = 17.33 +Step 2510/11700 | Loss: 2.7599 | LR: 4.69e-04 | 19174.4ms/step | 1.0 samples/s +Step 2520/11700 | Loss: 2.9161 | LR: 4.68e-04 | 7709.5ms/step | 2.6 samples/s +Step 2530/11700 | Loss: 2.8117 | LR: 4.68e-04 | 7711.0ms/step | 2.6 samples/s +Step 2540/11700 | Loss: 2.7472 | LR: 4.68e-04 | 7710.6ms/step | 2.6 samples/s +Step 2550/11700 | Loss: 2.9180 | LR: 4.67e-04 | 7707.6ms/step | 2.6 samples/s +Step 2560/11700 | Loss: 2.8611 | LR: 4.67e-04 | 7707.9ms/step | 2.6 samples/s +Step 2570/11700 | Loss: 2.8233 | LR: 4.67e-04 | 7709.3ms/step | 2.6 samples/s +Step 2580/11700 | Loss: 2.8447 | LR: 4.66e-04 | 7711.0ms/step | 2.6 samples/s +Step 2590/11700 | Loss: 2.7847 | LR: 4.66e-04 | 7709.6ms/step | 2.6 samples/s +Step 2600/11700 | Loss: 2.8124 | LR: 4.66e-04 | 7710.3ms/step | 2.6 samples/s +Step 2610/11700 | Loss: 2.7369 | LR: 4.65e-04 | 7707.8ms/step | 2.6 samples/s +Step 2620/11700 | Loss: 2.7982 | LR: 4.65e-04 | 7709.8ms/step | 2.6 samples/s +Step 2630/11700 | Loss: 2.7409 | LR: 4.65e-04 | 7707.6ms/step | 2.6 samples/s +Step 2640/11700 | Loss: 2.8477 | LR: 4.64e-04 | 7706.6ms/step | 2.6 samples/s +Step 2650/11700 | Loss: 2.9116 | LR: 4.64e-04 | 7709.8ms/step | 2.6 samples/s +Step 2660/11700 | Loss: 2.8430 | LR: 4.63e-04 | 7710.7ms/step | 2.6 samples/s +Step 2670/11700 | Loss: 2.7835 | LR: 4.63e-04 | 7711.6ms/step | 2.6 samples/s +Step 2680/11700 | Loss: 2.8270 | LR: 4.63e-04 | 7708.9ms/step | 2.6 samples/s +Step 2690/11700 | Loss: 2.9312 | LR: 4.62e-04 | 7709.1ms/step | 2.6 samples/s +Step 2700/11700 | Loss: 2.8220 | LR: 4.62e-04 | 7713.5ms/step | 2.6 samples/s +Step 2710/11700 | Loss: 2.8186 | LR: 4.62e-04 | 7710.5ms/step | 2.6 samples/s +Step 2720/11700 | Loss: 2.8269 | LR: 4.61e-04 | 7713.2ms/step | 2.6 samples/s +Step 2730/11700 | Loss: 2.7283 | LR: 4.61e-04 | 7710.7ms/step | 2.6 samples/s +Step 2740/11700 | Loss: 2.8155 | LR: 4.60e-04 | 7712.2ms/step | 2.6 samples/s +Step 2750/11700 | Loss: 2.8666 | LR: 4.60e-04 | 7710.5ms/step | 2.6 samples/s +Validation at step 2750: Loss = 2.8081, Perplexity = 16.58 +Step 2760/11700 | Loss: 2.6179 | LR: 4.60e-04 | 19185.1ms/step | 1.0 samples/s +Step 2770/11700 | Loss: 2.7682 | LR: 4.59e-04 | 7711.7ms/step | 2.6 samples/s +Step 2780/11700 | Loss: 2.7739 | LR: 4.59e-04 | 7708.4ms/step | 2.6 samples/s +Step 2790/11700 | Loss: 2.8229 | LR: 4.58e-04 | 7711.5ms/step | 2.6 samples/s +Step 2800/11700 | Loss: 2.7318 | LR: 4.58e-04 | 7709.8ms/step | 2.6 samples/s +Step 2810/11700 | Loss: 2.8317 | LR: 4.58e-04 | 7709.0ms/step | 2.6 samples/s +Step 2820/11700 | Loss: 2.8151 | LR: 4.57e-04 | 7708.1ms/step | 2.6 samples/s +Step 2830/11700 | Loss: 2.7838 | LR: 4.57e-04 | 7710.8ms/step | 2.6 samples/s +Step 2840/11700 | Loss: 2.8505 | LR: 4.56e-04 | 7709.1ms/step | 2.6 samples/s +Step 2850/11700 | Loss: 2.8951 | LR: 4.56e-04 | 7711.4ms/step | 2.6 samples/s +Step 2860/11700 | Loss: 2.7795 | LR: 4.56e-04 | 7709.9ms/step | 2.6 samples/s +Step 2870/11700 | Loss: 2.8504 | LR: 4.55e-04 | 7708.9ms/step | 2.6 samples/s +Step 2880/11700 | Loss: 2.5793 | LR: 4.55e-04 | 7708.8ms/step | 2.6 samples/s +Step 2890/11700 | Loss: 2.8273 | LR: 4.54e-04 | 7709.7ms/step | 2.6 samples/s +Step 2900/11700 | Loss: 2.7724 | LR: 4.54e-04 | 7708.7ms/step | 2.6 samples/s +Step 2910/11700 | Loss: 2.8468 | LR: 4.54e-04 | 7711.0ms/step | 2.6 samples/s +Step 2920/11700 | Loss: 2.7741 | LR: 4.53e-04 | 7712.1ms/step | 2.6 samples/s +Step 2930/11700 | Loss: 2.8561 | LR: 4.53e-04 | 7709.4ms/step | 2.6 samples/s +Step 2940/11700 | Loss: 2.7848 | LR: 4.52e-04 | 7707.6ms/step | 2.6 samples/s +Step 2950/11700 | Loss: 2.8354 | LR: 4.52e-04 | 7709.2ms/step | 2.6 samples/s +Step 2960/11700 | Loss: 2.7283 | LR: 4.51e-04 | 7705.9ms/step | 2.6 samples/s +Step 2970/11700 | Loss: 2.6797 | LR: 4.51e-04 | 7707.5ms/step | 2.6 samples/s +Step 2980/11700 | Loss: 2.7616 | LR: 4.51e-04 | 7701.6ms/step | 2.6 samples/s +Step 2990/11700 | Loss: 2.6946 | LR: 4.50e-04 | 7706.9ms/step | 2.6 samples/s +Step 3000/11700 | Loss: 2.7728 | LR: 4.50e-04 | 7709.9ms/step | 2.6 samples/s +Validation at step 3000: Loss = 2.7761, Perplexity = 16.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-3000 +Step 3010/11700 | Loss: 2.7421 | LR: 4.49e-04 | 19559.3ms/step | 1.0 samples/s +Step 3020/11700 | Loss: 2.8878 | LR: 4.49e-04 | 7715.6ms/step | 2.6 samples/s +Step 3030/11700 | Loss: 2.7227 | LR: 4.48e-04 | 7719.5ms/step | 2.6 samples/s +Step 3040/11700 | Loss: 2.8291 | LR: 4.48e-04 | 7718.4ms/step | 2.6 samples/s +Step 3050/11700 | Loss: 2.7641 | LR: 4.48e-04 | 7715.0ms/step | 2.6 samples/s +Step 3060/11700 | Loss: 2.9472 | LR: 4.47e-04 | 7716.0ms/step | 2.6 samples/s +Step 3070/11700 | Loss: 2.7255 | LR: 4.47e-04 | 7714.0ms/step | 2.6 samples/s +Step 3080/11700 | Loss: 2.7223 | LR: 4.46e-04 | 7712.4ms/step | 2.6 samples/s +Step 3090/11700 | Loss: 2.7288 | LR: 4.46e-04 | 7714.3ms/step | 2.6 samples/s +Step 3100/11700 | Loss: 2.8627 | LR: 4.45e-04 | 7713.6ms/step | 2.6 samples/s +Step 3110/11700 | Loss: 2.6390 | LR: 4.45e-04 | 7716.0ms/step | 2.6 samples/s +Step 3120/11700 | Loss: 2.7126 | LR: 4.44e-04 | 7715.4ms/step | 2.6 samples/s +Step 3130/11700 | Loss: 2.7487 | LR: 4.44e-04 | 7714.8ms/step | 2.6 samples/s +Step 3140/11700 | Loss: 2.7026 | LR: 4.43e-04 | 7715.0ms/step | 2.6 samples/s +Step 3150/11700 | Loss: 2.7208 | LR: 4.43e-04 | 7713.7ms/step | 2.6 samples/s +Step 3160/11700 | Loss: 2.7216 | LR: 4.43e-04 | 7710.9ms/step | 2.6 samples/s +Step 3170/11700 | Loss: 2.8129 | LR: 4.42e-04 | 7709.7ms/step | 2.6 samples/s +Step 3180/11700 | Loss: 2.7776 | LR: 4.42e-04 | 7712.9ms/step | 2.6 samples/s +Step 3190/11700 | Loss: 2.6493 | LR: 4.41e-04 | 7716.1ms/step | 2.6 samples/s +Step 3200/11700 | Loss: 2.6807 | LR: 4.41e-04 | 7716.2ms/step | 2.6 samples/s +Step 3210/11700 | Loss: 2.7042 | LR: 4.40e-04 | 7718.7ms/step | 2.6 samples/s +Step 3220/11700 | Loss: 2.7263 | LR: 4.40e-04 | 7717.4ms/step | 2.6 samples/s +Step 3230/11700 | Loss: 2.6777 | LR: 4.39e-04 | 7717.5ms/step | 2.6 samples/s +Step 3240/11700 | Loss: 2.7455 | LR: 4.39e-04 | 7714.9ms/step | 2.6 samples/s +Step 3250/11700 | Loss: 2.8320 | LR: 4.38e-04 | 7714.9ms/step | 2.6 samples/s +Validation at step 3250: Loss = 2.7376, Perplexity = 15.45 +Step 3260/11700 | Loss: 2.7464 | LR: 4.38e-04 | 19196.9ms/step | 1.0 samples/s +Step 3270/11700 | Loss: 2.6893 | LR: 4.37e-04 | 7715.5ms/step | 2.6 samples/s +Step 3280/11700 | Loss: 2.7472 | LR: 4.37e-04 | 7717.2ms/step | 2.6 samples/s +Step 3290/11700 | Loss: 2.6920 | LR: 4.37e-04 | 7716.4ms/step | 2.6 samples/s +Step 3300/11700 | Loss: 2.7550 | LR: 4.36e-04 | 7712.9ms/step | 2.6 samples/s +Step 3310/11700 | Loss: 2.7569 | LR: 4.36e-04 | 7714.5ms/step | 2.6 samples/s +Step 3320/11700 | Loss: 2.6759 | LR: 4.35e-04 | 7713.9ms/step | 2.6 samples/s +Step 3330/11700 | Loss: 2.6630 | LR: 4.35e-04 | 7715.6ms/step | 2.6 samples/s +Step 3340/11700 | Loss: 2.7988 | LR: 4.34e-04 | 7717.3ms/step | 2.6 samples/s +Step 3350/11700 | Loss: 2.6992 | LR: 4.34e-04 | 7713.1ms/step | 2.6 samples/s +Step 3360/11700 | Loss: 2.7821 | LR: 4.33e-04 | 7716.3ms/step | 2.6 samples/s +Step 3370/11700 | Loss: 2.6599 | LR: 4.33e-04 | 7715.0ms/step | 2.6 samples/s +Step 3380/11700 | Loss: 2.7246 | LR: 4.32e-04 | 7718.4ms/step | 2.6 samples/s +Step 3390/11700 | Loss: 2.6867 | LR: 4.32e-04 | 7720.8ms/step | 2.6 samples/s +Step 3400/11700 | Loss: 2.8002 | LR: 4.31e-04 | 7720.5ms/step | 2.6 samples/s +Step 3410/11700 | Loss: 2.7407 | LR: 4.31e-04 | 7722.9ms/step | 2.6 samples/s +Step 3420/11700 | Loss: 2.5686 | LR: 4.30e-04 | 7716.7ms/step | 2.6 samples/s +Step 3430/11700 | Loss: 2.7545 | LR: 4.30e-04 | 7718.9ms/step | 2.6 samples/s +Step 3440/11700 | Loss: 2.6100 | LR: 4.29e-04 | 7716.3ms/step | 2.6 samples/s +Step 3450/11700 | Loss: 2.7223 | LR: 4.29e-04 | 7718.4ms/step | 2.6 samples/s +Step 3460/11700 | Loss: 2.7068 | LR: 4.28e-04 | 7718.4ms/step | 2.6 samples/s +Step 3470/11700 | Loss: 2.5955 | LR: 4.28e-04 | 7717.0ms/step | 2.6 samples/s +Step 3480/11700 | Loss: 2.5179 | LR: 4.27e-04 | 7718.1ms/step | 2.6 samples/s +Step 3490/11700 | Loss: 2.8472 | LR: 4.27e-04 | 7715.2ms/step | 2.6 samples/s +Step 3500/11700 | Loss: 2.7590 | LR: 4.26e-04 | 7713.3ms/step | 2.6 samples/s +Validation at step 3500: Loss = 2.7072, Perplexity = 14.99 +Step 3510/11700 | Loss: 2.7364 | LR: 4.26e-04 | 19196.4ms/step | 1.0 samples/s +Step 3520/11700 | Loss: 2.7248 | LR: 4.25e-04 | 7715.7ms/step | 2.6 samples/s +Step 3530/11700 | Loss: 2.7368 | LR: 4.25e-04 | 7713.1ms/step | 2.6 samples/s +Step 3540/11700 | Loss: 2.7452 | LR: 4.24e-04 | 7709.6ms/step | 2.6 samples/s +Step 3550/11700 | Loss: 2.6640 | LR: 4.24e-04 | 7715.0ms/step | 2.6 samples/s +Step 3560/11700 | Loss: 2.8521 | LR: 4.23e-04 | 7716.5ms/step | 2.6 samples/s +Step 3570/11700 | Loss: 2.7210 | LR: 4.23e-04 | 7719.2ms/step | 2.6 samples/s +Step 3580/11700 | Loss: 2.7349 | LR: 4.22e-04 | 7714.2ms/step | 2.6 samples/s +Step 3590/11700 | Loss: 2.6364 | LR: 4.22e-04 | 7716.7ms/step | 2.6 samples/s +Step 3600/11700 | Loss: 2.6346 | LR: 4.21e-04 | 7719.1ms/step | 2.6 samples/s +Step 3610/11700 | Loss: 2.7654 | LR: 4.20e-04 | 7719.9ms/step | 2.6 samples/s +Step 3620/11700 | Loss: 2.6394 | LR: 4.20e-04 | 7718.9ms/step | 2.6 samples/s +Step 3630/11700 | Loss: 2.6439 | LR: 4.19e-04 | 7717.8ms/step | 2.6 samples/s +Step 3640/11700 | Loss: 2.5920 | LR: 4.19e-04 | 7718.4ms/step | 2.6 samples/s +Step 3650/11700 | Loss: 2.7211 | LR: 4.18e-04 | 7719.4ms/step | 2.6 samples/s +Step 3660/11700 | Loss: 2.6780 | LR: 4.18e-04 | 7716.8ms/step | 2.6 samples/s +Step 3670/11700 | Loss: 2.6334 | LR: 4.17e-04 | 7715.9ms/step | 2.6 samples/s +Step 3680/11700 | Loss: 2.7337 | LR: 4.17e-04 | 7717.3ms/step | 2.6 samples/s +Step 3690/11700 | Loss: 2.6729 | LR: 4.16e-04 | 7721.8ms/step | 2.6 samples/s +Step 3700/11700 | Loss: 2.7145 | LR: 4.16e-04 | 7720.0ms/step | 2.6 samples/s +Step 3710/11700 | Loss: 2.6142 | LR: 4.15e-04 | 7718.6ms/step | 2.6 samples/s +Step 3720/11700 | Loss: 2.6243 | LR: 4.15e-04 | 7718.3ms/step | 2.6 samples/s +Step 3730/11700 | Loss: 2.6864 | LR: 4.14e-04 | 7716.6ms/step | 2.6 samples/s +Step 3740/11700 | Loss: 2.6381 | LR: 4.14e-04 | 7715.8ms/step | 2.6 samples/s +Step 3750/11700 | Loss: 2.7138 | LR: 4.13e-04 | 7714.8ms/step | 2.6 samples/s +Validation at step 3750: Loss = 2.6806, Perplexity = 14.59 +Step 3760/11700 | Loss: 2.7229 | LR: 4.12e-04 | 19210.5ms/step | 1.0 samples/s +Step 3770/11700 | Loss: 2.7157 | LR: 4.12e-04 | 7721.9ms/step | 2.6 samples/s +Step 3780/11700 | Loss: 2.6655 | LR: 4.11e-04 | 7717.9ms/step | 2.6 samples/s +Step 3790/11700 | Loss: 2.8069 | LR: 4.11e-04 | 7715.5ms/step | 2.6 samples/s +Step 3800/11700 | Loss: 2.6959 | LR: 4.10e-04 | 7717.5ms/step | 2.6 samples/s +Step 3810/11700 | Loss: 2.6540 | LR: 4.10e-04 | 7714.1ms/step | 2.6 samples/s +Step 3820/11700 | Loss: 2.6414 | LR: 4.09e-04 | 7714.1ms/step | 2.6 samples/s +Step 3830/11700 | Loss: 2.6643 | LR: 4.09e-04 | 7716.2ms/step | 2.6 samples/s +Step 3840/11700 | Loss: 2.5925 | LR: 4.08e-04 | 7714.2ms/step | 2.6 samples/s +Step 3850/11700 | Loss: 2.7099 | LR: 4.07e-04 | 7714.6ms/step | 2.6 samples/s +Step 3860/11700 | Loss: 2.6287 | LR: 4.07e-04 | 7716.2ms/step | 2.6 samples/s +Step 3870/11700 | Loss: 2.6607 | LR: 4.06e-04 | 7714.3ms/step | 2.6 samples/s +Step 3880/11700 | Loss: 2.5449 | LR: 4.06e-04 | 7716.6ms/step | 2.6 samples/s +Step 3890/11700 | Loss: 2.7174 | LR: 4.05e-04 | 7715.7ms/step | 2.6 samples/s +Step 3900/11700 | Loss: 2.5914 | LR: 4.05e-04 | 7714.8ms/step | 2.6 samples/s +Step 3910/11700 | Loss: 2.6895 | LR: 4.04e-04 | 7714.2ms/step | 2.6 samples/s +Step 3920/11700 | Loss: 2.5903 | LR: 4.04e-04 | 7717.0ms/step | 2.6 samples/s +Step 3930/11700 | Loss: 2.6477 | LR: 4.03e-04 | 7719.8ms/step | 2.6 samples/s +Step 3940/11700 | Loss: 2.7823 | LR: 4.02e-04 | 7719.7ms/step | 2.6 samples/s +Step 3950/11700 | Loss: 2.6126 | LR: 4.02e-04 | 7715.8ms/step | 2.6 samples/s +Step 3960/11700 | Loss: 2.6623 | LR: 4.01e-04 | 7713.7ms/step | 2.6 samples/s +Step 3970/11700 | Loss: 2.6440 | LR: 4.01e-04 | 7706.2ms/step | 2.6 samples/s +Step 3980/11700 | Loss: 2.7042 | LR: 4.00e-04 | 7715.6ms/step | 2.6 samples/s +Step 3990/11700 | Loss: 2.6852 | LR: 4.00e-04 | 7715.8ms/step | 2.6 samples/s +Step 4000/11700 | Loss: 2.7364 | LR: 3.99e-04 | 7714.0ms/step | 2.6 samples/s +Validation at step 4000: Loss = 2.6616, Perplexity = 14.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-4000 +Step 4010/11700 | Loss: 2.5560 | LR: 3.98e-04 | 19524.0ms/step | 1.0 samples/s +Step 4020/11700 | Loss: 2.6539 | LR: 3.98e-04 | 7716.0ms/step | 2.6 samples/s +Step 4030/11700 | Loss: 2.5463 | LR: 3.97e-04 | 7718.2ms/step | 2.6 samples/s +Step 4040/11700 | Loss: 2.6863 | LR: 3.97e-04 | 7717.0ms/step | 2.6 samples/s +Step 4050/11700 | Loss: 2.6830 | LR: 3.96e-04 | 7712.9ms/step | 2.6 samples/s +Step 4060/11700 | Loss: 2.5578 | LR: 3.95e-04 | 7714.1ms/step | 2.6 samples/s +Step 4070/11700 | Loss: 2.7375 | LR: 3.95e-04 | 7715.8ms/step | 2.6 samples/s +Step 4080/11700 | Loss: 2.6267 | LR: 3.94e-04 | 7716.2ms/step | 2.6 samples/s +Step 4090/11700 | Loss: 2.6041 | LR: 3.94e-04 | 7717.3ms/step | 2.6 samples/s +Step 4100/11700 | Loss: 2.6523 | LR: 3.93e-04 | 7720.0ms/step | 2.6 samples/s +Step 4110/11700 | Loss: 2.6349 | LR: 3.93e-04 | 7719.6ms/step | 2.6 samples/s +Step 4120/11700 | Loss: 2.6988 | LR: 3.92e-04 | 7718.9ms/step | 2.6 samples/s +Step 4130/11700 | Loss: 2.5694 | LR: 3.91e-04 | 7711.8ms/step | 2.6 samples/s +Step 4140/11700 | Loss: 2.6197 | LR: 3.91e-04 | 7716.2ms/step | 2.6 samples/s +Step 4150/11700 | Loss: 2.5599 | LR: 3.90e-04 | 7718.1ms/step | 2.6 samples/s +Step 4160/11700 | Loss: 2.6303 | LR: 3.90e-04 | 7720.3ms/step | 2.6 samples/s +Step 4170/11700 | Loss: 2.6883 | LR: 3.89e-04 | 7719.4ms/step | 2.6 samples/s +Step 4180/11700 | Loss: 2.6264 | LR: 3.88e-04 | 7719.6ms/step | 2.6 samples/s +Step 4190/11700 | Loss: 2.6592 | LR: 3.88e-04 | 7720.7ms/step | 2.6 samples/s +Step 4200/11700 | Loss: 2.5726 | LR: 3.87e-04 | 7721.4ms/step | 2.6 samples/s +Step 4210/11700 | Loss: 2.6774 | LR: 3.87e-04 | 7719.0ms/step | 2.6 samples/s +Step 4220/11700 | Loss: 2.6415 | LR: 3.86e-04 | 7719.0ms/step | 2.6 samples/s +Step 4230/11700 | Loss: 2.5621 | LR: 3.85e-04 | 7720.1ms/step | 2.6 samples/s +Step 4240/11700 | Loss: 2.6710 | LR: 3.85e-04 | 7717.9ms/step | 2.6 samples/s +Step 4250/11700 | Loss: 2.7735 | LR: 3.84e-04 | 7717.6ms/step | 2.6 samples/s +Validation at step 4250: Loss = 2.6440, Perplexity = 14.07 +Step 4260/11700 | Loss: 2.7399 | LR: 3.84e-04 | 19205.9ms/step | 1.0 samples/s +Step 4270/11700 | Loss: 2.5886 | LR: 3.83e-04 | 7716.9ms/step | 2.6 samples/s +Step 4280/11700 | Loss: 2.6208 | LR: 3.82e-04 | 7717.8ms/step | 2.6 samples/s +Step 4290/11700 | Loss: 2.6278 | LR: 3.82e-04 | 7715.2ms/step | 2.6 samples/s +Step 4300/11700 | Loss: 2.6819 | LR: 3.81e-04 | 7709.7ms/step | 2.6 samples/s +Step 4310/11700 | Loss: 2.6059 | LR: 3.81e-04 | 7711.3ms/step | 2.6 samples/s +Step 4320/11700 | Loss: 2.5810 | LR: 3.80e-04 | 7714.6ms/step | 2.6 samples/s +Step 4330/11700 | Loss: 2.5922 | LR: 3.79e-04 | 7716.8ms/step | 2.6 samples/s +Step 4340/11700 | Loss: 2.6615 | LR: 3.79e-04 | 7715.8ms/step | 2.6 samples/s +Step 4350/11700 | Loss: 2.7378 | LR: 3.78e-04 | 7716.1ms/step | 2.6 samples/s +Step 4360/11700 | Loss: 2.5772 | LR: 3.77e-04 | 7715.6ms/step | 2.6 samples/s +Step 4370/11700 | Loss: 2.6086 | LR: 3.77e-04 | 7718.3ms/step | 2.6 samples/s +Step 4380/11700 | Loss: 2.5684 | LR: 3.76e-04 | 7716.2ms/step | 2.6 samples/s +Step 4390/11700 | Loss: 2.6421 | LR: 3.76e-04 | 7715.9ms/step | 2.6 samples/s +Step 4400/11700 | Loss: 2.6288 | LR: 3.75e-04 | 7716.1ms/step | 2.6 samples/s +Step 4410/11700 | Loss: 2.6481 | LR: 3.74e-04 | 7713.2ms/step | 2.6 samples/s +Step 4420/11700 | Loss: 2.6357 | LR: 3.74e-04 | 7713.4ms/step | 2.6 samples/s +Step 4430/11700 | Loss: 2.6620 | LR: 3.73e-04 | 7715.0ms/step | 2.6 samples/s +Step 4440/11700 | Loss: 2.6645 | LR: 3.73e-04 | 7714.9ms/step | 2.6 samples/s +Step 4450/11700 | Loss: 2.5937 | LR: 3.72e-04 | 7711.8ms/step | 2.6 samples/s +Step 4460/11700 | Loss: 2.7165 | LR: 3.71e-04 | 7716.8ms/step | 2.6 samples/s +Step 4470/11700 | Loss: 2.5258 | LR: 3.71e-04 | 7715.7ms/step | 2.6 samples/s +Step 4480/11700 | Loss: 2.5993 | LR: 3.70e-04 | 7713.2ms/step | 2.6 samples/s +Step 4490/11700 | Loss: 2.6097 | LR: 3.69e-04 | 7714.6ms/step | 2.6 samples/s +Step 4500/11700 | Loss: 2.6327 | LR: 3.69e-04 | 7711.7ms/step | 2.6 samples/s +Validation at step 4500: Loss = 2.6283, Perplexity = 13.85 +Step 4510/11700 | Loss: 2.5235 | LR: 3.68e-04 | 19201.2ms/step | 1.0 samples/s +Step 4520/11700 | Loss: 2.6448 | LR: 3.67e-04 | 7720.6ms/step | 2.6 samples/s +Step 4530/11700 | Loss: 2.7404 | LR: 3.67e-04 | 7717.3ms/step | 2.6 samples/s +Step 4540/11700 | Loss: 2.6650 | LR: 3.66e-04 | 7715.7ms/step | 2.6 samples/s +Step 4550/11700 | Loss: 2.5984 | LR: 3.66e-04 | 7718.0ms/step | 2.6 samples/s +Step 4560/11700 | Loss: 2.5697 | LR: 3.65e-04 | 7720.0ms/step | 2.6 samples/s +Step 4570/11700 | Loss: 2.5925 | LR: 3.64e-04 | 7720.2ms/step | 2.6 samples/s +Step 4580/11700 | Loss: 2.5609 | LR: 3.64e-04 | 7718.0ms/step | 2.6 samples/s +Step 4590/11700 | Loss: 2.7103 | LR: 3.63e-04 | 7719.2ms/step | 2.6 samples/s +Step 4600/11700 | Loss: 2.4534 | LR: 3.62e-04 | 7721.0ms/step | 2.6 samples/s +Step 4610/11700 | Loss: 2.5077 | LR: 3.62e-04 | 7724.6ms/step | 2.6 samples/s +Step 4620/11700 | Loss: 2.6173 | LR: 3.61e-04 | 7722.3ms/step | 2.6 samples/s +Step 4630/11700 | Loss: 2.6259 | LR: 3.60e-04 | 7719.0ms/step | 2.6 samples/s +Step 4640/11700 | Loss: 2.5724 | LR: 3.60e-04 | 7736.0ms/step | 2.6 samples/s +Step 4650/11700 | Loss: 2.5720 | LR: 3.59e-04 | 7721.5ms/step | 2.6 samples/s +Step 4660/11700 | Loss: 2.6425 | LR: 3.59e-04 | 7716.2ms/step | 2.6 samples/s +Step 4670/11700 | Loss: 2.5621 | LR: 3.58e-04 | 7720.7ms/step | 2.6 samples/s +Step 4680/11700 | Loss: 2.6481 | LR: 3.57e-04 | 7716.0ms/step | 2.6 samples/s +Step 4690/11700 | Loss: 2.5433 | LR: 3.57e-04 | 7718.8ms/step | 2.6 samples/s +Step 4700/11700 | Loss: 2.5439 | LR: 3.56e-04 | 7719.8ms/step | 2.6 samples/s +Step 4710/11700 | Loss: 2.7022 | LR: 3.55e-04 | 7721.1ms/step | 2.6 samples/s +Step 4720/11700 | Loss: 2.5688 | LR: 3.55e-04 | 7720.1ms/step | 2.6 samples/s +Step 4730/11700 | Loss: 2.6315 | LR: 3.54e-04 | 7718.1ms/step | 2.6 samples/s +Step 4740/11700 | Loss: 2.6422 | LR: 3.53e-04 | 7719.1ms/step | 2.6 samples/s +Step 4750/11700 | Loss: 2.5868 | LR: 3.53e-04 | 7720.2ms/step | 2.6 samples/s +Validation at step 4750: Loss = 2.6047, Perplexity = 13.53 +Step 4760/11700 | Loss: 2.6808 | LR: 3.52e-04 | 19215.9ms/step | 1.0 samples/s +Step 4770/11700 | Loss: 2.6380 | LR: 3.51e-04 | 7722.9ms/step | 2.6 samples/s +Step 4780/11700 | Loss: 2.6041 | LR: 3.51e-04 | 7718.5ms/step | 2.6 samples/s +Step 4790/11700 | Loss: 2.5810 | LR: 3.50e-04 | 7721.4ms/step | 2.6 samples/s +Step 4800/11700 | Loss: 2.5217 | LR: 3.49e-04 | 7724.4ms/step | 2.6 samples/s +Step 4810/11700 | Loss: 2.4948 | LR: 3.49e-04 | 7720.1ms/step | 2.6 samples/s +Step 4820/11700 | Loss: 2.5961 | LR: 3.48e-04 | 7724.8ms/step | 2.6 samples/s +Step 4830/11700 | Loss: 2.6159 | LR: 3.47e-04 | 7721.7ms/step | 2.6 samples/s +Step 4840/11700 | Loss: 2.6119 | LR: 3.47e-04 | 7722.0ms/step | 2.6 samples/s +Step 4850/11700 | Loss: 2.6202 | LR: 3.46e-04 | 7723.8ms/step | 2.6 samples/s +Step 4860/11700 | Loss: 2.5640 | LR: 3.45e-04 | 7726.6ms/step | 2.6 samples/s +Step 4870/11700 | Loss: 2.6902 | LR: 3.45e-04 | 7718.4ms/step | 2.6 samples/s +Step 4880/11700 | Loss: 2.6203 | LR: 3.44e-04 | 7720.0ms/step | 2.6 samples/s +Step 4890/11700 | Loss: 2.6811 | LR: 3.43e-04 | 7721.3ms/step | 2.6 samples/s +Step 4900/11700 | Loss: 2.6195 | LR: 3.43e-04 | 7720.2ms/step | 2.6 samples/s +Step 4910/11700 | Loss: 2.5678 | LR: 3.42e-04 | 7721.4ms/step | 2.6 samples/s +Step 4920/11700 | Loss: 2.5402 | LR: 3.41e-04 | 7722.3ms/step | 2.6 samples/s +Step 4930/11700 | Loss: 2.5083 | LR: 3.41e-04 | 7721.9ms/step | 2.6 samples/s +Step 4940/11700 | Loss: 2.6342 | LR: 3.40e-04 | 7721.2ms/step | 2.6 samples/s +Step 4950/11700 | Loss: 2.4520 | LR: 3.39e-04 | 7710.2ms/step | 2.6 samples/s +Step 4960/11700 | Loss: 2.6156 | LR: 3.39e-04 | 7714.1ms/step | 2.6 samples/s +Step 4970/11700 | Loss: 2.6009 | LR: 3.38e-04 | 7720.1ms/step | 2.6 samples/s +Step 4980/11700 | Loss: 2.7116 | LR: 3.37e-04 | 7722.6ms/step | 2.6 samples/s +Step 4990/11700 | Loss: 2.5289 | LR: 3.37e-04 | 7722.9ms/step | 2.6 samples/s +Step 5000/11700 | Loss: 2.6904 | LR: 3.36e-04 | 7722.5ms/step | 2.6 samples/s +Validation at step 5000: Loss = 2.5882, Perplexity = 13.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-5000 +Step 5010/11700 | Loss: 2.6925 | LR: 3.35e-04 | 19560.1ms/step | 1.0 samples/s +Step 5020/11700 | Loss: 2.6726 | LR: 3.35e-04 | 7719.7ms/step | 2.6 samples/s +Step 5030/11700 | Loss: 2.6659 | LR: 3.34e-04 | 7722.0ms/step | 2.6 samples/s +Step 5040/11700 | Loss: 2.4974 | LR: 3.33e-04 | 7720.9ms/step | 2.6 samples/s +Step 5050/11700 | Loss: 2.4383 | LR: 3.33e-04 | 7715.2ms/step | 2.6 samples/s +Step 5060/11700 | Loss: 2.6081 | LR: 3.32e-04 | 7712.2ms/step | 2.6 samples/s +Step 5070/11700 | Loss: 2.5672 | LR: 3.31e-04 | 7720.7ms/step | 2.6 samples/s +Step 5080/11700 | Loss: 2.7004 | LR: 3.31e-04 | 7720.3ms/step | 2.6 samples/s +Step 5090/11700 | Loss: 2.6020 | LR: 3.30e-04 | 7723.4ms/step | 2.6 samples/s +Step 5100/11700 | Loss: 2.6994 | LR: 3.29e-04 | 7720.8ms/step | 2.6 samples/s +Step 5110/11700 | Loss: 2.6000 | LR: 3.29e-04 | 7720.7ms/step | 2.6 samples/s +Step 5120/11700 | Loss: 2.5516 | LR: 3.28e-04 | 7720.7ms/step | 2.6 samples/s +Step 5130/11700 | Loss: 2.4659 | LR: 3.27e-04 | 7721.8ms/step | 2.6 samples/s +Step 5140/11700 | Loss: 2.5177 | LR: 3.27e-04 | 7721.9ms/step | 2.6 samples/s +Step 5150/11700 | Loss: 2.5863 | LR: 3.26e-04 | 7724.0ms/step | 2.6 samples/s +Step 5160/11700 | Loss: 2.5313 | LR: 3.25e-04 | 7722.3ms/step | 2.6 samples/s +Step 5170/11700 | Loss: 2.5376 | LR: 3.25e-04 | 7719.3ms/step | 2.6 samples/s +Step 5180/11700 | Loss: 2.5189 | LR: 3.24e-04 | 7718.1ms/step | 2.6 samples/s +Step 5190/11700 | Loss: 2.5913 | LR: 3.23e-04 | 7724.2ms/step | 2.6 samples/s +Step 5200/11700 | Loss: 2.4286 | LR: 3.22e-04 | 7722.4ms/step | 2.6 samples/s +Step 5210/11700 | Loss: 2.5437 | LR: 3.22e-04 | 7726.2ms/step | 2.6 samples/s +Step 5220/11700 | Loss: 2.5985 | LR: 3.21e-04 | 7722.8ms/step | 2.6 samples/s +Step 5230/11700 | Loss: 2.6746 | LR: 3.20e-04 | 7724.9ms/step | 2.6 samples/s +Step 5240/11700 | Loss: 2.5506 | LR: 3.20e-04 | 7723.8ms/step | 2.6 samples/s +Step 5250/11700 | Loss: 2.5896 | LR: 3.19e-04 | 7724.0ms/step | 2.6 samples/s +Validation at step 5250: Loss = 2.5714, Perplexity = 13.08 +Step 5260/11700 | Loss: 2.5822 | LR: 3.18e-04 | 19214.3ms/step | 1.0 samples/s +Step 5270/11700 | Loss: 2.5612 | LR: 3.18e-04 | 7722.1ms/step | 2.6 samples/s +Step 5280/11700 | Loss: 2.5177 | LR: 3.17e-04 | 7725.0ms/step | 2.6 samples/s +Step 5290/11700 | Loss: 2.5240 | LR: 3.16e-04 | 7719.2ms/step | 2.6 samples/s +Step 5300/11700 | Loss: 2.5436 | LR: 3.16e-04 | 7718.1ms/step | 2.6 samples/s +Step 5310/11700 | Loss: 2.5354 | LR: 3.15e-04 | 7720.3ms/step | 2.6 samples/s +Step 5320/11700 | Loss: 2.5096 | LR: 3.14e-04 | 7719.6ms/step | 2.6 samples/s +Step 5330/11700 | Loss: 2.4967 | LR: 3.13e-04 | 7719.3ms/step | 2.6 samples/s +Step 5340/11700 | Loss: 2.5308 | LR: 3.13e-04 | 7717.0ms/step | 2.6 samples/s +Step 5350/11700 | Loss: 2.5280 | LR: 3.12e-04 | 7722.9ms/step | 2.6 samples/s +Step 5360/11700 | Loss: 2.4557 | LR: 3.11e-04 | 7716.3ms/step | 2.6 samples/s +Step 5370/11700 | Loss: 2.5629 | LR: 3.11e-04 | 7720.4ms/step | 2.6 samples/s +Step 5380/11700 | Loss: 2.4602 | LR: 3.10e-04 | 7720.4ms/step | 2.6 samples/s +Step 5390/11700 | Loss: 2.5996 | LR: 3.09e-04 | 7720.2ms/step | 2.6 samples/s +Step 5400/11700 | Loss: 2.5090 | LR: 3.09e-04 | 7721.1ms/step | 2.6 samples/s +Step 5410/11700 | Loss: 2.4930 | LR: 3.08e-04 | 7719.7ms/step | 2.6 samples/s +Step 5420/11700 | Loss: 2.5601 | LR: 3.07e-04 | 7717.7ms/step | 2.6 samples/s +Step 5430/11700 | Loss: 2.6332 | LR: 3.07e-04 | 7715.9ms/step | 2.6 samples/s +Step 5440/11700 | Loss: 2.6383 | LR: 3.06e-04 | 7716.0ms/step | 2.6 samples/s +Step 5450/11700 | Loss: 2.5228 | LR: 3.05e-04 | 7718.5ms/step | 2.6 samples/s +Step 5460/11700 | Loss: 2.5616 | LR: 3.04e-04 | 7720.4ms/step | 2.6 samples/s +Step 5470/11700 | Loss: 2.4258 | LR: 3.04e-04 | 7719.5ms/step | 2.6 samples/s +Step 5480/11700 | Loss: 2.5287 | LR: 3.03e-04 | 7722.0ms/step | 2.6 samples/s +Step 5490/11700 | Loss: 2.6934 | LR: 3.02e-04 | 7717.7ms/step | 2.6 samples/s +Step 5500/11700 | Loss: 2.5979 | LR: 3.02e-04 | 7720.3ms/step | 2.6 samples/s +Validation at step 5500: Loss = 2.5559, Perplexity = 12.88 +Step 5510/11700 | Loss: 2.4350 | LR: 3.01e-04 | 19206.0ms/step | 1.0 samples/s +Step 5520/11700 | Loss: 2.4161 | LR: 3.00e-04 | 7721.8ms/step | 2.6 samples/s +Step 5530/11700 | Loss: 2.5737 | LR: 3.00e-04 | 7720.8ms/step | 2.6 samples/s +Step 5540/11700 | Loss: 2.5536 | LR: 2.99e-04 | 7719.4ms/step | 2.6 samples/s +Step 5550/11700 | Loss: 2.5611 | LR: 2.98e-04 | 7718.5ms/step | 2.6 samples/s +Step 5560/11700 | Loss: 2.6005 | LR: 2.97e-04 | 7717.0ms/step | 2.6 samples/s +Step 5570/11700 | Loss: 2.4746 | LR: 2.97e-04 | 7721.7ms/step | 2.6 samples/s +Step 5580/11700 | Loss: 2.5653 | LR: 2.96e-04 | 7720.9ms/step | 2.6 samples/s +Step 5590/11700 | Loss: 2.5913 | LR: 2.95e-04 | 7721.9ms/step | 2.6 samples/s +Step 5600/11700 | Loss: 2.5367 | LR: 2.95e-04 | 7721.5ms/step | 2.6 samples/s +Step 5610/11700 | Loss: 2.3941 | LR: 2.94e-04 | 7722.8ms/step | 2.6 samples/s +Step 5620/11700 | Loss: 2.5537 | LR: 2.93e-04 | 7726.3ms/step | 2.6 samples/s +Step 5630/11700 | Loss: 2.4441 | LR: 2.92e-04 | 7722.4ms/step | 2.6 samples/s +Step 5640/11700 | Loss: 2.5754 | LR: 2.92e-04 | 7721.3ms/step | 2.6 samples/s +Step 5650/11700 | Loss: 2.6290 | LR: 2.91e-04 | 7722.2ms/step | 2.6 samples/s +Step 5660/11700 | Loss: 2.5369 | LR: 2.90e-04 | 7722.2ms/step | 2.6 samples/s +Step 5670/11700 | Loss: 2.5196 | LR: 2.90e-04 | 7721.1ms/step | 2.6 samples/s +Step 5680/11700 | Loss: 2.5938 | LR: 2.89e-04 | 7721.6ms/step | 2.6 samples/s +Step 5690/11700 | Loss: 2.5071 | LR: 2.88e-04 | 7723.6ms/step | 2.6 samples/s +Step 5700/11700 | Loss: 2.5291 | LR: 2.88e-04 | 7724.4ms/step | 2.6 samples/s +Step 5710/11700 | Loss: 2.5035 | LR: 2.87e-04 | 7721.2ms/step | 2.6 samples/s +Step 5720/11700 | Loss: 2.4795 | LR: 2.86e-04 | 7718.9ms/step | 2.6 samples/s +Step 5730/11700 | Loss: 2.5940 | LR: 2.85e-04 | 7721.3ms/step | 2.6 samples/s +Step 5740/11700 | Loss: 2.4616 | LR: 2.85e-04 | 7720.2ms/step | 2.6 samples/s +Step 5750/11700 | Loss: 2.4970 | LR: 2.84e-04 | 7721.9ms/step | 2.6 samples/s +Validation at step 5750: Loss = 2.5448, Perplexity = 12.74 +Step 5760/11700 | Loss: 2.5357 | LR: 2.83e-04 | 19216.9ms/step | 1.0 samples/s +Step 5770/11700 | Loss: 2.6631 | LR: 2.83e-04 | 7717.8ms/step | 2.6 samples/s +Step 5780/11700 | Loss: 2.5033 | LR: 2.82e-04 | 7722.1ms/step | 2.6 samples/s +Step 5790/11700 | Loss: 2.6306 | LR: 2.81e-04 | 7719.7ms/step | 2.6 samples/s +Step 5800/11700 | Loss: 2.6255 | LR: 2.80e-04 | 7719.3ms/step | 2.6 samples/s +Step 5810/11700 | Loss: 2.5706 | LR: 2.80e-04 | 7721.5ms/step | 2.6 samples/s +Step 5820/11700 | Loss: 2.4624 | LR: 2.79e-04 | 7722.5ms/step | 2.6 samples/s +Step 5830/11700 | Loss: 2.5124 | LR: 2.78e-04 | 7716.7ms/step | 2.6 samples/s +Step 5840/11700 | Loss: 2.5031 | LR: 2.78e-04 | 7718.2ms/step | 2.6 samples/s +Step 5850/11700 | Loss: 2.5103 | LR: 2.77e-04 | 7715.6ms/step | 2.6 samples/s +Step 5860/11700 | Loss: 2.5819 | LR: 2.76e-04 | 7714.2ms/step | 2.6 samples/s +Step 5870/11700 | Loss: 2.5923 | LR: 2.75e-04 | 7717.4ms/step | 2.6 samples/s +Step 5880/11700 | Loss: 2.5760 | LR: 2.75e-04 | 7712.9ms/step | 2.6 samples/s +Step 5890/11700 | Loss: 2.5122 | LR: 2.74e-04 | 7716.2ms/step | 2.6 samples/s +Step 5900/11700 | Loss: 2.5238 | LR: 2.73e-04 | 7718.2ms/step | 2.6 samples/s +Step 5910/11700 | Loss: 2.5700 | LR: 2.73e-04 | 7722.8ms/step | 2.6 samples/s +Step 5920/11700 | Loss: 2.4785 | LR: 2.72e-04 | 7715.4ms/step | 2.6 samples/s +Step 5930/11700 | Loss: 2.5084 | LR: 2.71e-04 | 7710.8ms/step | 2.6 samples/s +Step 5940/11700 | Loss: 2.4368 | LR: 2.70e-04 | 7711.7ms/step | 2.6 samples/s +Step 5950/11700 | Loss: 2.6047 | LR: 2.70e-04 | 7711.6ms/step | 2.6 samples/s +Step 5960/11700 | Loss: 2.6292 | LR: 2.69e-04 | 7716.5ms/step | 2.6 samples/s +Step 5970/11700 | Loss: 2.5203 | LR: 2.68e-04 | 7719.5ms/step | 2.6 samples/s +Step 5980/11700 | Loss: 2.5397 | LR: 2.68e-04 | 7719.8ms/step | 2.6 samples/s +Step 5990/11700 | Loss: 2.5306 | LR: 2.67e-04 | 7715.8ms/step | 2.6 samples/s +Step 6000/11700 | Loss: 2.5254 | LR: 2.66e-04 | 7713.0ms/step | 2.6 samples/s +Validation at step 6000: Loss = 2.5337, Perplexity = 12.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-6000 +Step 6010/11700 | Loss: 2.5803 | LR: 2.65e-04 | 19537.5ms/step | 1.0 samples/s +Step 6020/11700 | Loss: 2.4420 | LR: 2.65e-04 | 7707.9ms/step | 2.6 samples/s +Step 6030/11700 | Loss: 2.5021 | LR: 2.64e-04 | 7710.7ms/step | 2.6 samples/s +Step 6040/11700 | Loss: 2.4662 | LR: 2.63e-04 | 7712.1ms/step | 2.6 samples/s +Step 6050/11700 | Loss: 2.4687 | LR: 2.63e-04 | 7716.6ms/step | 2.6 samples/s +Step 6060/11700 | Loss: 2.5715 | LR: 2.62e-04 | 7715.5ms/step | 2.6 samples/s +Step 6070/11700 | Loss: 2.5612 | LR: 2.61e-04 | 7713.9ms/step | 2.6 samples/s +Step 6080/11700 | Loss: 2.5905 | LR: 2.60e-04 | 7716.0ms/step | 2.6 samples/s +Step 6090/11700 | Loss: 2.4596 | LR: 2.60e-04 | 7715.8ms/step | 2.6 samples/s +Step 6100/11700 | Loss: 2.5330 | LR: 2.59e-04 | 7716.2ms/step | 2.6 samples/s +Step 6110/11700 | Loss: 2.4657 | LR: 2.58e-04 | 7714.0ms/step | 2.6 samples/s +Step 6120/11700 | Loss: 2.5917 | LR: 2.58e-04 | 7715.6ms/step | 2.6 samples/s +Step 6130/11700 | Loss: 2.5221 | LR: 2.57e-04 | 7713.8ms/step | 2.6 samples/s +Step 6140/11700 | Loss: 2.5328 | LR: 2.56e-04 | 7713.1ms/step | 2.6 samples/s +Step 6150/11700 | Loss: 2.5058 | LR: 2.55e-04 | 7716.8ms/step | 2.6 samples/s +Step 6160/11700 | Loss: 2.4438 | LR: 2.55e-04 | 7719.4ms/step | 2.6 samples/s +Step 6170/11700 | Loss: 2.4556 | LR: 2.54e-04 | 7713.4ms/step | 2.6 samples/s +Step 6180/11700 | Loss: 2.4228 | LR: 2.53e-04 | 7712.0ms/step | 2.6 samples/s +Step 6190/11700 | Loss: 2.4831 | LR: 2.53e-04 | 7712.0ms/step | 2.6 samples/s +Step 6200/11700 | Loss: 2.6099 | LR: 2.52e-04 | 7715.7ms/step | 2.6 samples/s +Step 6210/11700 | Loss: 2.5924 | LR: 2.51e-04 | 7716.3ms/step | 2.6 samples/s +Step 6220/11700 | Loss: 2.5394 | LR: 2.50e-04 | 7715.8ms/step | 2.6 samples/s +Step 6230/11700 | Loss: 2.6469 | LR: 2.50e-04 | 7716.3ms/step | 2.6 samples/s +Step 6240/11700 | Loss: 2.5055 | LR: 2.49e-04 | 7712.0ms/step | 2.6 samples/s +Step 6250/11700 | Loss: 2.5434 | LR: 2.48e-04 | 7713.8ms/step | 2.6 samples/s +Validation at step 6250: Loss = 2.5216, Perplexity = 12.45 +Step 6260/11700 | Loss: 2.4382 | LR: 2.47e-04 | 19204.4ms/step | 1.0 samples/s +Step 6270/11700 | Loss: 2.5341 | LR: 2.47e-04 | 7718.2ms/step | 2.6 samples/s +Step 6280/11700 | Loss: 2.5142 | LR: 2.46e-04 | 7715.3ms/step | 2.6 samples/s +Step 6290/11700 | Loss: 2.4916 | LR: 2.45e-04 | 7712.8ms/step | 2.6 samples/s +Step 6300/11700 | Loss: 2.4999 | LR: 2.45e-04 | 7714.6ms/step | 2.6 samples/s +Step 6310/11700 | Loss: 2.4935 | LR: 2.44e-04 | 7714.0ms/step | 2.6 samples/s +Step 6320/11700 | Loss: 2.5593 | LR: 2.43e-04 | 7718.7ms/step | 2.6 samples/s +Step 6330/11700 | Loss: 2.6215 | LR: 2.42e-04 | 7719.5ms/step | 2.6 samples/s +Step 6340/11700 | Loss: 2.5353 | LR: 2.42e-04 | 7717.8ms/step | 2.6 samples/s +Step 6350/11700 | Loss: 2.5418 | LR: 2.41e-04 | 7717.2ms/step | 2.6 samples/s +Step 6360/11700 | Loss: 2.4434 | LR: 2.40e-04 | 7716.7ms/step | 2.6 samples/s +Step 6370/11700 | Loss: 2.4257 | LR: 2.40e-04 | 7718.7ms/step | 2.6 samples/s +Step 6380/11700 | Loss: 2.3961 | LR: 2.39e-04 | 7720.2ms/step | 2.6 samples/s +Step 6390/11700 | Loss: 2.5280 | LR: 2.38e-04 | 7719.7ms/step | 2.6 samples/s +Step 6400/11700 | Loss: 2.5042 | LR: 2.37e-04 | 7727.0ms/step | 2.6 samples/s +Step 6410/11700 | Loss: 2.4333 | LR: 2.37e-04 | 7723.5ms/step | 2.6 samples/s +Step 6420/11700 | Loss: 2.4309 | LR: 2.36e-04 | 7716.7ms/step | 2.6 samples/s +Step 6430/11700 | Loss: 2.3267 | LR: 2.35e-04 | 7721.7ms/step | 2.6 samples/s +Step 6440/11700 | Loss: 2.4984 | LR: 2.35e-04 | 7719.0ms/step | 2.6 samples/s +Step 6450/11700 | Loss: 2.4384 | LR: 2.34e-04 | 7719.6ms/step | 2.6 samples/s +Step 6460/11700 | Loss: 2.5700 | LR: 2.33e-04 | 7719.8ms/step | 2.6 samples/s +Step 6470/11700 | Loss: 2.4347 | LR: 2.32e-04 | 7724.7ms/step | 2.6 samples/s +Step 6480/11700 | Loss: 2.6838 | LR: 2.32e-04 | 7728.3ms/step | 2.6 samples/s +Step 6490/11700 | Loss: 2.4904 | LR: 2.31e-04 | 7724.9ms/step | 2.6 samples/s +Step 6500/11700 | Loss: 2.5242 | LR: 2.30e-04 | 7725.7ms/step | 2.6 samples/s +Validation at step 6500: Loss = 2.5119, Perplexity = 12.33 +Step 6510/11700 | Loss: 2.5039 | LR: 2.30e-04 | 19238.7ms/step | 1.0 samples/s +Step 6520/11700 | Loss: 2.5041 | LR: 2.29e-04 | 7719.7ms/step | 2.6 samples/s +Step 6530/11700 | Loss: 2.4219 | LR: 2.28e-04 | 7721.1ms/step | 2.6 samples/s +Step 6540/11700 | Loss: 2.5747 | LR: 2.27e-04 | 7719.5ms/step | 2.6 samples/s +Step 6550/11700 | Loss: 2.4757 | LR: 2.27e-04 | 7718.3ms/step | 2.6 samples/s +Step 6560/11700 | Loss: 2.4263 | LR: 2.26e-04 | 7719.5ms/step | 2.6 samples/s +Step 6570/11700 | Loss: 2.3809 | LR: 2.25e-04 | 7717.9ms/step | 2.6 samples/s +Step 6580/11700 | Loss: 2.5411 | LR: 2.25e-04 | 7721.1ms/step | 2.6 samples/s +Step 6590/11700 | Loss: 2.5220 | LR: 2.24e-04 | 7723.8ms/step | 2.6 samples/s +Step 6600/11700 | Loss: 2.5348 | LR: 2.23e-04 | 7723.2ms/step | 2.6 samples/s +Step 6610/11700 | Loss: 2.6072 | LR: 2.22e-04 | 7720.4ms/step | 2.6 samples/s +Step 6620/11700 | Loss: 2.4071 | LR: 2.22e-04 | 7717.0ms/step | 2.6 samples/s +Step 6630/11700 | Loss: 2.5141 | LR: 2.21e-04 | 7715.0ms/step | 2.6 samples/s +Step 6640/11700 | Loss: 2.5221 | LR: 2.20e-04 | 7719.8ms/step | 2.6 samples/s +Step 6650/11700 | Loss: 2.4546 | LR: 2.20e-04 | 7718.4ms/step | 2.6 samples/s +Step 6660/11700 | Loss: 2.4780 | LR: 2.19e-04 | 7719.0ms/step | 2.6 samples/s +Step 6670/11700 | Loss: 2.4723 | LR: 2.18e-04 | 7718.1ms/step | 2.6 samples/s +Step 6680/11700 | Loss: 2.4691 | LR: 2.17e-04 | 7718.7ms/step | 2.6 samples/s +Step 6690/11700 | Loss: 2.5591 | LR: 2.17e-04 | 7721.0ms/step | 2.6 samples/s +Step 6700/11700 | Loss: 2.4978 | LR: 2.16e-04 | 7716.3ms/step | 2.6 samples/s +Step 6710/11700 | Loss: 2.5105 | LR: 2.15e-04 | 7718.2ms/step | 2.6 samples/s +Step 6720/11700 | Loss: 2.3939 | LR: 2.15e-04 | 7720.0ms/step | 2.6 samples/s +Step 6730/11700 | Loss: 2.4516 | LR: 2.14e-04 | 7717.1ms/step | 2.6 samples/s +Step 6740/11700 | Loss: 2.4693 | LR: 2.13e-04 | 7717.7ms/step | 2.6 samples/s +Step 6750/11700 | Loss: 2.4520 | LR: 2.12e-04 | 7718.4ms/step | 2.6 samples/s +Validation at step 6750: Loss = 2.5026, Perplexity = 12.21 +Step 6760/11700 | Loss: 2.5251 | LR: 2.12e-04 | 19224.0ms/step | 1.0 samples/s +Step 6770/11700 | Loss: 2.4921 | LR: 2.11e-04 | 7743.5ms/step | 2.6 samples/s +Step 6780/11700 | Loss: 2.4322 | LR: 2.10e-04 | 7713.6ms/step | 2.6 samples/s +Step 6790/11700 | Loss: 2.4377 | LR: 2.10e-04 | 7716.6ms/step | 2.6 samples/s +Step 6800/11700 | Loss: 2.5184 | LR: 2.09e-04 | 7726.5ms/step | 2.6 samples/s +Step 6810/11700 | Loss: 2.6861 | LR: 2.08e-04 | 7729.2ms/step | 2.6 samples/s +Step 6820/11700 | Loss: 2.5266 | LR: 2.08e-04 | 7721.5ms/step | 2.6 samples/s +Step 6830/11700 | Loss: 2.4372 | LR: 2.07e-04 | 7718.9ms/step | 2.6 samples/s +Step 6840/11700 | Loss: 2.4566 | LR: 2.06e-04 | 7720.1ms/step | 2.6 samples/s +Step 6850/11700 | Loss: 2.5301 | LR: 2.05e-04 | 7718.6ms/step | 2.6 samples/s +Step 6860/11700 | Loss: 2.4702 | LR: 2.05e-04 | 7718.3ms/step | 2.6 samples/s +Step 6870/11700 | Loss: 2.4578 | LR: 2.04e-04 | 7719.0ms/step | 2.6 samples/s +Step 6880/11700 | Loss: 2.4594 | LR: 2.03e-04 | 7721.2ms/step | 2.6 samples/s +Step 6890/11700 | Loss: 2.4803 | LR: 2.03e-04 | 7716.5ms/step | 2.6 samples/s +Step 6900/11700 | Loss: 2.6036 | LR: 2.02e-04 | 7716.6ms/step | 2.6 samples/s +Step 6910/11700 | Loss: 2.5710 | LR: 2.01e-04 | 7721.1ms/step | 2.6 samples/s +Step 6920/11700 | Loss: 2.4275 | LR: 2.00e-04 | 7707.9ms/step | 2.6 samples/s +Step 6930/11700 | Loss: 2.3760 | LR: 2.00e-04 | 7717.7ms/step | 2.6 samples/s +Step 6940/11700 | Loss: 2.5863 | LR: 1.99e-04 | 7721.1ms/step | 2.6 samples/s +Step 6950/11700 | Loss: 2.5347 | LR: 1.98e-04 | 7715.2ms/step | 2.6 samples/s +Step 6960/11700 | Loss: 2.4838 | LR: 1.98e-04 | 7722.0ms/step | 2.6 samples/s +Step 6970/11700 | Loss: 2.3307 | LR: 1.97e-04 | 7717.2ms/step | 2.6 samples/s +Step 6980/11700 | Loss: 2.4919 | LR: 1.96e-04 | 7721.4ms/step | 2.6 samples/s +Step 6990/11700 | Loss: 2.5594 | LR: 1.96e-04 | 7726.0ms/step | 2.6 samples/s +Step 7000/11700 | Loss: 2.4925 | LR: 1.95e-04 | 7715.5ms/step | 2.6 samples/s +Validation at step 7000: Loss = 2.4948, Perplexity = 12.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-7000 +Step 7010/11700 | Loss: 2.5951 | LR: 1.94e-04 | 19539.0ms/step | 1.0 samples/s +Step 7020/11700 | Loss: 2.4756 | LR: 1.93e-04 | 7711.3ms/step | 2.6 samples/s +Step 7030/11700 | Loss: 2.5686 | LR: 1.93e-04 | 7711.3ms/step | 2.6 samples/s +Step 7040/11700 | Loss: 2.5451 | LR: 1.92e-04 | 7710.7ms/step | 2.6 samples/s +Step 7050/11700 | Loss: 2.4170 | LR: 1.91e-04 | 7714.8ms/step | 2.6 samples/s +Step 7060/11700 | Loss: 2.5385 | LR: 1.91e-04 | 7719.0ms/step | 2.6 samples/s +Step 7070/11700 | Loss: 2.4664 | LR: 1.90e-04 | 7714.3ms/step | 2.6 samples/s +Step 7080/11700 | Loss: 2.4809 | LR: 1.89e-04 | 7709.3ms/step | 2.6 samples/s +Step 7090/11700 | Loss: 2.4835 | LR: 1.89e-04 | 7706.2ms/step | 2.6 samples/s +Step 7100/11700 | Loss: 2.5223 | LR: 1.88e-04 | 7716.8ms/step | 2.6 samples/s +Step 7110/11700 | Loss: 2.4628 | LR: 1.87e-04 | 7722.5ms/step | 2.6 samples/s +Step 7120/11700 | Loss: 2.4072 | LR: 1.87e-04 | 7719.9ms/step | 2.6 samples/s +Step 7130/11700 | Loss: 2.4150 | LR: 1.86e-04 | 7706.4ms/step | 2.6 samples/s +Step 7140/11700 | Loss: 2.4664 | LR: 1.85e-04 | 7693.0ms/step | 2.6 samples/s +Step 7150/11700 | Loss: 2.5466 | LR: 1.84e-04 | 7693.7ms/step | 2.6 samples/s +Step 7160/11700 | Loss: 2.4603 | LR: 1.84e-04 | 7696.9ms/step | 2.6 samples/s +Step 7170/11700 | Loss: 2.4982 | LR: 1.83e-04 | 7702.0ms/step | 2.6 samples/s +Step 7180/11700 | Loss: 2.4075 | LR: 1.82e-04 | 7708.9ms/step | 2.6 samples/s +Step 7190/11700 | Loss: 2.4080 | LR: 1.82e-04 | 7700.2ms/step | 2.6 samples/s +Step 7200/11700 | Loss: 2.3734 | LR: 1.81e-04 | 7693.0ms/step | 2.6 samples/s +Step 7210/11700 | Loss: 2.3176 | LR: 1.80e-04 | 7685.0ms/step | 2.6 samples/s +Step 7220/11700 | Loss: 2.4558 | LR: 1.80e-04 | 7698.7ms/step | 2.6 samples/s +Step 7230/11700 | Loss: 2.5001 | LR: 1.79e-04 | 7706.8ms/step | 2.6 samples/s +Step 7240/11700 | Loss: 2.4581 | LR: 1.78e-04 | 7702.3ms/step | 2.6 samples/s +Step 7250/11700 | Loss: 2.3292 | LR: 1.78e-04 | 7700.4ms/step | 2.6 samples/s +Validation at step 7250: Loss = 2.4871, Perplexity = 12.03 +Step 7260/11700 | Loss: 2.4478 | LR: 1.77e-04 | 19164.4ms/step | 1.0 samples/s +Step 7270/11700 | Loss: 2.5017 | LR: 1.76e-04 | 7697.6ms/step | 2.6 samples/s +Step 7280/11700 | Loss: 2.5066 | LR: 1.75e-04 | 7694.7ms/step | 2.6 samples/s +Step 7290/11700 | Loss: 2.5016 | LR: 1.75e-04 | 7684.6ms/step | 2.6 samples/s +Step 7300/11700 | Loss: 2.4233 | LR: 1.74e-04 | 7694.4ms/step | 2.6 samples/s +Step 7310/11700 | Loss: 2.5473 | LR: 1.73e-04 | 7707.5ms/step | 2.6 samples/s +Step 7320/11700 | Loss: 2.4460 | LR: 1.73e-04 | 7695.5ms/step | 2.6 samples/s +Step 7330/11700 | Loss: 2.5148 | LR: 1.72e-04 | 7691.5ms/step | 2.6 samples/s +Step 7340/11700 | Loss: 2.4542 | LR: 1.71e-04 | 7700.5ms/step | 2.6 samples/s +Step 7350/11700 | Loss: 2.4371 | LR: 1.71e-04 | 7677.0ms/step | 2.6 samples/s +Step 7360/11700 | Loss: 2.3903 | LR: 1.70e-04 | 7708.5ms/step | 2.6 samples/s +Step 7370/11700 | Loss: 2.4494 | LR: 1.69e-04 | 7694.6ms/step | 2.6 samples/s +Step 7380/11700 | Loss: 2.4878 | LR: 1.69e-04 | 7691.3ms/step | 2.6 samples/s +Step 7390/11700 | Loss: 2.4546 | LR: 1.68e-04 | 7687.6ms/step | 2.6 samples/s +Step 7400/11700 | Loss: 2.4886 | LR: 1.67e-04 | 7686.8ms/step | 2.6 samples/s +Step 7410/11700 | Loss: 2.4113 | LR: 1.67e-04 | 7713.6ms/step | 2.6 samples/s +Step 7420/11700 | Loss: 2.3948 | LR: 1.66e-04 | 7720.0ms/step | 2.6 samples/s +Step 7430/11700 | Loss: 2.4066 | LR: 1.65e-04 | 7718.2ms/step | 2.6 samples/s +Step 7440/11700 | Loss: 2.5102 | LR: 1.65e-04 | 7719.8ms/step | 2.6 samples/s +Step 7450/11700 | Loss: 2.4899 | LR: 1.64e-04 | 7721.0ms/step | 2.6 samples/s +Step 7460/11700 | Loss: 2.4973 | LR: 1.63e-04 | 7721.1ms/step | 2.6 samples/s +Step 7470/11700 | Loss: 2.4489 | LR: 1.63e-04 | 7721.0ms/step | 2.6 samples/s +Step 7480/11700 | Loss: 2.5814 | LR: 1.62e-04 | 7717.2ms/step | 2.6 samples/s +Step 7490/11700 | Loss: 2.4169 | LR: 1.61e-04 | 7721.4ms/step | 2.6 samples/s +Step 7500/11700 | Loss: 2.4069 | LR: 1.61e-04 | 7719.0ms/step | 2.6 samples/s +Validation at step 7500: Loss = 2.4813, Perplexity = 11.96 +Step 7510/11700 | Loss: 2.4913 | LR: 1.60e-04 | 19217.7ms/step | 1.0 samples/s +Step 7520/11700 | Loss: 2.5149 | LR: 1.59e-04 | 7720.4ms/step | 2.6 samples/s +Step 7530/11700 | Loss: 2.4184 | LR: 1.59e-04 | 7717.1ms/step | 2.6 samples/s +Step 7540/11700 | Loss: 2.4970 | LR: 1.58e-04 | 7740.1ms/step | 2.6 samples/s +Step 7550/11700 | Loss: 2.4429 | LR: 1.57e-04 | 7734.1ms/step | 2.6 samples/s +Step 7560/11700 | Loss: 2.5030 | LR: 1.57e-04 | 7734.3ms/step | 2.6 samples/s +Step 7570/11700 | Loss: 2.4626 | LR: 1.56e-04 | 7731.1ms/step | 2.6 samples/s +Step 7580/11700 | Loss: 2.4510 | LR: 1.55e-04 | 7733.4ms/step | 2.6 samples/s +Step 7590/11700 | Loss: 2.4319 | LR: 1.55e-04 | 7734.4ms/step | 2.6 samples/s +Step 7600/11700 | Loss: 2.4878 | LR: 1.54e-04 | 7735.9ms/step | 2.6 samples/s +Step 7610/11700 | Loss: 2.5823 | LR: 1.53e-04 | 7734.6ms/step | 2.6 samples/s +Step 7620/11700 | Loss: 2.5496 | LR: 1.53e-04 | 7730.4ms/step | 2.6 samples/s +Step 7630/11700 | Loss: 2.4238 | LR: 1.52e-04 | 7734.4ms/step | 2.6 samples/s +Step 7640/11700 | Loss: 2.4581 | LR: 1.51e-04 | 7734.3ms/step | 2.6 samples/s +Step 7650/11700 | Loss: 2.4939 | LR: 1.51e-04 | 7739.1ms/step | 2.6 samples/s +Step 7660/11700 | Loss: 2.4428 | LR: 1.50e-04 | 7739.5ms/step | 2.6 samples/s +Step 7670/11700 | Loss: 2.3800 | LR: 1.49e-04 | 7726.1ms/step | 2.6 samples/s +Step 7680/11700 | Loss: 2.4427 | LR: 1.49e-04 | 7724.5ms/step | 2.6 samples/s +Step 7690/11700 | Loss: 2.5223 | LR: 1.48e-04 | 7724.8ms/step | 2.6 samples/s +Step 7700/11700 | Loss: 2.5864 | LR: 1.47e-04 | 7722.5ms/step | 2.6 samples/s +Step 7710/11700 | Loss: 2.3891 | LR: 1.47e-04 | 7720.2ms/step | 2.6 samples/s +Step 7720/11700 | Loss: 2.4900 | LR: 1.46e-04 | 7720.7ms/step | 2.6 samples/s +Step 7730/11700 | Loss: 2.3844 | LR: 1.45e-04 | 7719.6ms/step | 2.6 samples/s +Step 7740/11700 | Loss: 2.5431 | LR: 1.45e-04 | 7719.3ms/step | 2.6 samples/s +Step 7750/11700 | Loss: 2.4592 | LR: 1.44e-04 | 7718.1ms/step | 2.6 samples/s +Validation at step 7750: Loss = 2.4746, Perplexity = 11.88 +Step 7760/11700 | Loss: 2.5756 | LR: 1.43e-04 | 19210.4ms/step | 1.0 samples/s +Step 7770/11700 | Loss: 2.4398 | LR: 1.43e-04 | 7719.4ms/step | 2.6 samples/s +Step 7780/11700 | Loss: 2.5353 | LR: 1.42e-04 | 7716.0ms/step | 2.6 samples/s +Step 7790/11700 | Loss: 2.4361 | LR: 1.41e-04 | 7715.3ms/step | 2.6 samples/s +Step 7800/11700 | Loss: 2.5205 | LR: 1.41e-04 | 7717.2ms/step | 2.6 samples/s +Step 7810/11700 | Loss: 2.3626 | LR: 1.40e-04 | 7717.3ms/step | 2.6 samples/s +Step 7820/11700 | Loss: 2.4066 | LR: 1.40e-04 | 7720.3ms/step | 2.6 samples/s +Step 7830/11700 | Loss: 2.4102 | LR: 1.39e-04 | 7717.1ms/step | 2.6 samples/s +Step 7840/11700 | Loss: 2.4712 | LR: 1.38e-04 | 7716.8ms/step | 2.6 samples/s +Step 7850/11700 | Loss: 2.4915 | LR: 1.38e-04 | 7714.8ms/step | 2.6 samples/s +Step 7860/11700 | Loss: 2.4880 | LR: 1.37e-04 | 7709.1ms/step | 2.6 samples/s +Step 7870/11700 | Loss: 2.5291 | LR: 1.36e-04 | 7714.9ms/step | 2.6 samples/s +Step 7880/11700 | Loss: 2.4851 | LR: 1.36e-04 | 7715.0ms/step | 2.6 samples/s +Step 7890/11700 | Loss: 2.4420 | LR: 1.35e-04 | 7714.3ms/step | 2.6 samples/s +Step 7900/11700 | Loss: 2.5435 | LR: 1.34e-04 | 7716.0ms/step | 2.6 samples/s +Step 7910/11700 | Loss: 2.3228 | LR: 1.34e-04 | 7701.1ms/step | 2.6 samples/s +Step 7920/11700 | Loss: 2.4256 | LR: 1.33e-04 | 7714.3ms/step | 2.6 samples/s +Step 7930/11700 | Loss: 2.4364 | LR: 1.33e-04 | 7713.0ms/step | 2.6 samples/s +Step 7940/11700 | Loss: 2.4865 | LR: 1.32e-04 | 7718.7ms/step | 2.6 samples/s +Step 7950/11700 | Loss: 2.5094 | LR: 1.31e-04 | 7717.2ms/step | 2.6 samples/s +Step 7960/11700 | Loss: 2.4356 | LR: 1.31e-04 | 7713.1ms/step | 2.6 samples/s +Step 7970/11700 | Loss: 2.3839 | LR: 1.30e-04 | 7715.1ms/step | 2.6 samples/s +Step 7980/11700 | Loss: 2.5378 | LR: 1.29e-04 | 7715.1ms/step | 2.6 samples/s +Step 7990/11700 | Loss: 2.5144 | LR: 1.29e-04 | 7715.0ms/step | 2.6 samples/s +Step 8000/11700 | Loss: 2.3971 | LR: 1.28e-04 | 7715.0ms/step | 2.6 samples/s +Validation at step 8000: Loss = 2.4698, Perplexity = 11.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-8000 +Step 8010/11700 | Loss: 2.4850 | LR: 1.27e-04 | 19515.8ms/step | 1.0 samples/s +Step 8020/11700 | Loss: 2.3603 | LR: 1.27e-04 | 7713.5ms/step | 2.6 samples/s +Step 8030/11700 | Loss: 2.4499 | LR: 1.26e-04 | 7718.7ms/step | 2.6 samples/s +Step 8040/11700 | Loss: 2.4566 | LR: 1.26e-04 | 7709.5ms/step | 2.6 samples/s +Step 8050/11700 | Loss: 2.4197 | LR: 1.25e-04 | 7712.4ms/step | 2.6 samples/s +Step 8060/11700 | Loss: 2.4474 | LR: 1.24e-04 | 7718.4ms/step | 2.6 samples/s +Step 8070/11700 | Loss: 2.5019 | LR: 1.24e-04 | 7716.1ms/step | 2.6 samples/s +Step 8080/11700 | Loss: 2.4997 | LR: 1.23e-04 | 7715.7ms/step | 2.6 samples/s +Step 8090/11700 | Loss: 2.4310 | LR: 1.23e-04 | 7712.9ms/step | 2.6 samples/s +Step 8100/11700 | Loss: 2.4676 | LR: 1.22e-04 | 7710.4ms/step | 2.6 samples/s +Step 8110/11700 | Loss: 2.4154 | LR: 1.21e-04 | 7715.4ms/step | 2.6 samples/s +Step 8120/11700 | Loss: 2.3953 | LR: 1.21e-04 | 7716.1ms/step | 2.6 samples/s +Step 8130/11700 | Loss: 2.3738 | LR: 1.20e-04 | 7714.3ms/step | 2.6 samples/s +Step 8140/11700 | Loss: 2.4277 | LR: 1.19e-04 | 7716.4ms/step | 2.6 samples/s +Step 8150/11700 | Loss: 2.3127 | LR: 1.19e-04 | 7718.4ms/step | 2.6 samples/s +Step 8160/11700 | Loss: 2.5410 | LR: 1.18e-04 | 7717.1ms/step | 2.6 samples/s +Step 8170/11700 | Loss: 2.4374 | LR: 1.18e-04 | 7716.1ms/step | 2.6 samples/s +Step 8180/11700 | Loss: 2.4965 | LR: 1.17e-04 | 7715.4ms/step | 2.6 samples/s +Step 8190/11700 | Loss: 2.4010 | LR: 1.16e-04 | 7714.0ms/step | 2.6 samples/s +Step 8200/11700 | Loss: 2.4532 | LR: 1.16e-04 | 7718.5ms/step | 2.6 samples/s +Step 8210/11700 | Loss: 2.4864 | LR: 1.15e-04 | 7721.4ms/step | 2.6 samples/s +Step 8220/11700 | Loss: 2.4864 | LR: 1.15e-04 | 7715.7ms/step | 2.6 samples/s +Step 8230/11700 | Loss: 2.4691 | LR: 1.14e-04 | 7719.3ms/step | 2.6 samples/s +Step 8240/11700 | Loss: 2.4398 | LR: 1.13e-04 | 7720.5ms/step | 2.6 samples/s +Step 8250/11700 | Loss: 2.4693 | LR: 1.13e-04 | 7717.7ms/step | 2.6 samples/s +Validation at step 8250: Loss = 2.4657, Perplexity = 11.77 +Step 8260/11700 | Loss: 2.4069 | LR: 1.12e-04 | 19196.9ms/step | 1.0 samples/s +Step 8270/11700 | Loss: 2.4163 | LR: 1.12e-04 | 7710.0ms/step | 2.6 samples/s +Step 8280/11700 | Loss: 2.5645 | LR: 1.11e-04 | 7715.8ms/step | 2.6 samples/s +Step 8290/11700 | Loss: 2.4268 | LR: 1.10e-04 | 7715.4ms/step | 2.6 samples/s +Step 8300/11700 | Loss: 2.4787 | LR: 1.10e-04 | 7717.3ms/step | 2.6 samples/s +Step 8310/11700 | Loss: 2.4113 | LR: 1.09e-04 | 7720.3ms/step | 2.6 samples/s +Step 8320/11700 | Loss: 2.4966 | LR: 1.09e-04 | 7717.2ms/step | 2.6 samples/s +Step 8330/11700 | Loss: 2.5032 | LR: 1.08e-04 | 7717.6ms/step | 2.6 samples/s +Step 8340/11700 | Loss: 2.4684 | LR: 1.07e-04 | 7716.7ms/step | 2.6 samples/s +Step 8350/11700 | Loss: 2.4815 | LR: 1.07e-04 | 7717.8ms/step | 2.6 samples/s +Step 8360/11700 | Loss: 2.4228 | LR: 1.06e-04 | 7719.4ms/step | 2.6 samples/s +Step 8370/11700 | Loss: 2.4132 | LR: 1.06e-04 | 7716.8ms/step | 2.6 samples/s +Step 8380/11700 | Loss: 2.4461 | LR: 1.05e-04 | 7717.7ms/step | 2.6 samples/s +Step 8390/11700 | Loss: 2.4526 | LR: 1.05e-04 | 7714.4ms/step | 2.6 samples/s +Step 8400/11700 | Loss: 2.5131 | LR: 1.04e-04 | 7716.0ms/step | 2.6 samples/s +Step 8410/11700 | Loss: 2.6352 | LR: 1.03e-04 | 7719.3ms/step | 2.6 samples/s +Step 8420/11700 | Loss: 2.3927 | LR: 1.03e-04 | 7719.4ms/step | 2.6 samples/s +Step 8430/11700 | Loss: 2.5133 | LR: 1.02e-04 | 7712.8ms/step | 2.6 samples/s +Step 8440/11700 | Loss: 2.5112 | LR: 1.02e-04 | 7714.7ms/step | 2.6 samples/s +Step 8450/11700 | Loss: 2.4550 | LR: 1.01e-04 | 7712.6ms/step | 2.6 samples/s +Step 8460/11700 | Loss: 2.4440 | LR: 1.00e-04 | 7712.0ms/step | 2.6 samples/s +Step 8470/11700 | Loss: 2.4398 | LR: 9.99e-05 | 7713.1ms/step | 2.6 samples/s +Step 8480/11700 | Loss: 2.4927 | LR: 9.93e-05 | 7713.1ms/step | 2.6 samples/s +Step 8490/11700 | Loss: 2.4441 | LR: 9.87e-05 | 7711.9ms/step | 2.6 samples/s +Step 8500/11700 | Loss: 2.3084 | LR: 9.82e-05 | 7711.9ms/step | 2.6 samples/s +Validation at step 8500: Loss = 2.4619, Perplexity = 11.73 +Step 8510/11700 | Loss: 2.4107 | LR: 9.76e-05 | 19197.0ms/step | 1.0 samples/s +Step 8520/11700 | Loss: 2.4575 | LR: 9.70e-05 | 7715.0ms/step | 2.6 samples/s +Step 8530/11700 | Loss: 2.4824 | LR: 9.65e-05 | 7715.9ms/step | 2.6 samples/s +Step 8540/11700 | Loss: 2.4529 | LR: 9.59e-05 | 7716.4ms/step | 2.6 samples/s +Step 8550/11700 | Loss: 2.3431 | LR: 9.53e-05 | 7714.6ms/step | 2.6 samples/s +Step 8560/11700 | Loss: 2.4540 | LR: 9.48e-05 | 7715.2ms/step | 2.6 samples/s +Step 8570/11700 | Loss: 2.4456 | LR: 9.42e-05 | 7714.1ms/step | 2.6 samples/s +Step 8580/11700 | Loss: 2.4315 | LR: 9.36e-05 | 7715.5ms/step | 2.6 samples/s +Step 8590/11700 | Loss: 2.4020 | LR: 9.31e-05 | 7712.5ms/step | 2.6 samples/s +Step 8600/11700 | Loss: 2.4731 | LR: 9.25e-05 | 7713.3ms/step | 2.6 samples/s +Step 8610/11700 | Loss: 2.4483 | LR: 9.20e-05 | 7711.9ms/step | 2.6 samples/s +Step 8620/11700 | Loss: 2.5025 | LR: 9.14e-05 | 7711.8ms/step | 2.6 samples/s +Step 8630/11700 | Loss: 2.4756 | LR: 9.09e-05 | 7709.3ms/step | 2.6 samples/s +Step 8640/11700 | Loss: 2.4122 | LR: 9.03e-05 | 7714.8ms/step | 2.6 samples/s +Step 8650/11700 | Loss: 2.3700 | LR: 8.98e-05 | 7714.9ms/step | 2.6 samples/s +Step 8660/11700 | Loss: 2.4331 | LR: 8.92e-05 | 7713.7ms/step | 2.6 samples/s +Step 8670/11700 | Loss: 2.5247 | LR: 8.87e-05 | 7717.6ms/step | 2.6 samples/s +Step 8680/11700 | Loss: 2.4024 | LR: 8.81e-05 | 7720.5ms/step | 2.6 samples/s +Step 8690/11700 | Loss: 2.5264 | LR: 8.76e-05 | 7711.9ms/step | 2.6 samples/s +Step 8700/11700 | Loss: 2.4512 | LR: 8.70e-05 | 7714.0ms/step | 2.6 samples/s +Step 8710/11700 | Loss: 2.3176 | LR: 8.65e-05 | 7719.7ms/step | 2.6 samples/s +Step 8720/11700 | Loss: 2.3917 | LR: 8.59e-05 | 7716.5ms/step | 2.6 samples/s +Step 8730/11700 | Loss: 2.4389 | LR: 8.54e-05 | 7715.6ms/step | 2.6 samples/s +Step 8740/11700 | Loss: 2.4680 | LR: 8.49e-05 | 7719.2ms/step | 2.6 samples/s +Step 8750/11700 | Loss: 2.4745 | LR: 8.43e-05 | 7712.8ms/step | 2.6 samples/s +Validation at step 8750: Loss = 2.4591, Perplexity = 11.69 +Step 8760/11700 | Loss: 2.3831 | LR: 8.38e-05 | 19199.5ms/step | 1.0 samples/s +Step 8770/11700 | Loss: 2.4408 | LR: 8.33e-05 | 7713.5ms/step | 2.6 samples/s +Step 8780/11700 | Loss: 2.4396 | LR: 8.27e-05 | 7715.7ms/step | 2.6 samples/s +Step 8790/11700 | Loss: 2.4786 | LR: 8.22e-05 | 7711.9ms/step | 2.6 samples/s +Step 8800/11700 | Loss: 2.5072 | LR: 8.17e-05 | 7709.5ms/step | 2.6 samples/s +Step 8810/11700 | Loss: 2.3616 | LR: 8.11e-05 | 7713.7ms/step | 2.6 samples/s +Step 8820/11700 | Loss: 2.3570 | LR: 8.06e-05 | 7714.7ms/step | 2.6 samples/s +Step 8830/11700 | Loss: 2.4162 | LR: 8.01e-05 | 7717.3ms/step | 2.6 samples/s +Step 8840/11700 | Loss: 2.4973 | LR: 7.95e-05 | 7714.8ms/step | 2.6 samples/s +Step 8850/11700 | Loss: 2.4642 | LR: 7.90e-05 | 7718.8ms/step | 2.6 samples/s +Step 8860/11700 | Loss: 2.3433 | LR: 7.85e-05 | 7717.8ms/step | 2.6 samples/s +Step 8870/11700 | Loss: 2.3987 | LR: 7.80e-05 | 7715.1ms/step | 2.6 samples/s +Step 8880/11700 | Loss: 2.4850 | LR: 7.75e-05 | 7712.5ms/step | 2.6 samples/s +Step 8890/11700 | Loss: 2.4459 | LR: 7.69e-05 | 7703.5ms/step | 2.6 samples/s +Step 8900/11700 | Loss: 2.4248 | LR: 7.64e-05 | 7709.0ms/step | 2.6 samples/s +Step 8910/11700 | Loss: 2.5221 | LR: 7.59e-05 | 7716.6ms/step | 2.6 samples/s +Step 8920/11700 | Loss: 2.5006 | LR: 7.54e-05 | 7717.7ms/step | 2.6 samples/s +Step 8930/11700 | Loss: 2.3848 | LR: 7.49e-05 | 7718.9ms/step | 2.6 samples/s +Step 8940/11700 | Loss: 2.4845 | LR: 7.44e-05 | 7717.5ms/step | 2.6 samples/s +Step 8950/11700 | Loss: 2.4151 | LR: 7.39e-05 | 7717.1ms/step | 2.6 samples/s +Step 8960/11700 | Loss: 2.4237 | LR: 7.34e-05 | 7716.7ms/step | 2.6 samples/s +Step 8970/11700 | Loss: 2.4320 | LR: 7.28e-05 | 7715.3ms/step | 2.6 samples/s +Step 8980/11700 | Loss: 2.4220 | LR: 7.23e-05 | 7717.9ms/step | 2.6 samples/s +Step 8990/11700 | Loss: 2.4595 | LR: 7.18e-05 | 7714.8ms/step | 2.6 samples/s +Step 9000/11700 | Loss: 2.4368 | LR: 7.13e-05 | 7718.6ms/step | 2.6 samples/s +Validation at step 9000: Loss = 2.4569, Perplexity = 11.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-9000 +Step 9010/11700 | Loss: 2.5245 | LR: 7.08e-05 | 19577.9ms/step | 1.0 samples/s +Step 9020/11700 | Loss: 2.4373 | LR: 7.03e-05 | 7714.5ms/step | 2.6 samples/s +Step 9030/11700 | Loss: 2.4009 | LR: 6.98e-05 | 7713.1ms/step | 2.6 samples/s +Step 9040/11700 | Loss: 2.3753 | LR: 6.93e-05 | 7719.5ms/step | 2.6 samples/s +Step 9050/11700 | Loss: 2.4707 | LR: 6.88e-05 | 7717.4ms/step | 2.6 samples/s +Step 9060/11700 | Loss: 2.4850 | LR: 6.83e-05 | 7718.3ms/step | 2.6 samples/s +Step 9070/11700 | Loss: 2.4983 | LR: 6.79e-05 | 7715.6ms/step | 2.6 samples/s +Step 9080/11700 | Loss: 2.3945 | LR: 6.74e-05 | 7713.1ms/step | 2.6 samples/s +Step 9090/11700 | Loss: 2.3810 | LR: 6.69e-05 | 7713.7ms/step | 2.6 samples/s +Step 9100/11700 | Loss: 2.4681 | LR: 6.64e-05 | 7711.3ms/step | 2.6 samples/s +Step 9110/11700 | Loss: 2.4675 | LR: 6.59e-05 | 7714.4ms/step | 2.6 samples/s +Step 9120/11700 | Loss: 2.4984 | LR: 6.54e-05 | 7714.1ms/step | 2.6 samples/s +Step 9130/11700 | Loss: 2.3642 | LR: 6.49e-05 | 7712.9ms/step | 2.6 samples/s +Step 9140/11700 | Loss: 2.3425 | LR: 6.45e-05 | 7702.6ms/step | 2.6 samples/s +Step 9150/11700 | Loss: 2.4541 | LR: 6.40e-05 | 7710.3ms/step | 2.6 samples/s +Step 9160/11700 | Loss: 2.5430 | LR: 6.35e-05 | 7701.9ms/step | 2.6 samples/s +Step 9170/11700 | Loss: 2.4931 | LR: 6.30e-05 | 7701.6ms/step | 2.6 samples/s +Step 9180/11700 | Loss: 2.4295 | LR: 6.25e-05 | 7708.4ms/step | 2.6 samples/s +Step 9190/11700 | Loss: 2.4519 | LR: 6.21e-05 | 7704.1ms/step | 2.6 samples/s +Step 9200/11700 | Loss: 2.3347 | LR: 6.16e-05 | 7705.3ms/step | 2.6 samples/s +Step 9210/11700 | Loss: 2.4398 | LR: 6.11e-05 | 7701.5ms/step | 2.6 samples/s +Step 9220/11700 | Loss: 2.5223 | LR: 6.07e-05 | 7700.6ms/step | 2.6 samples/s +Step 9230/11700 | Loss: 2.4295 | LR: 6.02e-05 | 7702.9ms/step | 2.6 samples/s +Step 9240/11700 | Loss: 2.4156 | LR: 5.97e-05 | 7706.3ms/step | 2.6 samples/s +Step 9250/11700 | Loss: 2.4470 | LR: 5.93e-05 | 7702.8ms/step | 2.6 samples/s +Validation at step 9250: Loss = 2.4552, Perplexity = 11.65 +Step 9260/11700 | Loss: 2.3702 | LR: 5.88e-05 | 19170.8ms/step | 1.0 samples/s +Step 9270/11700 | Loss: 2.4628 | LR: 5.83e-05 | 7704.4ms/step | 2.6 samples/s +Step 9280/11700 | Loss: 2.3184 | LR: 5.79e-05 | 7703.1ms/step | 2.6 samples/s +Step 9290/11700 | Loss: 2.4067 | LR: 5.74e-05 | 7709.9ms/step | 2.6 samples/s +Step 9300/11700 | Loss: 2.4617 | LR: 5.70e-05 | 7709.7ms/step | 2.6 samples/s +Step 9310/11700 | Loss: 2.3570 | LR: 5.65e-05 | 7705.7ms/step | 2.6 samples/s +Step 9320/11700 | Loss: 2.3999 | LR: 5.61e-05 | 7705.7ms/step | 2.6 samples/s +Step 9330/11700 | Loss: 2.3953 | LR: 5.56e-05 | 7705.0ms/step | 2.6 samples/s +Step 9340/11700 | Loss: 2.5060 | LR: 5.52e-05 | 7703.4ms/step | 2.6 samples/s +Step 9350/11700 | Loss: 2.3944 | LR: 5.47e-05 | 7709.5ms/step | 2.6 samples/s +Step 9360/11700 | Loss: 2.4264 | LR: 5.43e-05 | 7706.7ms/step | 2.6 samples/s +Step 9370/11700 | Loss: 2.4433 | LR: 5.38e-05 | 7703.1ms/step | 2.6 samples/s +Step 9380/11700 | Loss: 2.5465 | LR: 5.34e-05 | 7703.7ms/step | 2.6 samples/s +Step 9390/11700 | Loss: 2.4533 | LR: 5.29e-05 | 7712.2ms/step | 2.6 samples/s +Step 9400/11700 | Loss: 2.4302 | LR: 5.25e-05 | 7705.8ms/step | 2.6 samples/s +Step 9410/11700 | Loss: 2.4456 | LR: 5.20e-05 | 7709.3ms/step | 2.6 samples/s +Step 9420/11700 | Loss: 2.5089 | LR: 5.16e-05 | 7701.2ms/step | 2.6 samples/s +Step 9430/11700 | Loss: 2.3192 | LR: 5.12e-05 | 7705.0ms/step | 2.6 samples/s +Step 9440/11700 | Loss: 2.5359 | LR: 5.07e-05 | 7707.6ms/step | 2.6 samples/s +Step 9450/11700 | Loss: 2.4783 | LR: 5.03e-05 | 7705.3ms/step | 2.6 samples/s +Step 9460/11700 | Loss: 2.4782 | LR: 4.99e-05 | 7700.9ms/step | 2.6 samples/s +Step 9470/11700 | Loss: 2.4535 | LR: 4.94e-05 | 7705.7ms/step | 2.6 samples/s +Step 9480/11700 | Loss: 2.4118 | LR: 4.90e-05 | 7709.5ms/step | 2.6 samples/s +Step 9490/11700 | Loss: 2.5626 | LR: 4.86e-05 | 7705.8ms/step | 2.6 samples/s +Step 9500/11700 | Loss: 2.3689 | LR: 4.82e-05 | 7706.0ms/step | 2.6 samples/s +Validation at step 9500: Loss = 2.4539, Perplexity = 11.63 +Step 9510/11700 | Loss: 2.4562 | LR: 4.77e-05 | 19181.5ms/step | 1.0 samples/s +Step 9520/11700 | Loss: 2.4096 | LR: 4.73e-05 | 7700.4ms/step | 2.6 samples/s +Step 9530/11700 | Loss: 2.4124 | LR: 4.69e-05 | 7707.2ms/step | 2.6 samples/s +Step 9540/11700 | Loss: 2.3747 | LR: 4.65e-05 | 7705.6ms/step | 2.6 samples/s +Step 9550/11700 | Loss: 2.5802 | LR: 4.61e-05 | 7707.5ms/step | 2.6 samples/s +Step 9560/11700 | Loss: 2.3599 | LR: 4.57e-05 | 7704.1ms/step | 2.6 samples/s +Step 9570/11700 | Loss: 2.3970 | LR: 4.52e-05 | 7703.9ms/step | 2.6 samples/s +Step 9580/11700 | Loss: 2.5173 | LR: 4.48e-05 | 7699.5ms/step | 2.6 samples/s +Step 9590/11700 | Loss: 2.4486 | LR: 4.44e-05 | 7701.1ms/step | 2.6 samples/s +Step 9600/11700 | Loss: 2.4590 | LR: 4.40e-05 | 7705.4ms/step | 2.6 samples/s +Step 9610/11700 | Loss: 2.4581 | LR: 4.36e-05 | 7706.9ms/step | 2.6 samples/s +Step 9620/11700 | Loss: 2.4706 | LR: 4.32e-05 | 7707.0ms/step | 2.6 samples/s +Step 9630/11700 | Loss: 2.4179 | LR: 4.28e-05 | 7707.2ms/step | 2.6 samples/s +Step 9640/11700 | Loss: 2.4679 | LR: 4.24e-05 | 7706.1ms/step | 2.6 samples/s +Step 9650/11700 | Loss: 2.5466 | LR: 4.20e-05 | 7702.6ms/step | 2.6 samples/s +Step 9660/11700 | Loss: 2.5507 | LR: 4.16e-05 | 7692.1ms/step | 2.6 samples/s +Step 9670/11700 | Loss: 2.4491 | LR: 4.12e-05 | 7704.8ms/step | 2.6 samples/s +Step 9680/11700 | Loss: 2.2707 | LR: 4.08e-05 | 7704.1ms/step | 2.6 samples/s +Step 9690/11700 | Loss: 2.5089 | LR: 4.04e-05 | 7705.2ms/step | 2.6 samples/s +Step 9700/11700 | Loss: 2.3789 | LR: 4.00e-05 | 7714.7ms/step | 2.6 samples/s +Step 9710/11700 | Loss: 2.4302 | LR: 3.97e-05 | 7700.1ms/step | 2.6 samples/s +Step 9720/11700 | Loss: 2.5051 | LR: 3.93e-05 | 7705.1ms/step | 2.6 samples/s +Step 9730/11700 | Loss: 2.3471 | LR: 3.89e-05 | 7701.3ms/step | 2.6 samples/s +Step 9740/11700 | Loss: 2.5185 | LR: 3.85e-05 | 7704.3ms/step | 2.6 samples/s +Step 9750/11700 | Loss: 2.4419 | LR: 3.81e-05 | 7705.8ms/step | 2.6 samples/s +Validation at step 9750: Loss = 2.4530, Perplexity = 11.62 +Step 9760/11700 | Loss: 2.4270 | LR: 3.77e-05 | 19178.0ms/step | 1.0 samples/s +Step 9770/11700 | Loss: 2.3702 | LR: 3.74e-05 | 7708.2ms/step | 2.6 samples/s +Step 9780/11700 | Loss: 2.5654 | LR: 3.70e-05 | 7707.7ms/step | 2.6 samples/s +Step 9790/11700 | Loss: 2.4232 | LR: 3.66e-05 | 7705.6ms/step | 2.6 samples/s +Step 9800/11700 | Loss: 2.3719 | LR: 3.62e-05 | 7706.7ms/step | 2.6 samples/s +Step 9810/11700 | Loss: 2.4899 | LR: 3.59e-05 | 7703.6ms/step | 2.6 samples/s +Step 9820/11700 | Loss: 2.3372 | LR: 3.55e-05 | 7709.2ms/step | 2.6 samples/s +Step 9830/11700 | Loss: 2.3593 | LR: 3.51e-05 | 7707.3ms/step | 2.6 samples/s +Step 9840/11700 | Loss: 2.3922 | LR: 3.48e-05 | 7704.5ms/step | 2.6 samples/s +Step 9850/11700 | Loss: 2.4685 | LR: 3.44e-05 | 7707.5ms/step | 2.6 samples/s +Step 9860/11700 | Loss: 2.4474 | LR: 3.40e-05 | 7705.0ms/step | 2.6 samples/s +Step 9870/11700 | Loss: 2.3950 | LR: 3.37e-05 | 7703.5ms/step | 2.6 samples/s +Step 9880/11700 | Loss: 2.4353 | LR: 3.33e-05 | 7703.4ms/step | 2.6 samples/s +Step 9890/11700 | Loss: 2.3419 | LR: 3.30e-05 | 7701.9ms/step | 2.6 samples/s +Step 9900/11700 | Loss: 2.4758 | LR: 3.26e-05 | 7705.1ms/step | 2.6 samples/s +Step 9910/11700 | Loss: 2.4559 | LR: 3.22e-05 | 7704.9ms/step | 2.6 samples/s +Step 9920/11700 | Loss: 2.4227 | LR: 3.19e-05 | 7702.6ms/step | 2.6 samples/s +Step 9930/11700 | Loss: 2.5049 | LR: 3.15e-05 | 7689.1ms/step | 2.6 samples/s +Step 9940/11700 | Loss: 2.3164 | LR: 3.12e-05 | 7703.9ms/step | 2.6 samples/s +Step 9950/11700 | Loss: 2.4259 | LR: 3.09e-05 | 7708.7ms/step | 2.6 samples/s +Step 9960/11700 | Loss: 2.3728 | LR: 3.05e-05 | 7709.1ms/step | 2.6 samples/s +Step 9970/11700 | Loss: 2.4496 | LR: 3.02e-05 | 7701.2ms/step | 2.6 samples/s +Step 9980/11700 | Loss: 2.3795 | LR: 2.98e-05 | 7708.4ms/step | 2.6 samples/s +Step 9990/11700 | Loss: 2.3333 | LR: 2.95e-05 | 7705.2ms/step | 2.6 samples/s +Step 10000/11700 | Loss: 2.4480 | LR: 2.92e-05 | 7709.8ms/step | 2.6 samples/s +Validation at step 10000: Loss = 2.4525, Perplexity = 11.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-10000 +Step 10010/11700 | Loss: 2.5449 | LR: 2.88e-05 | 19552.1ms/step | 1.0 samples/s +Step 10020/11700 | Loss: 2.5302 | LR: 2.85e-05 | 7691.8ms/step | 2.6 samples/s +Step 10030/11700 | Loss: 2.4305 | LR: 2.82e-05 | 7696.4ms/step | 2.6 samples/s +Step 10040/11700 | Loss: 2.4814 | LR: 2.78e-05 | 7699.7ms/step | 2.6 samples/s +Step 10050/11700 | Loss: 2.5176 | LR: 2.75e-05 | 7696.9ms/step | 2.6 samples/s +Step 10060/11700 | Loss: 2.3921 | LR: 2.72e-05 | 7703.1ms/step | 2.6 samples/s +Step 10070/11700 | Loss: 2.5107 | LR: 2.68e-05 | 7701.8ms/step | 2.6 samples/s +Step 10080/11700 | Loss: 2.3864 | LR: 2.65e-05 | 7701.3ms/step | 2.6 samples/s +Step 10090/11700 | Loss: 2.4569 | LR: 2.62e-05 | 7693.0ms/step | 2.6 samples/s +Step 10100/11700 | Loss: 2.4196 | LR: 2.59e-05 | 7697.7ms/step | 2.6 samples/s +Step 10110/11700 | Loss: 2.4901 | LR: 2.56e-05 | 7700.3ms/step | 2.6 samples/s +Step 10120/11700 | Loss: 2.3582 | LR: 2.52e-05 | 7696.8ms/step | 2.6 samples/s +Step 10130/11700 | Loss: 2.5255 | LR: 2.49e-05 | 7702.4ms/step | 2.6 samples/s +Step 10140/11700 | Loss: 2.5584 | LR: 2.46e-05 | 7702.0ms/step | 2.6 samples/s +Step 10150/11700 | Loss: 2.4694 | LR: 2.43e-05 | 7698.0ms/step | 2.6 samples/s +Step 10160/11700 | Loss: 2.4539 | LR: 2.40e-05 | 7702.9ms/step | 2.6 samples/s +Step 10170/11700 | Loss: 2.4373 | LR: 2.37e-05 | 7698.0ms/step | 2.6 samples/s +Step 10180/11700 | Loss: 2.4605 | LR: 2.34e-05 | 7703.0ms/step | 2.6 samples/s +Step 10190/11700 | Loss: 2.3669 | LR: 2.31e-05 | 7707.6ms/step | 2.6 samples/s +Step 10200/11700 | Loss: 2.3933 | LR: 2.28e-05 | 7701.2ms/step | 2.6 samples/s +Step 10210/11700 | Loss: 2.4884 | LR: 2.25e-05 | 7698.1ms/step | 2.6 samples/s +Step 10220/11700 | Loss: 2.4461 | LR: 2.22e-05 | 7702.0ms/step | 2.6 samples/s +Step 10230/11700 | Loss: 2.5493 | LR: 2.19e-05 | 7698.2ms/step | 2.6 samples/s +Step 10240/11700 | Loss: 2.5371 | LR: 2.16e-05 | 7698.4ms/step | 2.6 samples/s +Step 10250/11700 | Loss: 2.4860 | LR: 2.13e-05 | 7701.3ms/step | 2.6 samples/s +Validation at step 10250: Loss = 2.4521, Perplexity = 11.61 +Step 10260/11700 | Loss: 2.4937 | LR: 2.10e-05 | 19166.4ms/step | 1.0 samples/s +Step 10270/11700 | Loss: 2.3966 | LR: 2.07e-05 | 7704.5ms/step | 2.6 samples/s +Step 10280/11700 | Loss: 2.4944 | LR: 2.05e-05 | 7704.7ms/step | 2.6 samples/s +Step 10290/11700 | Loss: 2.4788 | LR: 2.02e-05 | 7702.7ms/step | 2.6 samples/s +Step 10300/11700 | Loss: 2.3318 | LR: 1.99e-05 | 7700.3ms/step | 2.6 samples/s +Step 10310/11700 | Loss: 2.4386 | LR: 1.96e-05 | 7703.1ms/step | 2.6 samples/s +Step 10320/11700 | Loss: 2.3847 | LR: 1.93e-05 | 7703.5ms/step | 2.6 samples/s +Step 10330/11700 | Loss: 2.3758 | LR: 1.91e-05 | 7709.9ms/step | 2.6 samples/s +Step 10340/11700 | Loss: 2.5451 | LR: 1.88e-05 | 7705.0ms/step | 2.6 samples/s +Step 10350/11700 | Loss: 2.4757 | LR: 1.85e-05 | 7702.7ms/step | 2.6 samples/s +Step 10360/11700 | Loss: 2.4714 | LR: 1.82e-05 | 7701.8ms/step | 2.6 samples/s +Step 10370/11700 | Loss: 2.4522 | LR: 1.80e-05 | 7703.3ms/step | 2.6 samples/s +Step 10380/11700 | Loss: 2.2890 | LR: 1.77e-05 | 7701.1ms/step | 2.6 samples/s +Step 10390/11700 | Loss: 2.5852 | LR: 1.75e-05 | 7708.2ms/step | 2.6 samples/s +Step 10400/11700 | Loss: 2.4939 | LR: 1.72e-05 | 7711.8ms/step | 2.6 samples/s +Step 10410/11700 | Loss: 2.4434 | LR: 1.69e-05 | 7705.9ms/step | 2.6 samples/s +Step 10420/11700 | Loss: 2.4220 | LR: 1.67e-05 | 7687.9ms/step | 2.6 samples/s +Step 10430/11700 | Loss: 2.4157 | LR: 1.64e-05 | 7699.2ms/step | 2.6 samples/s +Step 10440/11700 | Loss: 2.3882 | LR: 1.62e-05 | 7698.7ms/step | 2.6 samples/s +Step 10450/11700 | Loss: 2.4650 | LR: 1.59e-05 | 7698.8ms/step | 2.6 samples/s +Step 10460/11700 | Loss: 2.4885 | LR: 1.57e-05 | 7703.7ms/step | 2.6 samples/s +Step 10470/11700 | Loss: 2.3756 | LR: 1.54e-05 | 7702.4ms/step | 2.6 samples/s +Step 10480/11700 | Loss: 2.4160 | LR: 1.52e-05 | 7702.5ms/step | 2.6 samples/s +Step 10490/11700 | Loss: 2.3577 | LR: 1.49e-05 | 7704.5ms/step | 2.6 samples/s +Step 10500/11700 | Loss: 2.4465 | LR: 1.47e-05 | 7701.0ms/step | 2.6 samples/s +Validation at step 10500: Loss = 2.4520, Perplexity = 11.61 +Step 10510/11700 | Loss: 2.5701 | LR: 1.44e-05 | 19163.5ms/step | 1.0 samples/s +Step 10520/11700 | Loss: 2.4714 | LR: 1.42e-05 | 7703.5ms/step | 2.6 samples/s +Step 10530/11700 | Loss: 2.4579 | LR: 1.40e-05 | 7701.7ms/step | 2.6 samples/s +Step 10540/11700 | Loss: 2.3073 | LR: 1.37e-05 | 7706.4ms/step | 2.6 samples/s +Step 10550/11700 | Loss: 2.4841 | LR: 1.35e-05 | 7698.4ms/step | 2.6 samples/s +Step 10560/11700 | Loss: 2.5833 | LR: 1.33e-05 | 7701.0ms/step | 2.6 samples/s +Step 10570/11700 | Loss: 2.3392 | LR: 1.30e-05 | 7699.8ms/step | 2.6 samples/s +Step 10580/11700 | Loss: 2.5154 | LR: 1.28e-05 | 7704.2ms/step | 2.6 samples/s +Step 10590/11700 | Loss: 2.3793 | LR: 1.26e-05 | 7700.2ms/step | 2.6 samples/s +Step 10600/11700 | Loss: 2.4676 | LR: 1.23e-05 | 7700.5ms/step | 2.6 samples/s +Step 10610/11700 | Loss: 2.4372 | LR: 1.21e-05 | 7701.3ms/step | 2.6 samples/s +Step 10620/11700 | Loss: 2.3563 | LR: 1.19e-05 | 7700.9ms/step | 2.6 samples/s +Step 10630/11700 | Loss: 2.4241 | LR: 1.17e-05 | 7702.5ms/step | 2.6 samples/s +Step 10640/11700 | Loss: 2.4080 | LR: 1.15e-05 | 7703.0ms/step | 2.6 samples/s +Step 10650/11700 | Loss: 2.4611 | LR: 1.13e-05 | 7701.0ms/step | 2.6 samples/s +Step 10660/11700 | Loss: 2.4137 | LR: 1.10e-05 | 7707.3ms/step | 2.6 samples/s +Step 10670/11700 | Loss: 2.4009 | LR: 1.08e-05 | 7693.8ms/step | 2.6 samples/s +Step 10680/11700 | Loss: 2.4768 | LR: 1.06e-05 | 7704.4ms/step | 2.6 samples/s +Step 10690/11700 | Loss: 2.4650 | LR: 1.04e-05 | 7702.5ms/step | 2.6 samples/s +Step 10700/11700 | Loss: 2.4336 | LR: 1.02e-05 | 7701.1ms/step | 2.6 samples/s +Step 10710/11700 | Loss: 2.4666 | LR: 1.00e-05 | 7698.4ms/step | 2.6 samples/s +Step 10720/11700 | Loss: 2.4017 | LR: 9.82e-06 | 7701.6ms/step | 2.6 samples/s +Step 10730/11700 | Loss: 2.5031 | LR: 9.62e-06 | 7700.1ms/step | 2.6 samples/s +Step 10740/11700 | Loss: 2.3314 | LR: 9.42e-06 | 7703.4ms/step | 2.6 samples/s +Step 10750/11700 | Loss: 2.3598 | LR: 9.23e-06 | 7702.5ms/step | 2.6 samples/s +Validation at step 10750: Loss = 2.4519, Perplexity = 11.61 +Step 10760/11700 | Loss: 2.5284 | LR: 9.04e-06 | 19168.2ms/step | 1.0 samples/s +Step 10770/11700 | Loss: 2.5031 | LR: 8.85e-06 | 7700.8ms/step | 2.6 samples/s +Step 10780/11700 | Loss: 2.5317 | LR: 8.66e-06 | 7705.3ms/step | 2.6 samples/s +Step 10790/11700 | Loss: 2.4418 | LR: 8.47e-06 | 7702.1ms/step | 2.6 samples/s +Step 10800/11700 | Loss: 2.5358 | LR: 8.29e-06 | 7704.5ms/step | 2.6 samples/s +Step 10810/11700 | Loss: 2.4331 | LR: 8.11e-06 | 7696.4ms/step | 2.6 samples/s +Step 10820/11700 | Loss: 2.5209 | LR: 7.93e-06 | 7697.7ms/step | 2.6 samples/s +Step 10830/11700 | Loss: 2.4200 | LR: 7.75e-06 | 7693.1ms/step | 2.6 samples/s +Step 10840/11700 | Loss: 2.4018 | LR: 7.57e-06 | 7695.6ms/step | 2.6 samples/s +Step 10850/11700 | Loss: 2.3528 | LR: 7.40e-06 | 7701.4ms/step | 2.6 samples/s +Step 10860/11700 | Loss: 2.3533 | LR: 7.22e-06 | 7706.6ms/step | 2.6 samples/s +Step 10870/11700 | Loss: 2.3819 | LR: 7.05e-06 | 7700.9ms/step | 2.6 samples/s +Step 10880/11700 | Loss: 2.3614 | LR: 6.89e-06 | 7702.0ms/step | 2.6 samples/s +Step 10890/11700 | Loss: 2.3476 | LR: 6.72e-06 | 7698.6ms/step | 2.6 samples/s +Step 10900/11700 | Loss: 2.4531 | LR: 6.56e-06 | 7698.2ms/step | 2.6 samples/s +Step 10910/11700 | Loss: 2.4933 | LR: 6.39e-06 | 7698.2ms/step | 2.6 samples/s +Step 10920/11700 | Loss: 2.4169 | LR: 6.23e-06 | 7708.5ms/step | 2.6 samples/s +Step 10930/11700 | Loss: 2.3385 | LR: 6.08e-06 | 7701.6ms/step | 2.6 samples/s +Step 10940/11700 | Loss: 2.3606 | LR: 5.92e-06 | 7706.6ms/step | 2.6 samples/s +Step 10950/11700 | Loss: 2.3672 | LR: 5.77e-06 | 7702.0ms/step | 2.6 samples/s +Step 10960/11700 | Loss: 2.5175 | LR: 5.61e-06 | 7703.5ms/step | 2.6 samples/s +Step 10970/11700 | Loss: 2.4739 | LR: 5.46e-06 | 7702.7ms/step | 2.6 samples/s +Step 10980/11700 | Loss: 2.4702 | LR: 5.31e-06 | 7703.9ms/step | 2.6 samples/s +Step 10990/11700 | Loss: 2.4089 | LR: 5.17e-06 | 7704.0ms/step | 2.6 samples/s +Step 11000/11700 | Loss: 2.3884 | LR: 5.02e-06 | 7702.3ms/step | 2.6 samples/s +Validation at step 11000: Loss = 2.4518, Perplexity = 11.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11000 +Step 11010/11700 | Loss: 2.3783 | LR: 4.88e-06 | 19544.6ms/step | 1.0 samples/s +Step 11020/11700 | Loss: 2.4470 | LR: 4.74e-06 | 7696.6ms/step | 2.6 samples/s +Step 11030/11700 | Loss: 2.4809 | LR: 4.60e-06 | 7695.9ms/step | 2.6 samples/s +Step 11040/11700 | Loss: 2.4720 | LR: 4.47e-06 | 7704.9ms/step | 2.6 samples/s +Step 11050/11700 | Loss: 2.4942 | LR: 4.33e-06 | 7692.8ms/step | 2.6 samples/s +Step 11060/11700 | Loss: 2.4278 | LR: 4.20e-06 | 7702.2ms/step | 2.6 samples/s +Step 11070/11700 | Loss: 2.3902 | LR: 4.07e-06 | 7700.8ms/step | 2.6 samples/s +Step 11080/11700 | Loss: 2.4501 | LR: 3.94e-06 | 7700.2ms/step | 2.6 samples/s +Step 11090/11700 | Loss: 2.4349 | LR: 3.82e-06 | 7700.8ms/step | 2.6 samples/s +Step 11100/11700 | Loss: 2.4803 | LR: 3.69e-06 | 7699.3ms/step | 2.6 samples/s +Step 11110/11700 | Loss: 2.4260 | LR: 3.57e-06 | 7704.3ms/step | 2.6 samples/s +Step 11120/11700 | Loss: 2.4703 | LR: 3.45e-06 | 7701.3ms/step | 2.6 samples/s +Step 11130/11700 | Loss: 2.5178 | LR: 3.34e-06 | 7698.0ms/step | 2.6 samples/s +Step 11140/11700 | Loss: 2.4209 | LR: 3.22e-06 | 7702.9ms/step | 2.6 samples/s +Step 11150/11700 | Loss: 2.4434 | LR: 3.11e-06 | 7704.5ms/step | 2.6 samples/s +Step 11160/11700 | Loss: 2.3901 | LR: 2.99e-06 | 7702.4ms/step | 2.6 samples/s +Step 11170/11700 | Loss: 2.5031 | LR: 2.88e-06 | 7705.7ms/step | 2.6 samples/s +Step 11180/11700 | Loss: 2.4145 | LR: 2.78e-06 | 7705.6ms/step | 2.6 samples/s +Step 11190/11700 | Loss: 2.4535 | LR: 2.67e-06 | 7710.9ms/step | 2.6 samples/s +Step 11200/11700 | Loss: 2.4476 | LR: 2.57e-06 | 7715.9ms/step | 2.6 samples/s +Step 11210/11700 | Loss: 2.4964 | LR: 2.47e-06 | 7718.2ms/step | 2.6 samples/s +Step 11220/11700 | Loss: 2.3891 | LR: 2.37e-06 | 7715.4ms/step | 2.6 samples/s +Step 11230/11700 | Loss: 2.3164 | LR: 2.27e-06 | 7710.0ms/step | 2.6 samples/s +Step 11240/11700 | Loss: 2.3842 | LR: 2.17e-06 | 7711.1ms/step | 2.6 samples/s +Step 11250/11700 | Loss: 2.4000 | LR: 2.08e-06 | 7718.7ms/step | 2.6 samples/s +Validation at step 11250: Loss = 2.4518, Perplexity = 11.61 +Step 11260/11700 | Loss: 2.4325 | LR: 1.99e-06 | 19201.8ms/step | 1.0 samples/s +Step 11270/11700 | Loss: 2.4776 | LR: 1.90e-06 | 7714.6ms/step | 2.6 samples/s +Step 11280/11700 | Loss: 2.4181 | LR: 1.81e-06 | 7710.4ms/step | 2.6 samples/s +Step 11290/11700 | Loss: 2.4872 | LR: 1.73e-06 | 7714.8ms/step | 2.6 samples/s +Step 11300/11700 | Loss: 2.4546 | LR: 1.64e-06 | 7717.0ms/step | 2.6 samples/s +Step 11310/11700 | Loss: 2.4231 | LR: 1.56e-06 | 7713.8ms/step | 2.6 samples/s +Step 11320/11700 | Loss: 2.4529 | LR: 1.48e-06 | 7709.7ms/step | 2.6 samples/s +Step 11330/11700 | Loss: 2.3795 | LR: 1.41e-06 | 7709.8ms/step | 2.6 samples/s +Step 11340/11700 | Loss: 2.3568 | LR: 1.33e-06 | 7709.2ms/step | 2.6 samples/s +Step 11350/11700 | Loss: 2.4806 | LR: 1.26e-06 | 7710.8ms/step | 2.6 samples/s +Step 11360/11700 | Loss: 2.4726 | LR: 1.19e-06 | 7707.4ms/step | 2.6 samples/s +Step 11370/11700 | Loss: 2.4328 | LR: 1.12e-06 | 7716.5ms/step | 2.6 samples/s +Step 11380/11700 | Loss: 2.4193 | LR: 1.05e-06 | 7713.0ms/step | 2.6 samples/s +Step 11390/11700 | Loss: 2.4377 | LR: 9.88e-07 | 7711.1ms/step | 2.6 samples/s +Step 11400/11700 | Loss: 2.4647 | LR: 9.25e-07 | 7710.0ms/step | 2.6 samples/s +Step 11410/11700 | Loss: 2.4407 | LR: 8.65e-07 | 7709.6ms/step | 2.6 samples/s +Step 11420/11700 | Loss: 2.2900 | LR: 8.06e-07 | 7708.8ms/step | 2.6 samples/s +Step 11430/11700 | Loss: 2.3918 | LR: 7.50e-07 | 7713.3ms/step | 2.6 samples/s +Step 11440/11700 | Loss: 2.4817 | LR: 6.95e-07 | 7714.7ms/step | 2.6 samples/s +Step 11450/11700 | Loss: 2.3282 | LR: 6.43e-07 | 7709.0ms/step | 2.6 samples/s +Step 11460/11700 | Loss: 2.4531 | LR: 5.92e-07 | 7710.0ms/step | 2.6 samples/s +Step 11470/11700 | Loss: 2.4988 | LR: 5.44e-07 | 7712.0ms/step | 2.6 samples/s +Step 11480/11700 | Loss: 2.3685 | LR: 4.98e-07 | 7718.0ms/step | 2.6 samples/s +Step 11490/11700 | Loss: 2.4695 | LR: 4.54e-07 | 7716.6ms/step | 2.6 samples/s +Step 11500/11700 | Loss: 2.4040 | LR: 4.11e-07 | 7715.3ms/step | 2.6 samples/s +Validation at step 11500: Loss = 2.4518, Perplexity = 11.61 +Step 11510/11700 | Loss: 2.3895 | LR: 3.71e-07 | 19186.4ms/step | 1.0 samples/s +Step 11520/11700 | Loss: 2.4635 | LR: 3.33e-07 | 7710.6ms/step | 2.6 samples/s +Step 11530/11700 | Loss: 2.4084 | LR: 2.97e-07 | 7713.4ms/step | 2.6 samples/s +Step 11540/11700 | Loss: 2.3849 | LR: 2.63e-07 | 7715.6ms/step | 2.6 samples/s +Step 11550/11700 | Loss: 2.5060 | LR: 2.31e-07 | 7717.8ms/step | 2.6 samples/s +Step 11560/11700 | Loss: 2.4039 | LR: 2.02e-07 | 7710.8ms/step | 2.6 samples/s +Step 11570/11700 | Loss: 2.5163 | LR: 1.74e-07 | 7717.5ms/step | 2.6 samples/s +Step 11580/11700 | Loss: 2.3533 | LR: 1.48e-07 | 7716.9ms/step | 2.6 samples/s +Step 11590/11700 | Loss: 2.4703 | LR: 1.24e-07 | 7713.2ms/step | 2.6 samples/s +Step 11600/11700 | Loss: 2.4717 | LR: 1.03e-07 | 7709.7ms/step | 2.6 samples/s +Step 11610/11700 | Loss: 2.4736 | LR: 8.33e-08 | 7718.2ms/step | 2.6 samples/s +Step 11620/11700 | Loss: 2.3893 | LR: 6.58e-08 | 7713.9ms/step | 2.6 samples/s +Step 11630/11700 | Loss: 2.3372 | LR: 5.04e-08 | 7712.0ms/step | 2.6 samples/s +Step 11640/11700 | Loss: 2.4724 | LR: 3.70e-08 | 7716.3ms/step | 2.6 samples/s +Step 11650/11700 | Loss: 2.5234 | LR: 2.57e-08 | 7715.2ms/step | 2.6 samples/s +Step 11660/11700 | Loss: 2.2979 | LR: 1.65e-08 | 7710.8ms/step | 2.6 samples/s +Step 11670/11700 | Loss: 2.4527 | LR: 9.26e-09 | 7713.1ms/step | 2.6 samples/s +Step 11680/11700 | Loss: 2.4643 | LR: 4.12e-09 | 7710.3ms/step | 2.6 samples/s +Step 11690/11700 | Loss: 2.3694 | LR: 1.03e-09 | 7712.7ms/step | 2.6 samples/s +Step 11700/11700 | Loss: 2.4336 | LR: 0.00e+00 | 7715.1ms/step | 2.6 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11700 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading history steps 1169-1169, summary, console lines 1229-1229 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▆▄▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ███████████████████████▁████████████████ +wandb: speed/steps_per_sec ██████▁██▁██████████████▁███████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.45182 +wandb: eval/perplexity 11.6094 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 7715.1129 +wandb: speed/samples_per_sec 2.59231 +wandb: speed/steps_per_sec 0.12962 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m fixed s1 30k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/kxni9nyw +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260707_035814-kxni9nyw/logs +Training complete! diff --git a/epfl_final/logs/l30k365m_fixed_s2.log b/epfl_final/logs/l30k365m_fixed_s2.log new file mode 100644 index 0000000000000000000000000000000000000000..5ec7e5891aae143455040c8e6ba494684c2e0fd9 --- /dev/null +++ b/epfl_final/logs/l30k365m_fixed_s2.log @@ -0,0 +1,1442 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 11700, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 20, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m fixed s2 30k", + "wandb_tags": [ + "qwen3", + "365m", + "30k", + "scaleladder", + "fixed", + "seed2" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s2", + "sparse_mask_seed": 2, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 768] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 768] [1, 30720, 768] 768 True +├─Linear: 1-2 [1, 30720, 768] [1, 30720, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 475.65 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 58494.81 +Params size (MB): 951.29 +Estimated Total Size (MB): 59446.35 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 1 +Gradient accumulation steps: 20 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 749299 +Optimizer steps per epoch: 37464 +Total optimizer steps: 11700 (capped by max_steps; 0.31 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035818-m10lf13v +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m fixed s2 30k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/m10lf13v +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/11700 | Loss: 12.0483 | LR: 6.67e-06 | 12124.8ms/step | 1.6 samples/s +Step 20/11700 | Loss: 11.9634 | LR: 1.33e-05 | 7730.6ms/step | 2.6 samples/s +Step 30/11700 | Loss: 11.7003 | LR: 2.00e-05 | 7733.9ms/step | 2.6 samples/s +Step 40/11700 | Loss: 11.1857 | LR: 2.67e-05 | 7713.5ms/step | 2.6 samples/s +Step 50/11700 | Loss: 10.8576 | LR: 3.33e-05 | 7699.7ms/step | 2.6 samples/s +Step 60/11700 | Loss: 10.5731 | LR: 4.00e-05 | 7690.4ms/step | 2.6 samples/s +Step 70/11700 | Loss: 10.2941 | LR: 4.67e-05 | 7684.3ms/step | 2.6 samples/s +Step 80/11700 | Loss: 10.0259 | LR: 5.33e-05 | 7677.4ms/step | 2.6 samples/s +Step 90/11700 | Loss: 9.7345 | LR: 6.00e-05 | 7674.1ms/step | 2.6 samples/s +Step 100/11700 | Loss: 9.3523 | LR: 6.67e-05 | 7686.2ms/step | 2.6 samples/s +Step 110/11700 | Loss: 8.8819 | LR: 7.33e-05 | 7686.3ms/step | 2.6 samples/s +Step 120/11700 | Loss: 8.3306 | LR: 8.00e-05 | 7687.3ms/step | 2.6 samples/s +Step 130/11700 | Loss: 7.8669 | LR: 8.67e-05 | 7693.2ms/step | 2.6 samples/s +Step 140/11700 | Loss: 7.5868 | LR: 9.33e-05 | 7697.2ms/step | 2.6 samples/s +Step 150/11700 | Loss: 7.3864 | LR: 1.00e-04 | 7703.8ms/step | 2.6 samples/s +Step 160/11700 | Loss: 7.1825 | LR: 1.07e-04 | 7714.1ms/step | 2.6 samples/s +Step 170/11700 | Loss: 7.0674 | LR: 1.13e-04 | 7722.6ms/step | 2.6 samples/s +Step 180/11700 | Loss: 6.9281 | LR: 1.20e-04 | 7727.1ms/step | 2.6 samples/s +Step 190/11700 | Loss: 6.8349 | LR: 1.27e-04 | 7734.0ms/step | 2.6 samples/s +Step 200/11700 | Loss: 6.6655 | LR: 1.33e-04 | 7735.5ms/step | 2.6 samples/s +Step 210/11700 | Loss: 6.5757 | LR: 1.40e-04 | 7739.9ms/step | 2.6 samples/s +Step 220/11700 | Loss: 6.4278 | LR: 1.47e-04 | 7745.3ms/step | 2.6 samples/s +Step 230/11700 | Loss: 6.3634 | LR: 1.53e-04 | 7744.0ms/step | 2.6 samples/s +Step 240/11700 | Loss: 6.2138 | LR: 1.60e-04 | 7742.8ms/step | 2.6 samples/s +Step 250/11700 | Loss: 6.2299 | LR: 1.67e-04 | 7745.0ms/step | 2.6 samples/s +Validation at step 250: Loss = 6.0940, Perplexity = 443.21 +Step 260/11700 | Loss: 6.1446 | LR: 1.73e-04 | 19932.3ms/step | 1.0 samples/s +Step 270/11700 | Loss: 6.0133 | LR: 1.80e-04 | 7789.6ms/step | 2.6 samples/s +Step 280/11700 | Loss: 5.9732 | LR: 1.87e-04 | 7795.7ms/step | 2.6 samples/s +Step 290/11700 | Loss: 5.8510 | LR: 1.93e-04 | 7793.1ms/step | 2.6 samples/s +Step 300/11700 | Loss: 5.8357 | LR: 2.00e-04 | 7812.2ms/step | 2.6 samples/s +Step 310/11700 | Loss: 5.8228 | LR: 2.07e-04 | 7782.8ms/step | 2.6 samples/s +Step 320/11700 | Loss: 5.7001 | LR: 2.13e-04 | 7783.7ms/step | 2.6 samples/s +Step 330/11700 | Loss: 5.5870 | LR: 2.20e-04 | 7798.1ms/step | 2.6 samples/s +Step 340/11700 | Loss: 5.5375 | LR: 2.27e-04 | 7797.9ms/step | 2.6 samples/s +Step 350/11700 | Loss: 5.3756 | LR: 2.33e-04 | 7781.6ms/step | 2.6 samples/s +Step 360/11700 | Loss: 5.4584 | LR: 2.40e-04 | 7792.3ms/step | 2.6 samples/s +Step 370/11700 | Loss: 5.4884 | LR: 2.47e-04 | 7796.5ms/step | 2.6 samples/s +Step 380/11700 | Loss: 5.4716 | LR: 2.53e-04 | 7780.1ms/step | 2.6 samples/s +Step 390/11700 | Loss: 5.4301 | LR: 2.60e-04 | 7787.4ms/step | 2.6 samples/s +Step 400/11700 | Loss: 5.2668 | LR: 2.67e-04 | 7798.2ms/step | 2.6 samples/s +Step 410/11700 | Loss: 5.2921 | LR: 2.73e-04 | 7798.0ms/step | 2.6 samples/s +Step 420/11700 | Loss: 5.2471 | LR: 2.80e-04 | 7795.8ms/step | 2.6 samples/s +Step 430/11700 | Loss: 5.2181 | LR: 2.87e-04 | 7787.1ms/step | 2.6 samples/s +Step 440/11700 | Loss: 5.2177 | LR: 2.93e-04 | 7795.5ms/step | 2.6 samples/s +Step 450/11700 | Loss: 5.1271 | LR: 3.00e-04 | 7799.0ms/step | 2.6 samples/s +Step 460/11700 | Loss: 5.1526 | LR: 3.07e-04 | 7792.4ms/step | 2.6 samples/s +Step 470/11700 | Loss: 5.1211 | LR: 3.13e-04 | 7798.8ms/step | 2.6 samples/s +Step 480/11700 | Loss: 5.1265 | LR: 3.20e-04 | 7790.0ms/step | 2.6 samples/s +Step 490/11700 | Loss: 5.0241 | LR: 3.27e-04 | 7788.2ms/step | 2.6 samples/s +Step 500/11700 | Loss: 4.9728 | LR: 3.33e-04 | 7799.0ms/step | 2.6 samples/s +Validation at step 500: Loss = 4.9183, Perplexity = 136.76 +Step 510/11700 | Loss: 4.8970 | LR: 3.40e-04 | 19399.3ms/step | 1.0 samples/s +Step 520/11700 | Loss: 4.9103 | LR: 3.47e-04 | 7790.7ms/step | 2.6 samples/s +Step 530/11700 | Loss: 4.8397 | LR: 3.53e-04 | 7797.1ms/step | 2.6 samples/s +Step 540/11700 | Loss: 4.7412 | LR: 3.60e-04 | 7794.5ms/step | 2.6 samples/s +Step 550/11700 | Loss: 4.7552 | LR: 3.67e-04 | 7803.4ms/step | 2.6 samples/s +Step 560/11700 | Loss: 4.7564 | LR: 3.73e-04 | 7808.8ms/step | 2.6 samples/s +Step 570/11700 | Loss: 4.5380 | LR: 3.80e-04 | 7803.9ms/step | 2.6 samples/s +Step 580/11700 | Loss: 4.5698 | LR: 3.87e-04 | 7795.9ms/step | 2.6 samples/s +Step 590/11700 | Loss: 4.5068 | LR: 3.93e-04 | 7792.7ms/step | 2.6 samples/s +Step 600/11700 | Loss: 4.5625 | LR: 4.00e-04 | 7801.5ms/step | 2.6 samples/s +Step 610/11700 | Loss: 4.5315 | LR: 4.07e-04 | 7803.1ms/step | 2.6 samples/s +Step 620/11700 | Loss: 4.5185 | LR: 4.13e-04 | 7801.5ms/step | 2.6 samples/s +Step 630/11700 | Loss: 4.3512 | LR: 4.20e-04 | 7807.5ms/step | 2.6 samples/s +Step 640/11700 | Loss: 4.4416 | LR: 4.27e-04 | 7795.8ms/step | 2.6 samples/s +Step 650/11700 | Loss: 4.3219 | LR: 4.33e-04 | 7792.0ms/step | 2.6 samples/s +Step 660/11700 | Loss: 4.4264 | LR: 4.40e-04 | 7804.0ms/step | 2.6 samples/s +Step 670/11700 | Loss: 4.3093 | LR: 4.47e-04 | 7798.1ms/step | 2.6 samples/s +Step 680/11700 | Loss: 4.2234 | LR: 4.53e-04 | 7803.6ms/step | 2.6 samples/s +Step 690/11700 | Loss: 4.0791 | LR: 4.60e-04 | 7801.1ms/step | 2.6 samples/s +Step 700/11700 | Loss: 4.2298 | LR: 4.67e-04 | 7799.7ms/step | 2.6 samples/s +Step 710/11700 | Loss: 4.2181 | LR: 4.73e-04 | 7800.5ms/step | 2.6 samples/s +Step 720/11700 | Loss: 4.1002 | LR: 4.80e-04 | 7801.8ms/step | 2.6 samples/s +Step 730/11700 | Loss: 3.9669 | LR: 4.87e-04 | 7805.9ms/step | 2.6 samples/s +Step 740/11700 | Loss: 4.1025 | LR: 4.93e-04 | 7798.7ms/step | 2.6 samples/s +Step 750/11700 | Loss: 4.0185 | LR: 5.00e-04 | 7804.9ms/step | 2.6 samples/s +Validation at step 750: Loss = 3.9681, Perplexity = 52.89 +Step 760/11700 | Loss: 4.0503 | LR: 5.00e-04 | 19394.1ms/step | 1.0 samples/s +Step 770/11700 | Loss: 3.8120 | LR: 5.00e-04 | 7799.2ms/step | 2.6 samples/s +Step 780/11700 | Loss: 4.0163 | LR: 5.00e-04 | 7795.1ms/step | 2.6 samples/s +Step 790/11700 | Loss: 3.7902 | LR: 5.00e-04 | 7793.5ms/step | 2.6 samples/s +Step 800/11700 | Loss: 3.7888 | LR: 5.00e-04 | 7801.3ms/step | 2.6 samples/s +Step 810/11700 | Loss: 3.8159 | LR: 5.00e-04 | 7808.5ms/step | 2.6 samples/s +Step 820/11700 | Loss: 3.7995 | LR: 5.00e-04 | 7794.3ms/step | 2.6 samples/s +Step 830/11700 | Loss: 3.7938 | LR: 5.00e-04 | 7795.7ms/step | 2.6 samples/s +Step 840/11700 | Loss: 3.8030 | LR: 5.00e-04 | 7809.7ms/step | 2.6 samples/s +Step 850/11700 | Loss: 3.8463 | LR: 5.00e-04 | 7796.9ms/step | 2.6 samples/s +Step 860/11700 | Loss: 3.6814 | LR: 5.00e-04 | 7801.2ms/step | 2.6 samples/s +Step 870/11700 | Loss: 3.7150 | LR: 5.00e-04 | 7794.7ms/step | 2.6 samples/s +Step 880/11700 | Loss: 3.7608 | LR: 5.00e-04 | 7801.3ms/step | 2.6 samples/s +Step 890/11700 | Loss: 3.6078 | LR: 5.00e-04 | 7803.5ms/step | 2.6 samples/s +Step 900/11700 | Loss: 3.5381 | LR: 5.00e-04 | 7808.3ms/step | 2.6 samples/s +Step 910/11700 | Loss: 3.7214 | LR: 5.00e-04 | 7799.8ms/step | 2.6 samples/s +Step 920/11700 | Loss: 3.6001 | LR: 5.00e-04 | 7808.0ms/step | 2.6 samples/s +Step 930/11700 | Loss: 3.5864 | LR: 5.00e-04 | 7806.1ms/step | 2.6 samples/s +Step 940/11700 | Loss: 3.6561 | LR: 5.00e-04 | 7814.9ms/step | 2.6 samples/s +Step 950/11700 | Loss: 3.4872 | LR: 5.00e-04 | 7801.6ms/step | 2.6 samples/s +Step 960/11700 | Loss: 3.5553 | LR: 5.00e-04 | 7797.7ms/step | 2.6 samples/s +Step 970/11700 | Loss: 3.6223 | LR: 5.00e-04 | 7798.4ms/step | 2.6 samples/s +Step 980/11700 | Loss: 3.3539 | LR: 4.99e-04 | 7808.4ms/step | 2.6 samples/s +Step 990/11700 | Loss: 3.5634 | LR: 4.99e-04 | 7795.5ms/step | 2.6 samples/s +Step 1000/11700 | Loss: 3.4551 | LR: 4.99e-04 | 7808.4ms/step | 2.6 samples/s +Validation at step 1000: Loss = 3.5078, Perplexity = 33.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-1000 +Step 1010/11700 | Loss: 3.5678 | LR: 4.99e-04 | 19714.2ms/step | 1.0 samples/s +Step 1020/11700 | Loss: 3.4669 | LR: 4.99e-04 | 7794.8ms/step | 2.6 samples/s +Step 1030/11700 | Loss: 3.4822 | LR: 4.99e-04 | 7795.1ms/step | 2.6 samples/s +Step 1040/11700 | Loss: 3.5082 | LR: 4.99e-04 | 7801.2ms/step | 2.6 samples/s +Step 1050/11700 | Loss: 3.4285 | LR: 4.99e-04 | 7815.4ms/step | 2.6 samples/s +Step 1060/11700 | Loss: 3.4857 | LR: 4.99e-04 | 7795.8ms/step | 2.6 samples/s +Step 1070/11700 | Loss: 3.4423 | LR: 4.99e-04 | 7808.3ms/step | 2.6 samples/s +Step 1080/11700 | Loss: 3.3041 | LR: 4.99e-04 | 7797.5ms/step | 2.6 samples/s +Step 1090/11700 | Loss: 3.3595 | LR: 4.99e-04 | 7804.3ms/step | 2.6 samples/s +Step 1100/11700 | Loss: 3.5647 | LR: 4.99e-04 | 7799.9ms/step | 2.6 samples/s +Step 1110/11700 | Loss: 3.5133 | LR: 4.99e-04 | 7800.2ms/step | 2.6 samples/s +Step 1120/11700 | Loss: 3.3988 | LR: 4.99e-04 | 7800.6ms/step | 2.6 samples/s +Step 1130/11700 | Loss: 3.4001 | LR: 4.99e-04 | 7796.6ms/step | 2.6 samples/s +Step 1140/11700 | Loss: 3.1744 | LR: 4.98e-04 | 7805.7ms/step | 2.6 samples/s +Step 1150/11700 | Loss: 3.3319 | LR: 4.98e-04 | 7808.8ms/step | 2.6 samples/s +Step 1160/11700 | Loss: 3.3401 | LR: 4.98e-04 | 7789.7ms/step | 2.6 samples/s +Step 1170/11700 | Loss: 3.3070 | LR: 4.98e-04 | 7798.5ms/step | 2.6 samples/s +Step 1180/11700 | Loss: 3.2484 | LR: 4.98e-04 | 7796.8ms/step | 2.6 samples/s +Step 1190/11700 | Loss: 3.2085 | LR: 4.98e-04 | 7806.3ms/step | 2.6 samples/s +Step 1200/11700 | Loss: 3.1442 | LR: 4.98e-04 | 7808.7ms/step | 2.6 samples/s +Step 1210/11700 | Loss: 3.4724 | LR: 4.98e-04 | 7798.6ms/step | 2.6 samples/s +Step 1220/11700 | Loss: 3.2654 | LR: 4.98e-04 | 7802.8ms/step | 2.6 samples/s +Step 1230/11700 | Loss: 3.3053 | LR: 4.98e-04 | 7791.3ms/step | 2.6 samples/s +Step 1240/11700 | Loss: 3.2461 | LR: 4.98e-04 | 7802.7ms/step | 2.6 samples/s +Step 1250/11700 | Loss: 3.2277 | LR: 4.97e-04 | 7800.7ms/step | 2.6 samples/s +Validation at step 1250: Loss = 3.2741, Perplexity = 26.42 +Step 1260/11700 | Loss: 3.3033 | LR: 4.97e-04 | 19401.9ms/step | 1.0 samples/s +Step 1270/11700 | Loss: 3.1401 | LR: 4.97e-04 | 7799.0ms/step | 2.6 samples/s +Step 1280/11700 | Loss: 3.2463 | LR: 4.97e-04 | 7811.2ms/step | 2.6 samples/s +Step 1290/11700 | Loss: 3.2376 | LR: 4.97e-04 | 7808.9ms/step | 2.6 samples/s +Step 1300/11700 | Loss: 3.2451 | LR: 4.97e-04 | 7810.3ms/step | 2.6 samples/s +Step 1310/11700 | Loss: 3.2959 | LR: 4.97e-04 | 7795.3ms/step | 2.6 samples/s +Step 1320/11700 | Loss: 3.2567 | LR: 4.97e-04 | 7801.7ms/step | 2.6 samples/s +Step 1330/11700 | Loss: 3.1817 | LR: 4.97e-04 | 7805.7ms/step | 2.6 samples/s +Step 1340/11700 | Loss: 3.2302 | LR: 4.96e-04 | 7808.4ms/step | 2.6 samples/s +Step 1350/11700 | Loss: 3.2422 | LR: 4.96e-04 | 7798.2ms/step | 2.6 samples/s +Step 1360/11700 | Loss: 3.2010 | LR: 4.96e-04 | 7791.6ms/step | 2.6 samples/s +Step 1370/11700 | Loss: 3.1433 | LR: 4.96e-04 | 7799.6ms/step | 2.6 samples/s +Step 1380/11700 | Loss: 3.0110 | LR: 4.96e-04 | 7800.7ms/step | 2.6 samples/s +Step 1390/11700 | Loss: 3.1734 | LR: 4.96e-04 | 7805.8ms/step | 2.6 samples/s +Step 1400/11700 | Loss: 3.2579 | LR: 4.96e-04 | 7797.5ms/step | 2.6 samples/s +Step 1410/11700 | Loss: 3.1981 | LR: 4.96e-04 | 7800.5ms/step | 2.6 samples/s +Step 1420/11700 | Loss: 3.0974 | LR: 4.95e-04 | 7795.3ms/step | 2.6 samples/s +Step 1430/11700 | Loss: 3.1448 | LR: 4.95e-04 | 7787.0ms/step | 2.6 samples/s +Step 1440/11700 | Loss: 3.1359 | LR: 4.95e-04 | 7786.0ms/step | 2.6 samples/s +Step 1450/11700 | Loss: 3.0600 | LR: 4.95e-04 | 7786.8ms/step | 2.6 samples/s +Step 1460/11700 | Loss: 3.1119 | LR: 4.95e-04 | 7789.0ms/step | 2.6 samples/s +Step 1470/11700 | Loss: 3.1236 | LR: 4.95e-04 | 7803.7ms/step | 2.6 samples/s +Step 1480/11700 | Loss: 3.0541 | LR: 4.95e-04 | 7790.0ms/step | 2.6 samples/s +Step 1490/11700 | Loss: 3.2103 | LR: 4.94e-04 | 7787.2ms/step | 2.6 samples/s +Step 1500/11700 | Loss: 3.0483 | LR: 4.94e-04 | 7793.7ms/step | 2.6 samples/s +Validation at step 1500: Loss = 3.1282, Perplexity = 22.83 +Step 1510/11700 | Loss: 3.1229 | LR: 4.94e-04 | 19416.2ms/step | 1.0 samples/s +Step 1520/11700 | Loss: 3.1079 | LR: 4.94e-04 | 7785.2ms/step | 2.6 samples/s +Step 1530/11700 | Loss: 3.1122 | LR: 4.94e-04 | 7786.4ms/step | 2.6 samples/s +Step 1540/11700 | Loss: 3.2660 | LR: 4.94e-04 | 7786.7ms/step | 2.6 samples/s +Step 1550/11700 | Loss: 3.1147 | LR: 4.93e-04 | 7787.0ms/step | 2.6 samples/s +Step 1560/11700 | Loss: 3.1368 | LR: 4.93e-04 | 7783.2ms/step | 2.6 samples/s +Step 1570/11700 | Loss: 3.1234 | LR: 4.93e-04 | 7793.1ms/step | 2.6 samples/s +Step 1580/11700 | Loss: 3.0872 | LR: 4.93e-04 | 7789.4ms/step | 2.6 samples/s +Step 1590/11700 | Loss: 3.2243 | LR: 4.93e-04 | 7789.7ms/step | 2.6 samples/s +Step 1600/11700 | Loss: 3.1110 | LR: 4.93e-04 | 7792.0ms/step | 2.6 samples/s +Step 1610/11700 | Loss: 3.1547 | LR: 4.92e-04 | 7790.0ms/step | 2.6 samples/s +Step 1620/11700 | Loss: 3.0310 | LR: 4.92e-04 | 7794.5ms/step | 2.6 samples/s +Step 1630/11700 | Loss: 2.9387 | LR: 4.92e-04 | 7791.8ms/step | 2.6 samples/s +Step 1640/11700 | Loss: 3.1185 | LR: 4.92e-04 | 7790.7ms/step | 2.6 samples/s +Step 1650/11700 | Loss: 3.0533 | LR: 4.92e-04 | 7790.0ms/step | 2.6 samples/s +Step 1660/11700 | Loss: 3.0873 | LR: 4.92e-04 | 7792.7ms/step | 2.6 samples/s +Step 1670/11700 | Loss: 3.2119 | LR: 4.91e-04 | 7789.2ms/step | 2.6 samples/s +Step 1680/11700 | Loss: 3.0581 | LR: 4.91e-04 | 7790.0ms/step | 2.6 samples/s +Step 1690/11700 | Loss: 2.9176 | LR: 4.91e-04 | 7790.8ms/step | 2.6 samples/s +Step 1700/11700 | Loss: 2.9755 | LR: 4.91e-04 | 7790.0ms/step | 2.6 samples/s +Step 1710/11700 | Loss: 3.1080 | LR: 4.91e-04 | 7789.2ms/step | 2.6 samples/s +Step 1720/11700 | Loss: 3.0318 | LR: 4.90e-04 | 7790.1ms/step | 2.6 samples/s +Step 1730/11700 | Loss: 3.0640 | LR: 4.90e-04 | 7789.7ms/step | 2.6 samples/s +Step 1740/11700 | Loss: 2.9609 | LR: 4.90e-04 | 7789.0ms/step | 2.6 samples/s +Step 1750/11700 | Loss: 2.9304 | LR: 4.90e-04 | 7790.0ms/step | 2.6 samples/s +Validation at step 1750: Loss = 3.0505, Perplexity = 21.13 +Step 1760/11700 | Loss: 3.0282 | LR: 4.90e-04 | 19367.9ms/step | 1.0 samples/s +Step 1770/11700 | Loss: 3.0279 | LR: 4.89e-04 | 7793.3ms/step | 2.6 samples/s +Step 1780/11700 | Loss: 3.0518 | LR: 4.89e-04 | 7790.9ms/step | 2.6 samples/s +Step 1790/11700 | Loss: 3.0216 | LR: 4.89e-04 | 7791.1ms/step | 2.6 samples/s +Step 1800/11700 | Loss: 3.0367 | LR: 4.89e-04 | 7789.9ms/step | 2.6 samples/s +Step 1810/11700 | Loss: 3.0937 | LR: 4.89e-04 | 7789.0ms/step | 2.6 samples/s +Step 1820/11700 | Loss: 3.0198 | LR: 4.88e-04 | 7789.0ms/step | 2.6 samples/s +Step 1830/11700 | Loss: 2.9920 | LR: 4.88e-04 | 7791.9ms/step | 2.6 samples/s +Step 1840/11700 | Loss: 2.9863 | LR: 4.88e-04 | 7790.1ms/step | 2.6 samples/s +Step 1850/11700 | Loss: 3.0090 | LR: 4.88e-04 | 7790.3ms/step | 2.6 samples/s +Step 1860/11700 | Loss: 2.9850 | LR: 4.87e-04 | 7786.5ms/step | 2.6 samples/s +Step 1870/11700 | Loss: 2.9133 | LR: 4.87e-04 | 7785.1ms/step | 2.6 samples/s +Step 1880/11700 | Loss: 2.9351 | LR: 4.87e-04 | 7785.4ms/step | 2.6 samples/s +Step 1890/11700 | Loss: 3.0574 | LR: 4.87e-04 | 7787.3ms/step | 2.6 samples/s +Step 1900/11700 | Loss: 3.0048 | LR: 4.87e-04 | 7781.6ms/step | 2.6 samples/s +Step 1910/11700 | Loss: 3.0309 | LR: 4.86e-04 | 7783.2ms/step | 2.6 samples/s +Step 1920/11700 | Loss: 2.8259 | LR: 4.86e-04 | 7783.2ms/step | 2.6 samples/s +Step 1930/11700 | Loss: 3.0181 | LR: 4.86e-04 | 7782.0ms/step | 2.6 samples/s +Step 1940/11700 | Loss: 2.9504 | LR: 4.86e-04 | 7781.8ms/step | 2.6 samples/s +Step 1950/11700 | Loss: 2.8694 | LR: 4.85e-04 | 7784.8ms/step | 2.6 samples/s +Step 1960/11700 | Loss: 2.9380 | LR: 4.85e-04 | 7789.4ms/step | 2.6 samples/s +Step 1970/11700 | Loss: 2.9598 | LR: 4.85e-04 | 7793.2ms/step | 2.6 samples/s +Step 1980/11700 | Loss: 2.9388 | LR: 4.85e-04 | 7788.9ms/step | 2.6 samples/s +Step 1990/11700 | Loss: 2.8826 | LR: 4.84e-04 | 7793.5ms/step | 2.6 samples/s +Step 2000/11700 | Loss: 3.0925 | LR: 4.84e-04 | 7797.4ms/step | 2.6 samples/s +Validation at step 2000: Loss = 2.9627, Perplexity = 19.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-2000 +Step 2010/11700 | Loss: 2.9809 | LR: 4.84e-04 | 19661.1ms/step | 1.0 samples/s +Step 2020/11700 | Loss: 3.0177 | LR: 4.84e-04 | 7783.3ms/step | 2.6 samples/s +Step 2030/11700 | Loss: 2.9192 | LR: 4.83e-04 | 7787.2ms/step | 2.6 samples/s +Step 2040/11700 | Loss: 2.9645 | LR: 4.83e-04 | 7790.2ms/step | 2.6 samples/s +Step 2050/11700 | Loss: 2.8349 | LR: 4.83e-04 | 7790.6ms/step | 2.6 samples/s +Step 2060/11700 | Loss: 2.9050 | LR: 4.83e-04 | 7791.0ms/step | 2.6 samples/s +Step 2070/11700 | Loss: 2.9080 | LR: 4.82e-04 | 7788.1ms/step | 2.6 samples/s +Step 2080/11700 | Loss: 2.8856 | LR: 4.82e-04 | 7794.9ms/step | 2.6 samples/s +Step 2090/11700 | Loss: 2.9159 | LR: 4.82e-04 | 7789.7ms/step | 2.6 samples/s +Step 2100/11700 | Loss: 3.0044 | LR: 4.81e-04 | 7790.2ms/step | 2.6 samples/s +Step 2110/11700 | Loss: 2.9992 | LR: 4.81e-04 | 7794.1ms/step | 2.6 samples/s +Step 2120/11700 | Loss: 2.9365 | LR: 4.81e-04 | 7790.7ms/step | 2.6 samples/s +Step 2130/11700 | Loss: 2.9314 | LR: 4.81e-04 | 7790.5ms/step | 2.6 samples/s +Step 2140/11700 | Loss: 3.0457 | LR: 4.80e-04 | 7784.8ms/step | 2.6 samples/s +Step 2150/11700 | Loss: 2.9629 | LR: 4.80e-04 | 7788.3ms/step | 2.6 samples/s +Step 2160/11700 | Loss: 2.9219 | LR: 4.80e-04 | 7790.7ms/step | 2.6 samples/s +Step 2170/11700 | Loss: 2.8508 | LR: 4.80e-04 | 7795.5ms/step | 2.6 samples/s +Step 2180/11700 | Loss: 2.8844 | LR: 4.79e-04 | 7790.5ms/step | 2.6 samples/s +Step 2190/11700 | Loss: 2.8776 | LR: 4.79e-04 | 7794.0ms/step | 2.6 samples/s +Step 2200/11700 | Loss: 2.9411 | LR: 4.79e-04 | 7791.9ms/step | 2.6 samples/s +Step 2210/11700 | Loss: 2.9755 | LR: 4.78e-04 | 7787.1ms/step | 2.6 samples/s +Step 2220/11700 | Loss: 2.7846 | LR: 4.78e-04 | 7789.3ms/step | 2.6 samples/s +Step 2230/11700 | Loss: 2.8206 | LR: 4.78e-04 | 7788.0ms/step | 2.6 samples/s +Step 2240/11700 | Loss: 2.8053 | LR: 4.78e-04 | 7791.6ms/step | 2.6 samples/s +Step 2250/11700 | Loss: 2.9920 | LR: 4.77e-04 | 7792.1ms/step | 2.6 samples/s +Validation at step 2250: Loss = 2.8970, Perplexity = 18.12 +Step 2260/11700 | Loss: 2.7742 | LR: 4.77e-04 | 19373.9ms/step | 1.0 samples/s +Step 2270/11700 | Loss: 2.9844 | LR: 4.77e-04 | 7793.7ms/step | 2.6 samples/s +Step 2280/11700 | Loss: 2.9035 | LR: 4.76e-04 | 7790.1ms/step | 2.6 samples/s +Step 2290/11700 | Loss: 2.9236 | LR: 4.76e-04 | 7790.4ms/step | 2.6 samples/s +Step 2300/11700 | Loss: 2.8607 | LR: 4.76e-04 | 7788.2ms/step | 2.6 samples/s +Step 2310/11700 | Loss: 2.8478 | LR: 4.75e-04 | 7786.6ms/step | 2.6 samples/s +Step 2320/11700 | Loss: 2.9960 | LR: 4.75e-04 | 7787.2ms/step | 2.6 samples/s +Step 2330/11700 | Loss: 2.8204 | LR: 4.75e-04 | 7787.6ms/step | 2.6 samples/s +Step 2340/11700 | Loss: 2.8540 | LR: 4.74e-04 | 7788.9ms/step | 2.6 samples/s +Step 2350/11700 | Loss: 2.7768 | LR: 4.74e-04 | 7785.2ms/step | 2.6 samples/s +Step 2360/11700 | Loss: 2.7951 | LR: 4.74e-04 | 7788.9ms/step | 2.6 samples/s +Step 2370/11700 | Loss: 2.7778 | LR: 4.73e-04 | 7786.7ms/step | 2.6 samples/s +Step 2380/11700 | Loss: 2.7728 | LR: 4.73e-04 | 7789.7ms/step | 2.6 samples/s +Step 2390/11700 | Loss: 2.9198 | LR: 4.73e-04 | 7787.4ms/step | 2.6 samples/s +Step 2400/11700 | Loss: 2.8568 | LR: 4.73e-04 | 7790.4ms/step | 2.6 samples/s +Step 2410/11700 | Loss: 2.8731 | LR: 4.72e-04 | 7791.7ms/step | 2.6 samples/s +Step 2420/11700 | Loss: 2.7317 | LR: 4.72e-04 | 7791.8ms/step | 2.6 samples/s +Step 2430/11700 | Loss: 2.9058 | LR: 4.72e-04 | 7789.4ms/step | 2.6 samples/s +Step 2440/11700 | Loss: 2.9292 | LR: 4.71e-04 | 7789.3ms/step | 2.6 samples/s +Step 2450/11700 | Loss: 2.7653 | LR: 4.71e-04 | 7786.2ms/step | 2.6 samples/s +Step 2460/11700 | Loss: 2.8779 | LR: 4.71e-04 | 7790.1ms/step | 2.6 samples/s +Step 2470/11700 | Loss: 2.8823 | LR: 4.70e-04 | 7789.6ms/step | 2.6 samples/s +Step 2480/11700 | Loss: 2.8120 | LR: 4.70e-04 | 7784.7ms/step | 2.6 samples/s +Step 2490/11700 | Loss: 2.9037 | LR: 4.69e-04 | 7784.2ms/step | 2.6 samples/s +Step 2500/11700 | Loss: 2.7833 | LR: 4.69e-04 | 7816.2ms/step | 2.6 samples/s +Validation at step 2500: Loss = 2.8503, Perplexity = 17.29 +Step 2510/11700 | Loss: 2.7603 | LR: 4.69e-04 | 19407.0ms/step | 1.0 samples/s +Step 2520/11700 | Loss: 2.9160 | LR: 4.68e-04 | 7785.2ms/step | 2.6 samples/s +Step 2530/11700 | Loss: 2.8090 | LR: 4.68e-04 | 7801.7ms/step | 2.6 samples/s +Step 2540/11700 | Loss: 2.7422 | LR: 4.68e-04 | 7794.4ms/step | 2.6 samples/s +Step 2550/11700 | Loss: 2.9117 | LR: 4.67e-04 | 7785.7ms/step | 2.6 samples/s +Step 2560/11700 | Loss: 2.8538 | LR: 4.67e-04 | 7782.2ms/step | 2.6 samples/s +Step 2570/11700 | Loss: 2.8179 | LR: 4.67e-04 | 7782.5ms/step | 2.6 samples/s +Step 2580/11700 | Loss: 2.8388 | LR: 4.66e-04 | 7785.6ms/step | 2.6 samples/s +Step 2590/11700 | Loss: 2.7792 | LR: 4.66e-04 | 7779.4ms/step | 2.6 samples/s +Step 2600/11700 | Loss: 2.8060 | LR: 4.66e-04 | 7778.4ms/step | 2.6 samples/s +Step 2610/11700 | Loss: 2.7331 | LR: 4.65e-04 | 7776.5ms/step | 2.6 samples/s +Step 2620/11700 | Loss: 2.7939 | LR: 4.65e-04 | 7780.9ms/step | 2.6 samples/s +Step 2630/11700 | Loss: 2.7359 | LR: 4.65e-04 | 7785.4ms/step | 2.6 samples/s +Step 2640/11700 | Loss: 2.8422 | LR: 4.64e-04 | 7784.6ms/step | 2.6 samples/s +Step 2650/11700 | Loss: 2.9054 | LR: 4.64e-04 | 7787.8ms/step | 2.6 samples/s +Step 2660/11700 | Loss: 2.8373 | LR: 4.63e-04 | 7783.0ms/step | 2.6 samples/s +Step 2670/11700 | Loss: 2.7785 | LR: 4.63e-04 | 7785.0ms/step | 2.6 samples/s +Step 2680/11700 | Loss: 2.8231 | LR: 4.63e-04 | 7785.0ms/step | 2.6 samples/s +Step 2690/11700 | Loss: 2.9284 | LR: 4.62e-04 | 7783.3ms/step | 2.6 samples/s +Step 2700/11700 | Loss: 2.8220 | LR: 4.62e-04 | 7784.6ms/step | 2.6 samples/s +Step 2710/11700 | Loss: 2.8176 | LR: 4.62e-04 | 7786.1ms/step | 2.6 samples/s +Step 2720/11700 | Loss: 2.8246 | LR: 4.61e-04 | 7783.6ms/step | 2.6 samples/s +Step 2730/11700 | Loss: 2.7267 | LR: 4.61e-04 | 7785.5ms/step | 2.6 samples/s +Step 2740/11700 | Loss: 2.8178 | LR: 4.60e-04 | 7787.6ms/step | 2.6 samples/s +Step 2750/11700 | Loss: 2.8689 | LR: 4.60e-04 | 7785.5ms/step | 2.6 samples/s +Validation at step 2750: Loss = 2.8061, Perplexity = 16.55 +Step 2760/11700 | Loss: 2.6175 | LR: 4.60e-04 | 19366.7ms/step | 1.0 samples/s +Step 2770/11700 | Loss: 2.7706 | LR: 4.59e-04 | 7789.1ms/step | 2.6 samples/s +Step 2780/11700 | Loss: 2.7836 | LR: 4.59e-04 | 7782.8ms/step | 2.6 samples/s +Step 2790/11700 | Loss: 2.8283 | LR: 4.58e-04 | 7784.0ms/step | 2.6 samples/s +Step 2800/11700 | Loss: 2.7333 | LR: 4.58e-04 | 7785.5ms/step | 2.6 samples/s +Step 2810/11700 | Loss: 2.8317 | LR: 4.58e-04 | 7785.2ms/step | 2.6 samples/s +Step 2820/11700 | Loss: 2.8161 | LR: 4.57e-04 | 7787.3ms/step | 2.6 samples/s +Step 2830/11700 | Loss: 2.7831 | LR: 4.57e-04 | 7786.2ms/step | 2.6 samples/s +Step 2840/11700 | Loss: 2.8587 | LR: 4.56e-04 | 7785.0ms/step | 2.6 samples/s +Step 2850/11700 | Loss: 2.8937 | LR: 4.56e-04 | 7785.4ms/step | 2.6 samples/s +Step 2860/11700 | Loss: 2.7747 | LR: 4.56e-04 | 7787.2ms/step | 2.6 samples/s +Step 2870/11700 | Loss: 2.8488 | LR: 4.55e-04 | 7784.2ms/step | 2.6 samples/s +Step 2880/11700 | Loss: 2.5763 | LR: 4.55e-04 | 7783.3ms/step | 2.6 samples/s +Step 2890/11700 | Loss: 2.8293 | LR: 4.54e-04 | 7783.5ms/step | 2.6 samples/s +Step 2900/11700 | Loss: 2.7750 | LR: 4.54e-04 | 7785.0ms/step | 2.6 samples/s +Step 2910/11700 | Loss: 2.8464 | LR: 4.54e-04 | 7784.0ms/step | 2.6 samples/s +Step 2920/11700 | Loss: 2.7725 | LR: 4.53e-04 | 7786.0ms/step | 2.6 samples/s +Step 2930/11700 | Loss: 2.8552 | LR: 4.53e-04 | 7784.0ms/step | 2.6 samples/s +Step 2940/11700 | Loss: 2.7844 | LR: 4.52e-04 | 7782.8ms/step | 2.6 samples/s +Step 2950/11700 | Loss: 2.8336 | LR: 4.52e-04 | 7781.0ms/step | 2.6 samples/s +Step 2960/11700 | Loss: 2.7274 | LR: 4.51e-04 | 7783.2ms/step | 2.6 samples/s +Step 2970/11700 | Loss: 2.6788 | LR: 4.51e-04 | 7794.4ms/step | 2.6 samples/s +Step 2980/11700 | Loss: 2.7615 | LR: 4.51e-04 | 7793.8ms/step | 2.6 samples/s +Step 2990/11700 | Loss: 2.6929 | LR: 4.50e-04 | 7801.3ms/step | 2.6 samples/s +Step 3000/11700 | Loss: 2.7701 | LR: 4.50e-04 | 7794.1ms/step | 2.6 samples/s +Validation at step 3000: Loss = 2.7740, Perplexity = 16.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-3000 +Step 3010/11700 | Loss: 2.7411 | LR: 4.49e-04 | 19676.9ms/step | 1.0 samples/s +Step 3020/11700 | Loss: 2.8868 | LR: 4.49e-04 | 7795.1ms/step | 2.6 samples/s +Step 3030/11700 | Loss: 2.7223 | LR: 4.48e-04 | 7794.7ms/step | 2.6 samples/s +Step 3040/11700 | Loss: 2.8285 | LR: 4.48e-04 | 7792.4ms/step | 2.6 samples/s +Step 3050/11700 | Loss: 2.7628 | LR: 4.48e-04 | 7800.1ms/step | 2.6 samples/s +Step 3060/11700 | Loss: 2.9437 | LR: 4.47e-04 | 7808.6ms/step | 2.6 samples/s +Step 3070/11700 | Loss: 2.7199 | LR: 4.47e-04 | 7812.9ms/step | 2.6 samples/s +Step 3080/11700 | Loss: 2.7179 | LR: 4.46e-04 | 7812.6ms/step | 2.6 samples/s +Step 3090/11700 | Loss: 2.7264 | LR: 4.46e-04 | 7812.1ms/step | 2.6 samples/s +Step 3100/11700 | Loss: 2.8584 | LR: 4.45e-04 | 7817.1ms/step | 2.6 samples/s +Step 3110/11700 | Loss: 2.6370 | LR: 4.45e-04 | 7814.9ms/step | 2.6 samples/s +Step 3120/11700 | Loss: 2.7122 | LR: 4.44e-04 | 7814.4ms/step | 2.6 samples/s +Step 3130/11700 | Loss: 2.7483 | LR: 4.44e-04 | 7813.7ms/step | 2.6 samples/s +Step 3140/11700 | Loss: 2.7006 | LR: 4.43e-04 | 7817.5ms/step | 2.6 samples/s +Step 3150/11700 | Loss: 2.7190 | LR: 4.43e-04 | 7814.2ms/step | 2.6 samples/s +Step 3160/11700 | Loss: 2.7208 | LR: 4.43e-04 | 7816.5ms/step | 2.6 samples/s +Step 3170/11700 | Loss: 2.8133 | LR: 4.42e-04 | 7827.0ms/step | 2.6 samples/s +Step 3180/11700 | Loss: 2.7785 | LR: 4.42e-04 | 7817.1ms/step | 2.6 samples/s +Step 3190/11700 | Loss: 2.6510 | LR: 4.41e-04 | 7817.0ms/step | 2.6 samples/s +Step 3200/11700 | Loss: 2.6789 | LR: 4.41e-04 | 7815.7ms/step | 2.6 samples/s +Step 3210/11700 | Loss: 2.7041 | LR: 4.40e-04 | 7814.9ms/step | 2.6 samples/s +Step 3220/11700 | Loss: 2.7256 | LR: 4.40e-04 | 7812.1ms/step | 2.6 samples/s +Step 3230/11700 | Loss: 2.6738 | LR: 4.39e-04 | 7815.5ms/step | 2.6 samples/s +Step 3240/11700 | Loss: 2.7417 | LR: 4.39e-04 | 7815.8ms/step | 2.6 samples/s +Step 3250/11700 | Loss: 2.8304 | LR: 4.38e-04 | 7811.9ms/step | 2.6 samples/s +Validation at step 3250: Loss = 2.7376, Perplexity = 15.45 +Step 3260/11700 | Loss: 2.7471 | LR: 4.38e-04 | 19442.5ms/step | 1.0 samples/s +Step 3270/11700 | Loss: 2.6905 | LR: 4.37e-04 | 7808.9ms/step | 2.6 samples/s +Step 3280/11700 | Loss: 2.7481 | LR: 4.37e-04 | 7809.9ms/step | 2.6 samples/s +Step 3290/11700 | Loss: 2.6938 | LR: 4.37e-04 | 7810.0ms/step | 2.6 samples/s +Step 3300/11700 | Loss: 2.7568 | LR: 4.36e-04 | 7814.8ms/step | 2.6 samples/s +Step 3310/11700 | Loss: 2.7561 | LR: 4.36e-04 | 7812.8ms/step | 2.6 samples/s +Step 3320/11700 | Loss: 2.6733 | LR: 4.35e-04 | 7811.3ms/step | 2.6 samples/s +Step 3330/11700 | Loss: 2.6597 | LR: 4.35e-04 | 7805.7ms/step | 2.6 samples/s +Step 3340/11700 | Loss: 2.7964 | LR: 4.34e-04 | 7805.2ms/step | 2.6 samples/s +Step 3350/11700 | Loss: 2.6977 | LR: 4.34e-04 | 7801.7ms/step | 2.6 samples/s +Step 3360/11700 | Loss: 2.7787 | LR: 4.33e-04 | 7801.8ms/step | 2.6 samples/s +Step 3370/11700 | Loss: 2.6637 | LR: 4.33e-04 | 7802.9ms/step | 2.6 samples/s +Step 3380/11700 | Loss: 2.7256 | LR: 4.32e-04 | 7804.8ms/step | 2.6 samples/s +Step 3390/11700 | Loss: 2.6890 | LR: 4.32e-04 | 7805.7ms/step | 2.6 samples/s +Step 3400/11700 | Loss: 2.7992 | LR: 4.31e-04 | 7808.2ms/step | 2.6 samples/s +Step 3410/11700 | Loss: 2.7400 | LR: 4.31e-04 | 7806.0ms/step | 2.6 samples/s +Step 3420/11700 | Loss: 2.5689 | LR: 4.30e-04 | 7810.9ms/step | 2.6 samples/s +Step 3430/11700 | Loss: 2.7553 | LR: 4.30e-04 | 7821.2ms/step | 2.6 samples/s +Step 3440/11700 | Loss: 2.6094 | LR: 4.29e-04 | 7818.0ms/step | 2.6 samples/s +Step 3450/11700 | Loss: 2.7223 | LR: 4.29e-04 | 7817.1ms/step | 2.6 samples/s +Step 3460/11700 | Loss: 2.7058 | LR: 4.28e-04 | 7816.4ms/step | 2.6 samples/s +Step 3470/11700 | Loss: 2.5958 | LR: 4.28e-04 | 7822.1ms/step | 2.6 samples/s +Step 3480/11700 | Loss: 2.5197 | LR: 4.27e-04 | 7820.6ms/step | 2.6 samples/s +Step 3490/11700 | Loss: 2.8477 | LR: 4.27e-04 | 7814.5ms/step | 2.6 samples/s +Step 3500/11700 | Loss: 2.7588 | LR: 4.26e-04 | 7813.9ms/step | 2.6 samples/s +Validation at step 3500: Loss = 2.7077, Perplexity = 14.99 +Step 3510/11700 | Loss: 2.7405 | LR: 4.26e-04 | 19417.3ms/step | 1.0 samples/s +Step 3520/11700 | Loss: 2.7293 | LR: 4.25e-04 | 7802.7ms/step | 2.6 samples/s +Step 3530/11700 | Loss: 2.7394 | LR: 4.25e-04 | 7811.2ms/step | 2.6 samples/s +Step 3540/11700 | Loss: 2.7461 | LR: 4.24e-04 | 7815.2ms/step | 2.6 samples/s +Step 3550/11700 | Loss: 2.6650 | LR: 4.24e-04 | 7813.1ms/step | 2.6 samples/s +Step 3560/11700 | Loss: 2.8531 | LR: 4.23e-04 | 7811.6ms/step | 2.6 samples/s +Step 3570/11700 | Loss: 2.7221 | LR: 4.23e-04 | 7816.0ms/step | 2.6 samples/s +Step 3580/11700 | Loss: 2.7368 | LR: 4.22e-04 | 7815.5ms/step | 2.6 samples/s +Step 3590/11700 | Loss: 2.6369 | LR: 4.22e-04 | 7814.4ms/step | 2.6 samples/s +Step 3600/11700 | Loss: 2.6327 | LR: 4.21e-04 | 7814.3ms/step | 2.6 samples/s +Step 3610/11700 | Loss: 2.7634 | LR: 4.20e-04 | 7815.1ms/step | 2.6 samples/s +Step 3620/11700 | Loss: 2.6312 | LR: 4.20e-04 | 7813.2ms/step | 2.6 samples/s +Step 3630/11700 | Loss: 2.6417 | LR: 4.19e-04 | 7792.0ms/step | 2.6 samples/s +Step 3640/11700 | Loss: 2.5898 | LR: 4.19e-04 | 7776.9ms/step | 2.6 samples/s +Step 3650/11700 | Loss: 2.7195 | LR: 4.18e-04 | 7778.9ms/step | 2.6 samples/s +Step 3660/11700 | Loss: 2.6868 | LR: 4.18e-04 | 7779.1ms/step | 2.6 samples/s +Step 3670/11700 | Loss: 2.6408 | LR: 4.17e-04 | 7778.0ms/step | 2.6 samples/s +Step 3680/11700 | Loss: 2.7360 | LR: 4.17e-04 | 7778.2ms/step | 2.6 samples/s +Step 3690/11700 | Loss: 2.6718 | LR: 4.16e-04 | 7777.4ms/step | 2.6 samples/s +Step 3700/11700 | Loss: 2.7135 | LR: 4.16e-04 | 7769.5ms/step | 2.6 samples/s +Step 3710/11700 | Loss: 2.6116 | LR: 4.15e-04 | 7771.8ms/step | 2.6 samples/s +Step 3720/11700 | Loss: 2.6384 | LR: 4.15e-04 | 7774.0ms/step | 2.6 samples/s +Step 3730/11700 | Loss: 2.6904 | LR: 4.14e-04 | 7773.4ms/step | 2.6 samples/s +Step 3740/11700 | Loss: 2.6396 | LR: 4.14e-04 | 7771.9ms/step | 2.6 samples/s +Step 3750/11700 | Loss: 2.7130 | LR: 4.13e-04 | 7771.9ms/step | 2.6 samples/s +Validation at step 3750: Loss = 2.6804, Perplexity = 14.59 +Step 3760/11700 | Loss: 2.7227 | LR: 4.12e-04 | 19324.2ms/step | 1.0 samples/s +Step 3770/11700 | Loss: 2.7143 | LR: 4.12e-04 | 7774.8ms/step | 2.6 samples/s +Step 3780/11700 | Loss: 2.6673 | LR: 4.11e-04 | 7772.8ms/step | 2.6 samples/s +Step 3790/11700 | Loss: 2.8107 | LR: 4.11e-04 | 7772.2ms/step | 2.6 samples/s +Step 3800/11700 | Loss: 2.6996 | LR: 4.10e-04 | 7772.2ms/step | 2.6 samples/s +Step 3810/11700 | Loss: 2.6574 | LR: 4.10e-04 | 7772.5ms/step | 2.6 samples/s +Step 3820/11700 | Loss: 2.6426 | LR: 4.09e-04 | 7774.0ms/step | 2.6 samples/s +Step 3830/11700 | Loss: 2.6652 | LR: 4.09e-04 | 7772.8ms/step | 2.6 samples/s +Step 3840/11700 | Loss: 2.5947 | LR: 4.08e-04 | 7771.1ms/step | 2.6 samples/s +Step 3850/11700 | Loss: 2.7118 | LR: 4.07e-04 | 7772.0ms/step | 2.6 samples/s +Step 3860/11700 | Loss: 2.6307 | LR: 4.07e-04 | 7774.0ms/step | 2.6 samples/s +Step 3870/11700 | Loss: 2.6647 | LR: 4.06e-04 | 7773.7ms/step | 2.6 samples/s +Step 3880/11700 | Loss: 2.5481 | LR: 4.06e-04 | 7773.2ms/step | 2.6 samples/s +Step 3890/11700 | Loss: 2.7217 | LR: 4.05e-04 | 7773.6ms/step | 2.6 samples/s +Step 3900/11700 | Loss: 2.5928 | LR: 4.05e-04 | 7775.4ms/step | 2.6 samples/s +Step 3910/11700 | Loss: 2.6814 | LR: 4.04e-04 | 7773.9ms/step | 2.6 samples/s +Step 3920/11700 | Loss: 2.5886 | LR: 4.04e-04 | 7771.1ms/step | 2.6 samples/s +Step 3930/11700 | Loss: 2.6458 | LR: 4.03e-04 | 7760.7ms/step | 2.6 samples/s +Step 3940/11700 | Loss: 2.7853 | LR: 4.02e-04 | 7773.0ms/step | 2.6 samples/s +Step 3950/11700 | Loss: 2.6102 | LR: 4.02e-04 | 7773.3ms/step | 2.6 samples/s +Step 3960/11700 | Loss: 2.6625 | LR: 4.01e-04 | 7771.7ms/step | 2.6 samples/s +Step 3970/11700 | Loss: 2.6422 | LR: 4.01e-04 | 7775.3ms/step | 2.6 samples/s +Step 3980/11700 | Loss: 2.7038 | LR: 4.00e-04 | 7774.4ms/step | 2.6 samples/s +Step 3990/11700 | Loss: 2.6862 | LR: 4.00e-04 | 7768.7ms/step | 2.6 samples/s +Step 4000/11700 | Loss: 2.7374 | LR: 3.99e-04 | 7770.1ms/step | 2.6 samples/s +Validation at step 4000: Loss = 2.6626, Perplexity = 14.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-4000 +Step 4010/11700 | Loss: 2.5564 | LR: 3.98e-04 | 19617.7ms/step | 1.0 samples/s +Step 4020/11700 | Loss: 2.6545 | LR: 3.98e-04 | 7765.9ms/step | 2.6 samples/s +Step 4030/11700 | Loss: 2.5487 | LR: 3.97e-04 | 7767.2ms/step | 2.6 samples/s +Step 4040/11700 | Loss: 2.6881 | LR: 3.97e-04 | 7770.2ms/step | 2.6 samples/s +Step 4050/11700 | Loss: 2.6830 | LR: 3.96e-04 | 7770.2ms/step | 2.6 samples/s +Step 4060/11700 | Loss: 2.5570 | LR: 3.95e-04 | 7772.1ms/step | 2.6 samples/s +Step 4070/11700 | Loss: 2.7404 | LR: 3.95e-04 | 7772.5ms/step | 2.6 samples/s +Step 4080/11700 | Loss: 2.6298 | LR: 3.94e-04 | 7770.6ms/step | 2.6 samples/s +Step 4090/11700 | Loss: 2.6052 | LR: 3.94e-04 | 7764.1ms/step | 2.6 samples/s +Step 4100/11700 | Loss: 2.6464 | LR: 3.93e-04 | 7769.4ms/step | 2.6 samples/s +Step 4110/11700 | Loss: 2.6319 | LR: 3.93e-04 | 7770.8ms/step | 2.6 samples/s +Step 4120/11700 | Loss: 2.7015 | LR: 3.92e-04 | 7770.0ms/step | 2.6 samples/s +Step 4130/11700 | Loss: 2.5645 | LR: 3.91e-04 | 7771.6ms/step | 2.6 samples/s +Step 4140/11700 | Loss: 2.6128 | LR: 3.91e-04 | 7769.1ms/step | 2.6 samples/s +Step 4150/11700 | Loss: 2.5576 | LR: 3.90e-04 | 7770.0ms/step | 2.6 samples/s +Step 4160/11700 | Loss: 2.6320 | LR: 3.90e-04 | 7771.9ms/step | 2.6 samples/s +Step 4170/11700 | Loss: 2.6890 | LR: 3.89e-04 | 7770.3ms/step | 2.6 samples/s +Step 4180/11700 | Loss: 2.6293 | LR: 3.88e-04 | 7769.5ms/step | 2.6 samples/s +Step 4190/11700 | Loss: 2.6602 | LR: 3.88e-04 | 7771.0ms/step | 2.6 samples/s +Step 4200/11700 | Loss: 2.5753 | LR: 3.87e-04 | 7771.3ms/step | 2.6 samples/s +Step 4210/11700 | Loss: 2.6791 | LR: 3.87e-04 | 7767.4ms/step | 2.6 samples/s +Step 4220/11700 | Loss: 2.6452 | LR: 3.86e-04 | 7767.9ms/step | 2.6 samples/s +Step 4230/11700 | Loss: 2.5682 | LR: 3.85e-04 | 7770.0ms/step | 2.6 samples/s +Step 4240/11700 | Loss: 2.6828 | LR: 3.85e-04 | 7771.0ms/step | 2.6 samples/s +Step 4250/11700 | Loss: 2.7766 | LR: 3.84e-04 | 7767.1ms/step | 2.6 samples/s +Validation at step 4250: Loss = 2.6456, Perplexity = 14.09 +Step 4260/11700 | Loss: 2.7427 | LR: 3.84e-04 | 19297.1ms/step | 1.0 samples/s +Step 4270/11700 | Loss: 2.5905 | LR: 3.83e-04 | 7765.6ms/step | 2.6 samples/s +Step 4280/11700 | Loss: 2.6221 | LR: 3.82e-04 | 7769.3ms/step | 2.6 samples/s +Step 4290/11700 | Loss: 2.6287 | LR: 3.82e-04 | 7769.8ms/step | 2.6 samples/s +Step 4300/11700 | Loss: 2.6821 | LR: 3.81e-04 | 7768.7ms/step | 2.6 samples/s +Step 4310/11700 | Loss: 2.6061 | LR: 3.81e-04 | 7770.6ms/step | 2.6 samples/s +Step 4320/11700 | Loss: 2.5827 | LR: 3.80e-04 | 7769.7ms/step | 2.6 samples/s +Step 4330/11700 | Loss: 2.5930 | LR: 3.79e-04 | 7768.8ms/step | 2.6 samples/s +Step 4340/11700 | Loss: 2.6633 | LR: 3.79e-04 | 7768.1ms/step | 2.6 samples/s +Step 4350/11700 | Loss: 2.7383 | LR: 3.78e-04 | 7767.7ms/step | 2.6 samples/s +Step 4360/11700 | Loss: 2.5752 | LR: 3.77e-04 | 7768.7ms/step | 2.6 samples/s +Step 4370/11700 | Loss: 2.6121 | LR: 3.77e-04 | 7767.8ms/step | 2.6 samples/s +Step 4380/11700 | Loss: 2.5713 | LR: 3.76e-04 | 7767.4ms/step | 2.6 samples/s +Step 4390/11700 | Loss: 2.6445 | LR: 3.76e-04 | 7770.1ms/step | 2.6 samples/s +Step 4400/11700 | Loss: 2.6299 | LR: 3.75e-04 | 7770.1ms/step | 2.6 samples/s +Step 4410/11700 | Loss: 2.6507 | LR: 3.74e-04 | 7766.4ms/step | 2.6 samples/s +Step 4420/11700 | Loss: 2.6375 | LR: 3.74e-04 | 7769.8ms/step | 2.6 samples/s +Step 4430/11700 | Loss: 2.6646 | LR: 3.73e-04 | 7769.3ms/step | 2.6 samples/s +Step 4440/11700 | Loss: 2.6654 | LR: 3.73e-04 | 7768.4ms/step | 2.6 samples/s +Step 4450/11700 | Loss: 2.5990 | LR: 3.72e-04 | 7766.4ms/step | 2.6 samples/s +Step 4460/11700 | Loss: 2.7188 | LR: 3.71e-04 | 7764.9ms/step | 2.6 samples/s +Step 4470/11700 | Loss: 2.5219 | LR: 3.71e-04 | 7770.3ms/step | 2.6 samples/s +Step 4480/11700 | Loss: 2.5990 | LR: 3.70e-04 | 7769.1ms/step | 2.6 samples/s +Step 4490/11700 | Loss: 2.6111 | LR: 3.69e-04 | 7773.5ms/step | 2.6 samples/s +Step 4500/11700 | Loss: 2.6242 | LR: 3.69e-04 | 7768.8ms/step | 2.6 samples/s +Validation at step 4500: Loss = 2.6277, Perplexity = 13.84 +Step 4510/11700 | Loss: 2.5234 | LR: 3.68e-04 | 19313.3ms/step | 1.0 samples/s +Step 4520/11700 | Loss: 2.6524 | LR: 3.67e-04 | 7821.0ms/step | 2.6 samples/s +Step 4530/11700 | Loss: 2.7433 | LR: 3.67e-04 | 7825.0ms/step | 2.6 samples/s +Step 4540/11700 | Loss: 2.6663 | LR: 3.66e-04 | 7835.9ms/step | 2.6 samples/s +Step 4550/11700 | Loss: 2.5982 | LR: 3.66e-04 | 7831.0ms/step | 2.6 samples/s +Step 4560/11700 | Loss: 2.5687 | LR: 3.65e-04 | 7843.5ms/step | 2.5 samples/s +Step 4570/11700 | Loss: 2.5915 | LR: 3.64e-04 | 7804.2ms/step | 2.6 samples/s +Step 4580/11700 | Loss: 2.5595 | LR: 3.64e-04 | 7779.2ms/step | 2.6 samples/s +Step 4590/11700 | Loss: 2.7110 | LR: 3.63e-04 | 7774.3ms/step | 2.6 samples/s +Step 4600/11700 | Loss: 2.4597 | LR: 3.62e-04 | 7795.6ms/step | 2.6 samples/s +Step 4610/11700 | Loss: 2.5113 | LR: 3.62e-04 | 7771.1ms/step | 2.6 samples/s +Step 4620/11700 | Loss: 2.6229 | LR: 3.61e-04 | 7788.1ms/step | 2.6 samples/s +Step 4630/11700 | Loss: 2.6296 | LR: 3.60e-04 | 7831.2ms/step | 2.6 samples/s +Step 4640/11700 | Loss: 2.5770 | LR: 3.60e-04 | 7817.1ms/step | 2.6 samples/s +Step 4650/11700 | Loss: 2.5732 | LR: 3.59e-04 | 7832.6ms/step | 2.6 samples/s +Step 4660/11700 | Loss: 2.6424 | LR: 3.59e-04 | 7792.2ms/step | 2.6 samples/s +Step 4670/11700 | Loss: 2.5610 | LR: 3.58e-04 | 7785.6ms/step | 2.6 samples/s +Step 4680/11700 | Loss: 2.6483 | LR: 3.57e-04 | 7798.0ms/step | 2.6 samples/s +Step 4690/11700 | Loss: 2.5449 | LR: 3.57e-04 | 7801.0ms/step | 2.6 samples/s +Step 4700/11700 | Loss: 2.5464 | LR: 3.56e-04 | 7800.2ms/step | 2.6 samples/s +Step 4710/11700 | Loss: 2.7074 | LR: 3.55e-04 | 7800.7ms/step | 2.6 samples/s +Step 4720/11700 | Loss: 2.5725 | LR: 3.55e-04 | 7805.1ms/step | 2.6 samples/s +Step 4730/11700 | Loss: 2.6332 | LR: 3.54e-04 | 7803.3ms/step | 2.6 samples/s +Step 4740/11700 | Loss: 2.6430 | LR: 3.53e-04 | 7806.7ms/step | 2.6 samples/s +Step 4750/11700 | Loss: 2.5887 | LR: 3.53e-04 | 7798.8ms/step | 2.6 samples/s +Validation at step 4750: Loss = 2.6070, Perplexity = 13.56 +Step 4760/11700 | Loss: 2.6835 | LR: 3.52e-04 | 19397.2ms/step | 1.0 samples/s +Step 4770/11700 | Loss: 2.6405 | LR: 3.51e-04 | 7808.6ms/step | 2.6 samples/s +Step 4780/11700 | Loss: 2.6052 | LR: 3.51e-04 | 7806.5ms/step | 2.6 samples/s +Step 4790/11700 | Loss: 2.5807 | LR: 3.50e-04 | 7800.5ms/step | 2.6 samples/s +Step 4800/11700 | Loss: 2.5236 | LR: 3.49e-04 | 7801.4ms/step | 2.6 samples/s +Step 4810/11700 | Loss: 2.4949 | LR: 3.49e-04 | 7805.7ms/step | 2.6 samples/s +Step 4820/11700 | Loss: 2.5963 | LR: 3.48e-04 | 7800.5ms/step | 2.6 samples/s +Step 4830/11700 | Loss: 2.6169 | LR: 3.47e-04 | 7801.7ms/step | 2.6 samples/s +Step 4840/11700 | Loss: 2.6136 | LR: 3.47e-04 | 7805.0ms/step | 2.6 samples/s +Step 4850/11700 | Loss: 2.6221 | LR: 3.46e-04 | 7804.0ms/step | 2.6 samples/s +Step 4860/11700 | Loss: 2.5662 | LR: 3.45e-04 | 7801.1ms/step | 2.6 samples/s +Step 4870/11700 | Loss: 2.6917 | LR: 3.45e-04 | 7802.8ms/step | 2.6 samples/s +Step 4880/11700 | Loss: 2.6209 | LR: 3.44e-04 | 7802.0ms/step | 2.6 samples/s +Step 4890/11700 | Loss: 2.6831 | LR: 3.43e-04 | 7804.2ms/step | 2.6 samples/s +Step 4900/11700 | Loss: 2.6238 | LR: 3.43e-04 | 7797.8ms/step | 2.6 samples/s +Step 4910/11700 | Loss: 2.5729 | LR: 3.42e-04 | 7789.4ms/step | 2.6 samples/s +Step 4920/11700 | Loss: 2.5437 | LR: 3.41e-04 | 7799.5ms/step | 2.6 samples/s +Step 4930/11700 | Loss: 2.5115 | LR: 3.41e-04 | 7801.3ms/step | 2.6 samples/s +Step 4940/11700 | Loss: 2.6362 | LR: 3.40e-04 | 7801.5ms/step | 2.6 samples/s +Step 4950/11700 | Loss: 2.4540 | LR: 3.39e-04 | 7801.8ms/step | 2.6 samples/s +Step 4960/11700 | Loss: 2.6159 | LR: 3.39e-04 | 7800.7ms/step | 2.6 samples/s +Step 4970/11700 | Loss: 2.6017 | LR: 3.38e-04 | 7802.5ms/step | 2.6 samples/s +Step 4980/11700 | Loss: 2.7127 | LR: 3.37e-04 | 7792.0ms/step | 2.6 samples/s +Step 4990/11700 | Loss: 2.5300 | LR: 3.37e-04 | 7799.8ms/step | 2.6 samples/s +Step 5000/11700 | Loss: 2.6930 | LR: 3.36e-04 | 7800.4ms/step | 2.6 samples/s +Validation at step 5000: Loss = 2.5904, Perplexity = 13.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-5000 +Step 5010/11700 | Loss: 2.6921 | LR: 3.35e-04 | 19668.7ms/step | 1.0 samples/s +Step 5020/11700 | Loss: 2.6735 | LR: 3.35e-04 | 7799.0ms/step | 2.6 samples/s +Step 5030/11700 | Loss: 2.6672 | LR: 3.34e-04 | 7799.1ms/step | 2.6 samples/s +Step 5040/11700 | Loss: 2.5005 | LR: 3.33e-04 | 7800.2ms/step | 2.6 samples/s +Step 5050/11700 | Loss: 2.4407 | LR: 3.33e-04 | 7800.9ms/step | 2.6 samples/s +Step 5060/11700 | Loss: 2.6105 | LR: 3.32e-04 | 7800.9ms/step | 2.6 samples/s +Step 5070/11700 | Loss: 2.5694 | LR: 3.31e-04 | 7799.0ms/step | 2.6 samples/s +Step 5080/11700 | Loss: 2.7023 | LR: 3.31e-04 | 7800.1ms/step | 2.6 samples/s +Step 5090/11700 | Loss: 2.6046 | LR: 3.30e-04 | 7800.1ms/step | 2.6 samples/s +Step 5100/11700 | Loss: 2.7009 | LR: 3.29e-04 | 7798.5ms/step | 2.6 samples/s +Step 5110/11700 | Loss: 2.6003 | LR: 3.29e-04 | 7783.4ms/step | 2.6 samples/s +Step 5120/11700 | Loss: 2.5508 | LR: 3.28e-04 | 7779.4ms/step | 2.6 samples/s +Step 5130/11700 | Loss: 2.4662 | LR: 3.27e-04 | 7792.7ms/step | 2.6 samples/s +Step 5140/11700 | Loss: 2.5191 | LR: 3.27e-04 | 7800.6ms/step | 2.6 samples/s +Step 5150/11700 | Loss: 2.5877 | LR: 3.26e-04 | 7800.6ms/step | 2.6 samples/s +Step 5160/11700 | Loss: 2.5329 | LR: 3.25e-04 | 7810.2ms/step | 2.6 samples/s +Step 5170/11700 | Loss: 2.5402 | LR: 3.25e-04 | 7812.4ms/step | 2.6 samples/s +Step 5180/11700 | Loss: 2.5251 | LR: 3.24e-04 | 7809.8ms/step | 2.6 samples/s +Step 5190/11700 | Loss: 2.5973 | LR: 3.23e-04 | 7818.7ms/step | 2.6 samples/s +Step 5200/11700 | Loss: 2.4316 | LR: 3.22e-04 | 7834.8ms/step | 2.6 samples/s +Step 5210/11700 | Loss: 2.5473 | LR: 3.22e-04 | 7820.7ms/step | 2.6 samples/s +Step 5220/11700 | Loss: 2.6009 | LR: 3.21e-04 | 7821.2ms/step | 2.6 samples/s +Step 5230/11700 | Loss: 2.6767 | LR: 3.20e-04 | 7825.7ms/step | 2.6 samples/s +Step 5240/11700 | Loss: 2.5516 | LR: 3.20e-04 | 7833.1ms/step | 2.6 samples/s +Step 5250/11700 | Loss: 2.5910 | LR: 3.19e-04 | 7828.2ms/step | 2.6 samples/s +Validation at step 5250: Loss = 2.5729, Perplexity = 13.10 +Step 5260/11700 | Loss: 2.5837 | LR: 3.18e-04 | 19482.4ms/step | 1.0 samples/s +Step 5270/11700 | Loss: 2.5638 | LR: 3.18e-04 | 7825.4ms/step | 2.6 samples/s +Step 5280/11700 | Loss: 2.5207 | LR: 3.17e-04 | 7825.7ms/step | 2.6 samples/s +Step 5290/11700 | Loss: 2.5263 | LR: 3.16e-04 | 7816.7ms/step | 2.6 samples/s +Step 5300/11700 | Loss: 2.5433 | LR: 3.16e-04 | 7825.7ms/step | 2.6 samples/s +Step 5310/11700 | Loss: 2.5335 | LR: 3.15e-04 | 7814.3ms/step | 2.6 samples/s +Step 5320/11700 | Loss: 2.5086 | LR: 3.14e-04 | 7822.3ms/step | 2.6 samples/s +Step 5330/11700 | Loss: 2.4975 | LR: 3.13e-04 | 7835.3ms/step | 2.6 samples/s +Step 5340/11700 | Loss: 2.5322 | LR: 3.13e-04 | 7829.6ms/step | 2.6 samples/s +Step 5350/11700 | Loss: 2.5290 | LR: 3.12e-04 | 7826.0ms/step | 2.6 samples/s +Step 5360/11700 | Loss: 2.4582 | LR: 3.11e-04 | 7826.4ms/step | 2.6 samples/s +Step 5370/11700 | Loss: 2.5644 | LR: 3.11e-04 | 7821.5ms/step | 2.6 samples/s +Step 5380/11700 | Loss: 2.4625 | LR: 3.10e-04 | 7804.8ms/step | 2.6 samples/s +Step 5390/11700 | Loss: 2.6025 | LR: 3.09e-04 | 7822.0ms/step | 2.6 samples/s +Step 5400/11700 | Loss: 2.5107 | LR: 3.09e-04 | 7820.1ms/step | 2.6 samples/s +Step 5410/11700 | Loss: 2.4943 | LR: 3.08e-04 | 7817.2ms/step | 2.6 samples/s +Step 5420/11700 | Loss: 2.5665 | LR: 3.07e-04 | 7816.8ms/step | 2.6 samples/s +Step 5430/11700 | Loss: 2.6351 | LR: 3.07e-04 | 7819.9ms/step | 2.6 samples/s +Step 5440/11700 | Loss: 2.6398 | LR: 3.06e-04 | 7819.8ms/step | 2.6 samples/s +Step 5450/11700 | Loss: 2.5230 | LR: 3.05e-04 | 7820.9ms/step | 2.6 samples/s +Step 5460/11700 | Loss: 2.5624 | LR: 3.04e-04 | 7815.4ms/step | 2.6 samples/s +Step 5470/11700 | Loss: 2.4260 | LR: 3.04e-04 | 7819.7ms/step | 2.6 samples/s +Step 5480/11700 | Loss: 2.5305 | LR: 3.03e-04 | 7819.7ms/step | 2.6 samples/s +Step 5490/11700 | Loss: 2.6940 | LR: 3.02e-04 | 7813.4ms/step | 2.6 samples/s +Step 5500/11700 | Loss: 2.5992 | LR: 3.02e-04 | 7792.6ms/step | 2.6 samples/s +Validation at step 5500: Loss = 2.5577, Perplexity = 12.91 +Step 5510/11700 | Loss: 2.4369 | LR: 3.01e-04 | 19413.8ms/step | 1.0 samples/s +Step 5520/11700 | Loss: 2.4177 | LR: 3.00e-04 | 7819.4ms/step | 2.6 samples/s +Step 5530/11700 | Loss: 2.5758 | LR: 3.00e-04 | 7825.8ms/step | 2.6 samples/s +Step 5540/11700 | Loss: 2.5552 | LR: 2.99e-04 | 7826.8ms/step | 2.6 samples/s +Step 5550/11700 | Loss: 2.5621 | LR: 2.98e-04 | 7826.4ms/step | 2.6 samples/s +Step 5560/11700 | Loss: 2.6024 | LR: 2.97e-04 | 7828.4ms/step | 2.6 samples/s +Step 5570/11700 | Loss: 2.4767 | LR: 2.97e-04 | 7828.7ms/step | 2.6 samples/s +Step 5580/11700 | Loss: 2.5686 | LR: 2.96e-04 | 7828.6ms/step | 2.6 samples/s +Step 5590/11700 | Loss: 2.5936 | LR: 2.95e-04 | 7826.6ms/step | 2.6 samples/s +Step 5600/11700 | Loss: 2.5386 | LR: 2.95e-04 | 7827.9ms/step | 2.6 samples/s +Step 5610/11700 | Loss: 2.3961 | LR: 2.94e-04 | 7827.8ms/step | 2.6 samples/s +Step 5620/11700 | Loss: 2.5558 | LR: 2.93e-04 | 7827.2ms/step | 2.6 samples/s +Step 5630/11700 | Loss: 2.4468 | LR: 2.92e-04 | 7825.5ms/step | 2.6 samples/s +Step 5640/11700 | Loss: 2.5780 | LR: 2.92e-04 | 7827.2ms/step | 2.6 samples/s +Step 5650/11700 | Loss: 2.6305 | LR: 2.91e-04 | 7827.8ms/step | 2.6 samples/s +Step 5660/11700 | Loss: 2.5379 | LR: 2.90e-04 | 7826.4ms/step | 2.6 samples/s +Step 5670/11700 | Loss: 2.5231 | LR: 2.90e-04 | 7824.9ms/step | 2.6 samples/s +Step 5680/11700 | Loss: 2.5948 | LR: 2.89e-04 | 7826.6ms/step | 2.6 samples/s +Step 5690/11700 | Loss: 2.5084 | LR: 2.88e-04 | 7828.2ms/step | 2.6 samples/s +Step 5700/11700 | Loss: 2.5294 | LR: 2.88e-04 | 7827.5ms/step | 2.6 samples/s +Step 5710/11700 | Loss: 2.5041 | LR: 2.87e-04 | 7827.7ms/step | 2.6 samples/s +Step 5720/11700 | Loss: 2.4807 | LR: 2.86e-04 | 7827.5ms/step | 2.6 samples/s +Step 5730/11700 | Loss: 2.5959 | LR: 2.85e-04 | 7822.6ms/step | 2.6 samples/s +Step 5740/11700 | Loss: 2.4633 | LR: 2.85e-04 | 7820.7ms/step | 2.6 samples/s +Step 5750/11700 | Loss: 2.4992 | LR: 2.84e-04 | 7821.6ms/step | 2.6 samples/s +Validation at step 5750: Loss = 2.5464, Perplexity = 12.76 +Step 5760/11700 | Loss: 2.5367 | LR: 2.83e-04 | 19466.7ms/step | 1.0 samples/s +Step 5770/11700 | Loss: 2.6648 | LR: 2.83e-04 | 7822.6ms/step | 2.6 samples/s +Step 5780/11700 | Loss: 2.5031 | LR: 2.82e-04 | 7814.1ms/step | 2.6 samples/s +Step 5790/11700 | Loss: 2.6283 | LR: 2.81e-04 | 7803.9ms/step | 2.6 samples/s +Step 5800/11700 | Loss: 2.5959 | LR: 2.80e-04 | 7808.1ms/step | 2.6 samples/s +Step 5810/11700 | Loss: 2.5697 | LR: 2.80e-04 | 7811.0ms/step | 2.6 samples/s +Step 5820/11700 | Loss: 2.4641 | LR: 2.79e-04 | 7814.7ms/step | 2.6 samples/s +Step 5830/11700 | Loss: 2.5155 | LR: 2.78e-04 | 7812.6ms/step | 2.6 samples/s +Step 5840/11700 | Loss: 2.5064 | LR: 2.78e-04 | 7813.7ms/step | 2.6 samples/s +Step 5850/11700 | Loss: 2.5130 | LR: 2.77e-04 | 7821.1ms/step | 2.6 samples/s +Step 5860/11700 | Loss: 2.5860 | LR: 2.76e-04 | 7813.8ms/step | 2.6 samples/s +Step 5870/11700 | Loss: 2.5945 | LR: 2.75e-04 | 7804.9ms/step | 2.6 samples/s +Step 5880/11700 | Loss: 2.5727 | LR: 2.75e-04 | 7811.4ms/step | 2.6 samples/s +Step 5890/11700 | Loss: 2.5089 | LR: 2.74e-04 | 7800.9ms/step | 2.6 samples/s +Step 5900/11700 | Loss: 2.5219 | LR: 2.73e-04 | 7816.9ms/step | 2.6 samples/s +Step 5910/11700 | Loss: 2.5700 | LR: 2.73e-04 | 7823.1ms/step | 2.6 samples/s +Step 5920/11700 | Loss: 2.4796 | LR: 2.72e-04 | 7825.0ms/step | 2.6 samples/s +Step 5930/11700 | Loss: 2.5087 | LR: 2.71e-04 | 7820.4ms/step | 2.6 samples/s +Step 5940/11700 | Loss: 2.4384 | LR: 2.70e-04 | 7805.5ms/step | 2.6 samples/s +Step 5950/11700 | Loss: 2.6062 | LR: 2.70e-04 | 7798.7ms/step | 2.6 samples/s +Step 5960/11700 | Loss: 2.6311 | LR: 2.69e-04 | 7799.0ms/step | 2.6 samples/s +Step 5970/11700 | Loss: 2.5213 | LR: 2.68e-04 | 7799.7ms/step | 2.6 samples/s +Step 5980/11700 | Loss: 2.5393 | LR: 2.68e-04 | 7800.1ms/step | 2.6 samples/s +Step 5990/11700 | Loss: 2.5306 | LR: 2.67e-04 | 7798.4ms/step | 2.6 samples/s +Step 6000/11700 | Loss: 2.5261 | LR: 2.66e-04 | 7802.0ms/step | 2.6 samples/s +Validation at step 6000: Loss = 2.5339, Perplexity = 12.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-6000 +Step 6010/11700 | Loss: 2.5800 | LR: 2.65e-04 | 19724.1ms/step | 1.0 samples/s +Step 6020/11700 | Loss: 2.4430 | LR: 2.65e-04 | 7804.9ms/step | 2.6 samples/s +Step 6030/11700 | Loss: 2.5022 | LR: 2.64e-04 | 7804.9ms/step | 2.6 samples/s +Step 6040/11700 | Loss: 2.4668 | LR: 2.63e-04 | 7810.1ms/step | 2.6 samples/s +Step 6050/11700 | Loss: 2.4693 | LR: 2.63e-04 | 7805.1ms/step | 2.6 samples/s +Step 6060/11700 | Loss: 2.5706 | LR: 2.62e-04 | 7807.2ms/step | 2.6 samples/s +Step 6070/11700 | Loss: 2.5613 | LR: 2.61e-04 | 7807.5ms/step | 2.6 samples/s +Step 6080/11700 | Loss: 2.5921 | LR: 2.60e-04 | 7803.2ms/step | 2.6 samples/s +Step 6090/11700 | Loss: 2.4603 | LR: 2.60e-04 | 7799.6ms/step | 2.6 samples/s +Step 6100/11700 | Loss: 2.5337 | LR: 2.59e-04 | 7805.8ms/step | 2.6 samples/s +Step 6110/11700 | Loss: 2.4657 | LR: 2.58e-04 | 7804.7ms/step | 2.6 samples/s +Step 6120/11700 | Loss: 2.5920 | LR: 2.58e-04 | 7803.7ms/step | 2.6 samples/s +Step 6130/11700 | Loss: 2.5223 | LR: 2.57e-04 | 7803.0ms/step | 2.6 samples/s +Step 6140/11700 | Loss: 2.5337 | LR: 2.56e-04 | 7810.0ms/step | 2.6 samples/s +Step 6150/11700 | Loss: 2.5059 | LR: 2.55e-04 | 7806.4ms/step | 2.6 samples/s +Step 6160/11700 | Loss: 2.4447 | LR: 2.55e-04 | 7809.7ms/step | 2.6 samples/s +Step 6170/11700 | Loss: 2.4559 | LR: 2.54e-04 | 7802.9ms/step | 2.6 samples/s +Step 6180/11700 | Loss: 2.4236 | LR: 2.53e-04 | 7801.0ms/step | 2.6 samples/s +Step 6190/11700 | Loss: 2.4851 | LR: 2.53e-04 | 7804.4ms/step | 2.6 samples/s +Step 6200/11700 | Loss: 2.6110 | LR: 2.52e-04 | 7806.8ms/step | 2.6 samples/s +Step 6210/11700 | Loss: 2.5930 | LR: 2.51e-04 | 7806.8ms/step | 2.6 samples/s +Step 6220/11700 | Loss: 2.5394 | LR: 2.50e-04 | 7814.3ms/step | 2.6 samples/s +Step 6230/11700 | Loss: 2.6473 | LR: 2.50e-04 | 7806.7ms/step | 2.6 samples/s +Step 6240/11700 | Loss: 2.5061 | LR: 2.49e-04 | 7799.9ms/step | 2.6 samples/s +Step 6250/11700 | Loss: 2.5439 | LR: 2.48e-04 | 7806.9ms/step | 2.6 samples/s +Validation at step 6250: Loss = 2.5226, Perplexity = 12.46 +Step 6260/11700 | Loss: 2.4395 | LR: 2.47e-04 | 19422.1ms/step | 1.0 samples/s +Step 6270/11700 | Loss: 2.5352 | LR: 2.47e-04 | 7796.3ms/step | 2.6 samples/s +Step 6280/11700 | Loss: 2.5144 | LR: 2.46e-04 | 7793.5ms/step | 2.6 samples/s +Step 6290/11700 | Loss: 2.4924 | LR: 2.45e-04 | 7803.4ms/step | 2.6 samples/s +Step 6300/11700 | Loss: 2.5009 | LR: 2.45e-04 | 7811.3ms/step | 2.6 samples/s +Step 6310/11700 | Loss: 2.4944 | LR: 2.44e-04 | 7814.9ms/step | 2.6 samples/s +Step 6320/11700 | Loss: 2.5597 | LR: 2.43e-04 | 7824.6ms/step | 2.6 samples/s +Step 6330/11700 | Loss: 2.6233 | LR: 2.42e-04 | 7823.0ms/step | 2.6 samples/s +Step 6340/11700 | Loss: 2.5349 | LR: 2.42e-04 | 7816.6ms/step | 2.6 samples/s +Step 6350/11700 | Loss: 2.5434 | LR: 2.41e-04 | 7814.5ms/step | 2.6 samples/s +Step 6360/11700 | Loss: 2.4347 | LR: 2.40e-04 | 7814.6ms/step | 2.6 samples/s +Step 6370/11700 | Loss: 2.4254 | LR: 2.40e-04 | 7823.4ms/step | 2.6 samples/s +Step 6380/11700 | Loss: 2.3966 | LR: 2.39e-04 | 7820.4ms/step | 2.6 samples/s +Step 6390/11700 | Loss: 2.5298 | LR: 2.38e-04 | 7821.7ms/step | 2.6 samples/s +Step 6400/11700 | Loss: 2.5052 | LR: 2.37e-04 | 7824.5ms/step | 2.6 samples/s +Step 6410/11700 | Loss: 2.4332 | LR: 2.37e-04 | 7827.2ms/step | 2.6 samples/s +Step 6420/11700 | Loss: 2.4320 | LR: 2.36e-04 | 7819.5ms/step | 2.6 samples/s +Step 6430/11700 | Loss: 2.3275 | LR: 2.35e-04 | 7817.7ms/step | 2.6 samples/s +Step 6440/11700 | Loss: 2.4999 | LR: 2.35e-04 | 7825.5ms/step | 2.6 samples/s +Step 6450/11700 | Loss: 2.4388 | LR: 2.34e-04 | 7825.4ms/step | 2.6 samples/s +Step 6460/11700 | Loss: 2.5712 | LR: 2.33e-04 | 7822.4ms/step | 2.6 samples/s +Step 6470/11700 | Loss: 2.4351 | LR: 2.32e-04 | 7812.8ms/step | 2.6 samples/s +Step 6480/11700 | Loss: 2.6843 | LR: 2.32e-04 | 7816.0ms/step | 2.6 samples/s +Step 6490/11700 | Loss: 2.4916 | LR: 2.31e-04 | 7818.0ms/step | 2.6 samples/s +Step 6500/11700 | Loss: 2.5258 | LR: 2.30e-04 | 7811.3ms/step | 2.6 samples/s +Validation at step 6500: Loss = 2.5124, Perplexity = 12.33 +Step 6510/11700 | Loss: 2.5044 | LR: 2.30e-04 | 19437.9ms/step | 1.0 samples/s +Step 6520/11700 | Loss: 2.5051 | LR: 2.29e-04 | 7811.8ms/step | 2.6 samples/s +Step 6530/11700 | Loss: 2.4231 | LR: 2.28e-04 | 7814.4ms/step | 2.6 samples/s +Step 6540/11700 | Loss: 2.5749 | LR: 2.27e-04 | 7811.4ms/step | 2.6 samples/s +Step 6550/11700 | Loss: 2.4755 | LR: 2.27e-04 | 7813.0ms/step | 2.6 samples/s +Step 6560/11700 | Loss: 2.4271 | LR: 2.26e-04 | 7803.8ms/step | 2.6 samples/s +Step 6570/11700 | Loss: 2.3821 | LR: 2.25e-04 | 7808.8ms/step | 2.6 samples/s +Step 6580/11700 | Loss: 2.5421 | LR: 2.25e-04 | 7820.2ms/step | 2.6 samples/s +Step 6590/11700 | Loss: 2.5226 | LR: 2.24e-04 | 7814.6ms/step | 2.6 samples/s +Step 6600/11700 | Loss: 2.5348 | LR: 2.23e-04 | 7822.2ms/step | 2.6 samples/s +Step 6610/11700 | Loss: 2.6074 | LR: 2.22e-04 | 7824.0ms/step | 2.6 samples/s +Step 6620/11700 | Loss: 2.4071 | LR: 2.22e-04 | 7823.6ms/step | 2.6 samples/s +Step 6630/11700 | Loss: 2.5150 | LR: 2.21e-04 | 7822.6ms/step | 2.6 samples/s +Step 6640/11700 | Loss: 2.5228 | LR: 2.20e-04 | 7808.7ms/step | 2.6 samples/s +Step 6650/11700 | Loss: 2.4555 | LR: 2.20e-04 | 7817.0ms/step | 2.6 samples/s +Step 6660/11700 | Loss: 2.4791 | LR: 2.19e-04 | 7821.1ms/step | 2.6 samples/s +Step 6670/11700 | Loss: 2.4724 | LR: 2.18e-04 | 7819.3ms/step | 2.6 samples/s +Step 6680/11700 | Loss: 2.4702 | LR: 2.17e-04 | 7820.7ms/step | 2.6 samples/s +Step 6690/11700 | Loss: 2.5599 | LR: 2.17e-04 | 7825.7ms/step | 2.6 samples/s +Step 6700/11700 | Loss: 2.4991 | LR: 2.16e-04 | 7825.2ms/step | 2.6 samples/s +Step 6710/11700 | Loss: 2.5114 | LR: 2.15e-04 | 7814.8ms/step | 2.6 samples/s +Step 6720/11700 | Loss: 2.3938 | LR: 2.15e-04 | 7809.6ms/step | 2.6 samples/s +Step 6730/11700 | Loss: 2.4530 | LR: 2.14e-04 | 7784.3ms/step | 2.6 samples/s +Step 6740/11700 | Loss: 2.4708 | LR: 2.13e-04 | 7792.0ms/step | 2.6 samples/s +Step 6750/11700 | Loss: 2.4540 | LR: 2.12e-04 | 7822.4ms/step | 2.6 samples/s +Validation at step 6750: Loss = 2.5047, Perplexity = 12.24 +Step 6760/11700 | Loss: 2.5274 | LR: 2.12e-04 | 19428.3ms/step | 1.0 samples/s +Step 6770/11700 | Loss: 2.4930 | LR: 2.11e-04 | 7792.0ms/step | 2.6 samples/s +Step 6780/11700 | Loss: 2.4326 | LR: 2.10e-04 | 7789.1ms/step | 2.6 samples/s +Step 6790/11700 | Loss: 2.4376 | LR: 2.10e-04 | 7788.4ms/step | 2.6 samples/s +Step 6800/11700 | Loss: 2.5184 | LR: 2.09e-04 | 7790.7ms/step | 2.6 samples/s +Step 6810/11700 | Loss: 2.6868 | LR: 2.08e-04 | 7785.0ms/step | 2.6 samples/s +Step 6820/11700 | Loss: 2.5269 | LR: 2.08e-04 | 7781.8ms/step | 2.6 samples/s +Step 6830/11700 | Loss: 2.4382 | LR: 2.07e-04 | 7783.3ms/step | 2.6 samples/s +Step 6840/11700 | Loss: 2.4574 | LR: 2.06e-04 | 7781.7ms/step | 2.6 samples/s +Step 6850/11700 | Loss: 2.5312 | LR: 2.05e-04 | 7790.3ms/step | 2.6 samples/s +Step 6860/11700 | Loss: 2.4709 | LR: 2.05e-04 | 7784.6ms/step | 2.6 samples/s +Step 6870/11700 | Loss: 2.4593 | LR: 2.04e-04 | 7784.5ms/step | 2.6 samples/s +Step 6880/11700 | Loss: 2.4602 | LR: 2.03e-04 | 7784.8ms/step | 2.6 samples/s +Step 6890/11700 | Loss: 2.4811 | LR: 2.03e-04 | 7784.6ms/step | 2.6 samples/s +Step 6900/11700 | Loss: 2.6042 | LR: 2.02e-04 | 7785.0ms/step | 2.6 samples/s +Step 6910/11700 | Loss: 2.5719 | LR: 2.01e-04 | 7786.7ms/step | 2.6 samples/s +Step 6920/11700 | Loss: 2.4285 | LR: 2.00e-04 | 7784.4ms/step | 2.6 samples/s +Step 6930/11700 | Loss: 2.3769 | LR: 2.00e-04 | 7782.7ms/step | 2.6 samples/s +Step 6940/11700 | Loss: 2.5880 | LR: 1.99e-04 | 7781.0ms/step | 2.6 samples/s +Step 6950/11700 | Loss: 2.5362 | LR: 1.98e-04 | 7782.7ms/step | 2.6 samples/s +Step 6960/11700 | Loss: 2.4841 | LR: 1.98e-04 | 7780.0ms/step | 2.6 samples/s +Step 6970/11700 | Loss: 2.3318 | LR: 1.97e-04 | 7782.7ms/step | 2.6 samples/s +Step 6980/11700 | Loss: 2.4924 | LR: 1.96e-04 | 7779.4ms/step | 2.6 samples/s +Step 6990/11700 | Loss: 2.5604 | LR: 1.96e-04 | 7780.4ms/step | 2.6 samples/s +Step 7000/11700 | Loss: 2.4931 | LR: 1.95e-04 | 7783.1ms/step | 2.6 samples/s +Validation at step 7000: Loss = 2.4955, Perplexity = 12.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-7000 +Step 7010/11700 | Loss: 2.5957 | LR: 1.94e-04 | 19639.1ms/step | 1.0 samples/s +Step 7020/11700 | Loss: 2.4764 | LR: 1.93e-04 | 7776.9ms/step | 2.6 samples/s +Step 7030/11700 | Loss: 2.5694 | LR: 1.93e-04 | 7784.2ms/step | 2.6 samples/s +Step 7040/11700 | Loss: 2.5455 | LR: 1.92e-04 | 7785.4ms/step | 2.6 samples/s +Step 7050/11700 | Loss: 2.4175 | LR: 1.91e-04 | 7788.1ms/step | 2.6 samples/s +Step 7060/11700 | Loss: 2.5392 | LR: 1.91e-04 | 7771.8ms/step | 2.6 samples/s +Step 7070/11700 | Loss: 2.4667 | LR: 1.90e-04 | 7781.7ms/step | 2.6 samples/s +Step 7080/11700 | Loss: 2.4816 | LR: 1.89e-04 | 7797.6ms/step | 2.6 samples/s +Step 7090/11700 | Loss: 2.4846 | LR: 1.89e-04 | 7804.4ms/step | 2.6 samples/s +Step 7100/11700 | Loss: 2.5226 | LR: 1.88e-04 | 7812.6ms/step | 2.6 samples/s +Step 7110/11700 | Loss: 2.4640 | LR: 1.87e-04 | 7820.1ms/step | 2.6 samples/s +Step 7120/11700 | Loss: 2.4084 | LR: 1.87e-04 | 7798.4ms/step | 2.6 samples/s +Step 7130/11700 | Loss: 2.4072 | LR: 1.86e-04 | 7790.7ms/step | 2.6 samples/s +Step 7140/11700 | Loss: 2.4666 | LR: 1.85e-04 | 7796.5ms/step | 2.6 samples/s +Step 7150/11700 | Loss: 2.5472 | LR: 1.84e-04 | 7817.5ms/step | 2.6 samples/s +Step 7160/11700 | Loss: 2.4615 | LR: 1.84e-04 | 7813.7ms/step | 2.6 samples/s +Step 7170/11700 | Loss: 2.4994 | LR: 1.83e-04 | 7809.9ms/step | 2.6 samples/s +Step 7180/11700 | Loss: 2.4072 | LR: 1.82e-04 | 7800.1ms/step | 2.6 samples/s +Step 7190/11700 | Loss: 2.4095 | LR: 1.82e-04 | 7794.5ms/step | 2.6 samples/s +Step 7200/11700 | Loss: 2.3742 | LR: 1.81e-04 | 7819.6ms/step | 2.6 samples/s +Step 7210/11700 | Loss: 2.3187 | LR: 1.80e-04 | 7812.0ms/step | 2.6 samples/s +Step 7220/11700 | Loss: 2.4567 | LR: 1.80e-04 | 7803.6ms/step | 2.6 samples/s +Step 7230/11700 | Loss: 2.5009 | LR: 1.79e-04 | 7796.9ms/step | 2.6 samples/s +Step 7240/11700 | Loss: 2.4589 | LR: 1.78e-04 | 7801.2ms/step | 2.6 samples/s +Step 7250/11700 | Loss: 2.3306 | LR: 1.78e-04 | 7802.5ms/step | 2.6 samples/s +Validation at step 7250: Loss = 2.4880, Perplexity = 12.04 +Step 7260/11700 | Loss: 2.4482 | LR: 1.77e-04 | 19411.7ms/step | 1.0 samples/s +Step 7270/11700 | Loss: 2.5023 | LR: 1.76e-04 | 7807.0ms/step | 2.6 samples/s +Step 7280/11700 | Loss: 2.5073 | LR: 1.75e-04 | 7800.9ms/step | 2.6 samples/s +Step 7290/11700 | Loss: 2.5019 | LR: 1.75e-04 | 7802.9ms/step | 2.6 samples/s +Step 7300/11700 | Loss: 2.4238 | LR: 1.74e-04 | 7802.8ms/step | 2.6 samples/s +Step 7310/11700 | Loss: 2.5480 | LR: 1.73e-04 | 7799.3ms/step | 2.6 samples/s +Step 7320/11700 | Loss: 2.4474 | LR: 1.73e-04 | 7799.4ms/step | 2.6 samples/s +Step 7330/11700 | Loss: 2.5160 | LR: 1.72e-04 | 7816.7ms/step | 2.6 samples/s +Step 7340/11700 | Loss: 2.4545 | LR: 1.71e-04 | 7840.5ms/step | 2.6 samples/s +Step 7350/11700 | Loss: 2.4382 | LR: 1.71e-04 | 7824.5ms/step | 2.6 samples/s +Step 7360/11700 | Loss: 2.3920 | LR: 1.70e-04 | 7815.3ms/step | 2.6 samples/s +Step 7370/11700 | Loss: 2.4508 | LR: 1.69e-04 | 7842.8ms/step | 2.6 samples/s +Step 7380/11700 | Loss: 2.4892 | LR: 1.69e-04 | 7848.6ms/step | 2.5 samples/s +Step 7390/11700 | Loss: 2.4554 | LR: 1.68e-04 | 7847.3ms/step | 2.5 samples/s +Step 7400/11700 | Loss: 2.4894 | LR: 1.67e-04 | 7848.3ms/step | 2.5 samples/s +Step 7410/11700 | Loss: 2.4119 | LR: 1.67e-04 | 7826.7ms/step | 2.6 samples/s +Step 7420/11700 | Loss: 2.3951 | LR: 1.66e-04 | 7822.0ms/step | 2.6 samples/s +Step 7430/11700 | Loss: 2.4082 | LR: 1.65e-04 | 7843.1ms/step | 2.5 samples/s +Step 7440/11700 | Loss: 2.5110 | LR: 1.65e-04 | 7850.1ms/step | 2.5 samples/s +Step 7450/11700 | Loss: 2.4915 | LR: 1.64e-04 | 7850.3ms/step | 2.5 samples/s +Step 7460/11700 | Loss: 2.4979 | LR: 1.63e-04 | 7846.9ms/step | 2.5 samples/s +Step 7470/11700 | Loss: 2.4507 | LR: 1.63e-04 | 7824.8ms/step | 2.6 samples/s +Step 7480/11700 | Loss: 2.5832 | LR: 1.62e-04 | 7810.2ms/step | 2.6 samples/s +Step 7490/11700 | Loss: 2.4159 | LR: 1.61e-04 | 7837.1ms/step | 2.6 samples/s +Step 7500/11700 | Loss: 2.4072 | LR: 1.61e-04 | 7842.2ms/step | 2.6 samples/s +Validation at step 7500: Loss = 2.4818, Perplexity = 11.96 +Step 7510/11700 | Loss: 2.4921 | LR: 1.60e-04 | 19529.6ms/step | 1.0 samples/s +Step 7520/11700 | Loss: 2.5159 | LR: 1.59e-04 | 7817.7ms/step | 2.6 samples/s +Step 7530/11700 | Loss: 2.4188 | LR: 1.59e-04 | 7830.7ms/step | 2.6 samples/s +Step 7540/11700 | Loss: 2.4978 | LR: 1.58e-04 | 7841.5ms/step | 2.6 samples/s +Step 7550/11700 | Loss: 2.4444 | LR: 1.57e-04 | 7844.4ms/step | 2.5 samples/s +Step 7560/11700 | Loss: 2.5038 | LR: 1.57e-04 | 7847.1ms/step | 2.5 samples/s +Step 7570/11700 | Loss: 2.4642 | LR: 1.56e-04 | 7848.9ms/step | 2.5 samples/s +Step 7580/11700 | Loss: 2.4506 | LR: 1.55e-04 | 7823.4ms/step | 2.6 samples/s +Step 7590/11700 | Loss: 2.4324 | LR: 1.55e-04 | 7809.9ms/step | 2.6 samples/s +Step 7600/11700 | Loss: 2.4882 | LR: 1.54e-04 | 7840.8ms/step | 2.6 samples/s +Step 7610/11700 | Loss: 2.5838 | LR: 1.53e-04 | 7845.5ms/step | 2.5 samples/s +Step 7620/11700 | Loss: 2.5510 | LR: 1.53e-04 | 7847.4ms/step | 2.5 samples/s +Step 7630/11700 | Loss: 2.4245 | LR: 1.52e-04 | 7842.1ms/step | 2.6 samples/s +Step 7640/11700 | Loss: 2.4589 | LR: 1.51e-04 | 7836.6ms/step | 2.6 samples/s +Step 7650/11700 | Loss: 2.4951 | LR: 1.51e-04 | 7801.7ms/step | 2.6 samples/s +Step 7660/11700 | Loss: 2.4432 | LR: 1.50e-04 | 7822.1ms/step | 2.6 samples/s +Step 7670/11700 | Loss: 2.3815 | LR: 1.49e-04 | 7842.9ms/step | 2.6 samples/s +Step 7680/11700 | Loss: 2.4447 | LR: 1.49e-04 | 7848.1ms/step | 2.5 samples/s +Step 7690/11700 | Loss: 2.5231 | LR: 1.48e-04 | 7849.1ms/step | 2.5 samples/s +Step 7700/11700 | Loss: 2.5877 | LR: 1.47e-04 | 7849.0ms/step | 2.5 samples/s +Step 7710/11700 | Loss: 2.3894 | LR: 1.47e-04 | 7820.8ms/step | 2.6 samples/s +Step 7720/11700 | Loss: 2.4913 | LR: 1.46e-04 | 7798.0ms/step | 2.6 samples/s +Step 7730/11700 | Loss: 2.3849 | LR: 1.45e-04 | 7822.2ms/step | 2.6 samples/s +Step 7740/11700 | Loss: 2.5442 | LR: 1.45e-04 | 7841.8ms/step | 2.6 samples/s +Step 7750/11700 | Loss: 2.4609 | LR: 1.44e-04 | 7847.4ms/step | 2.5 samples/s +Validation at step 7750: Loss = 2.4755, Perplexity = 11.89 +Step 7760/11700 | Loss: 2.5763 | LR: 1.43e-04 | 19523.8ms/step | 1.0 samples/s +Step 7770/11700 | Loss: 2.4406 | LR: 1.43e-04 | 7800.5ms/step | 2.6 samples/s +Step 7780/11700 | Loss: 2.5355 | LR: 1.42e-04 | 7806.2ms/step | 2.6 samples/s +Step 7790/11700 | Loss: 2.4365 | LR: 1.41e-04 | 7836.1ms/step | 2.6 samples/s +Step 7800/11700 | Loss: 2.5216 | LR: 1.41e-04 | 7839.5ms/step | 2.6 samples/s +Step 7810/11700 | Loss: 2.3641 | LR: 1.40e-04 | 7836.4ms/step | 2.6 samples/s +Step 7820/11700 | Loss: 2.4079 | LR: 1.40e-04 | 7840.9ms/step | 2.6 samples/s +Step 7830/11700 | Loss: 2.4113 | LR: 1.39e-04 | 7802.0ms/step | 2.6 samples/s +Step 7840/11700 | Loss: 2.4726 | LR: 1.38e-04 | 7797.8ms/step | 2.6 samples/s +Step 7850/11700 | Loss: 2.4930 | LR: 1.38e-04 | 7823.5ms/step | 2.6 samples/s +Step 7860/11700 | Loss: 2.4896 | LR: 1.37e-04 | 7838.7ms/step | 2.6 samples/s +Step 7870/11700 | Loss: 2.5296 | LR: 1.36e-04 | 7839.1ms/step | 2.6 samples/s +Step 7880/11700 | Loss: 2.4859 | LR: 1.36e-04 | 7841.4ms/step | 2.6 samples/s +Step 7890/11700 | Loss: 2.4428 | LR: 1.35e-04 | 7843.0ms/step | 2.6 samples/s +Step 7900/11700 | Loss: 2.5441 | LR: 1.34e-04 | 7816.2ms/step | 2.6 samples/s +Step 7910/11700 | Loss: 2.3239 | LR: 1.34e-04 | 7823.0ms/step | 2.6 samples/s +Step 7920/11700 | Loss: 2.4270 | LR: 1.33e-04 | 7837.9ms/step | 2.6 samples/s +Step 7930/11700 | Loss: 2.4368 | LR: 1.33e-04 | 7840.7ms/step | 2.6 samples/s +Step 7940/11700 | Loss: 2.4876 | LR: 1.32e-04 | 7824.9ms/step | 2.6 samples/s +Step 7950/11700 | Loss: 2.5097 | LR: 1.31e-04 | 7837.0ms/step | 2.6 samples/s +Step 7960/11700 | Loss: 2.4358 | LR: 1.31e-04 | 7823.6ms/step | 2.6 samples/s +Step 7970/11700 | Loss: 2.3846 | LR: 1.30e-04 | 7811.3ms/step | 2.6 samples/s +Step 7980/11700 | Loss: 2.5386 | LR: 1.29e-04 | 7838.9ms/step | 2.6 samples/s +Step 7990/11700 | Loss: 2.5148 | LR: 1.29e-04 | 7847.3ms/step | 2.5 samples/s +Step 8000/11700 | Loss: 2.3986 | LR: 1.28e-04 | 7846.4ms/step | 2.5 samples/s +Validation at step 8000: Loss = 2.4708, Perplexity = 11.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-8000 +Step 8010/11700 | Loss: 2.4861 | LR: 1.27e-04 | 19781.3ms/step | 1.0 samples/s +Step 8020/11700 | Loss: 2.3606 | LR: 1.27e-04 | 7836.4ms/step | 2.6 samples/s +Step 8030/11700 | Loss: 2.4509 | LR: 1.26e-04 | 7840.8ms/step | 2.6 samples/s +Step 8040/11700 | Loss: 2.4581 | LR: 1.26e-04 | 7838.8ms/step | 2.6 samples/s +Step 8050/11700 | Loss: 2.4202 | LR: 1.25e-04 | 7838.6ms/step | 2.6 samples/s +Step 8060/11700 | Loss: 2.4478 | LR: 1.24e-04 | 7838.5ms/step | 2.6 samples/s +Step 8070/11700 | Loss: 2.5031 | LR: 1.24e-04 | 7812.9ms/step | 2.6 samples/s +Step 8080/11700 | Loss: 2.5007 | LR: 1.23e-04 | 7826.1ms/step | 2.6 samples/s +Step 8090/11700 | Loss: 2.4323 | LR: 1.23e-04 | 7851.9ms/step | 2.5 samples/s +Step 8100/11700 | Loss: 2.4683 | LR: 1.22e-04 | 7853.8ms/step | 2.5 samples/s +Step 8110/11700 | Loss: 2.4158 | LR: 1.21e-04 | 7836.3ms/step | 2.6 samples/s +Step 8120/11700 | Loss: 2.3962 | LR: 1.21e-04 | 7835.9ms/step | 2.6 samples/s +Step 8130/11700 | Loss: 2.3748 | LR: 1.20e-04 | 7809.5ms/step | 2.6 samples/s +Step 8140/11700 | Loss: 2.4292 | LR: 1.19e-04 | 7816.4ms/step | 2.6 samples/s +Step 8150/11700 | Loss: 2.3144 | LR: 1.19e-04 | 7830.2ms/step | 2.6 samples/s +Step 8160/11700 | Loss: 2.5424 | LR: 1.18e-04 | 7836.9ms/step | 2.6 samples/s +Step 8170/11700 | Loss: 2.4382 | LR: 1.18e-04 | 7837.4ms/step | 2.6 samples/s +Step 8180/11700 | Loss: 2.4804 | LR: 1.17e-04 | 7840.1ms/step | 2.6 samples/s +Step 8190/11700 | Loss: 2.4022 | LR: 1.16e-04 | 7803.9ms/step | 2.6 samples/s +Step 8200/11700 | Loss: 2.4545 | LR: 1.16e-04 | 7809.3ms/step | 2.6 samples/s +Step 8210/11700 | Loss: 2.4871 | LR: 1.15e-04 | 7827.0ms/step | 2.6 samples/s +Step 8220/11700 | Loss: 2.4882 | LR: 1.15e-04 | 7832.7ms/step | 2.6 samples/s +Step 8230/11700 | Loss: 2.4703 | LR: 1.14e-04 | 7835.8ms/step | 2.6 samples/s +Step 8240/11700 | Loss: 2.4400 | LR: 1.13e-04 | 7838.6ms/step | 2.6 samples/s +Step 8250/11700 | Loss: 2.4703 | LR: 1.13e-04 | 7816.9ms/step | 2.6 samples/s +Validation at step 8250: Loss = 2.4666, Perplexity = 11.78 +Step 8260/11700 | Loss: 2.4082 | LR: 1.12e-04 | 19447.1ms/step | 1.0 samples/s +Step 8270/11700 | Loss: 2.4175 | LR: 1.12e-04 | 7834.0ms/step | 2.6 samples/s +Step 8280/11700 | Loss: 2.5655 | LR: 1.11e-04 | 7830.9ms/step | 2.6 samples/s +Step 8290/11700 | Loss: 2.4276 | LR: 1.10e-04 | 7829.4ms/step | 2.6 samples/s +Step 8300/11700 | Loss: 2.4794 | LR: 1.10e-04 | 7799.0ms/step | 2.6 samples/s +Step 8310/11700 | Loss: 2.4122 | LR: 1.09e-04 | 7820.3ms/step | 2.6 samples/s +Step 8320/11700 | Loss: 2.4975 | LR: 1.09e-04 | 7836.8ms/step | 2.6 samples/s +Step 8330/11700 | Loss: 2.5047 | LR: 1.08e-04 | 7836.2ms/step | 2.6 samples/s +Step 8340/11700 | Loss: 2.4698 | LR: 1.07e-04 | 7830.4ms/step | 2.6 samples/s +Step 8350/11700 | Loss: 2.4818 | LR: 1.07e-04 | 7831.5ms/step | 2.6 samples/s +Step 8360/11700 | Loss: 2.4248 | LR: 1.06e-04 | 7800.0ms/step | 2.6 samples/s +Step 8370/11700 | Loss: 2.4140 | LR: 1.06e-04 | 7809.9ms/step | 2.6 samples/s +Step 8380/11700 | Loss: 2.4472 | LR: 1.05e-04 | 7823.3ms/step | 2.6 samples/s +Step 8390/11700 | Loss: 2.4532 | LR: 1.05e-04 | 7826.0ms/step | 2.6 samples/s +Step 8400/11700 | Loss: 2.5144 | LR: 1.04e-04 | 7824.4ms/step | 2.6 samples/s +Step 8410/11700 | Loss: 2.6364 | LR: 1.03e-04 | 7820.5ms/step | 2.6 samples/s +Step 8420/11700 | Loss: 2.3937 | LR: 1.03e-04 | 7806.8ms/step | 2.6 samples/s +Step 8430/11700 | Loss: 2.5142 | LR: 1.02e-04 | 7813.2ms/step | 2.6 samples/s +Step 8440/11700 | Loss: 2.5117 | LR: 1.02e-04 | 7827.9ms/step | 2.6 samples/s +Step 8450/11700 | Loss: 2.4557 | LR: 1.01e-04 | 7833.2ms/step | 2.6 samples/s +Step 8460/11700 | Loss: 2.4446 | LR: 1.00e-04 | 7834.4ms/step | 2.6 samples/s +Step 8470/11700 | Loss: 2.4416 | LR: 9.99e-05 | 7830.1ms/step | 2.6 samples/s +Step 8480/11700 | Loss: 2.4932 | LR: 9.93e-05 | 7819.8ms/step | 2.6 samples/s +Step 8490/11700 | Loss: 2.4453 | LR: 9.87e-05 | 7807.3ms/step | 2.6 samples/s +Step 8500/11700 | Loss: 2.3093 | LR: 9.82e-05 | 7828.3ms/step | 2.6 samples/s +Validation at step 8500: Loss = 2.4629, Perplexity = 11.74 +Step 8510/11700 | Loss: 2.4116 | LR: 9.76e-05 | 19460.0ms/step | 1.0 samples/s +Step 8520/11700 | Loss: 2.4585 | LR: 9.70e-05 | 7827.4ms/step | 2.6 samples/s +Step 8530/11700 | Loss: 2.4830 | LR: 9.65e-05 | 7792.6ms/step | 2.6 samples/s +Step 8540/11700 | Loss: 2.4533 | LR: 9.59e-05 | 7822.9ms/step | 2.6 samples/s +Step 8550/11700 | Loss: 2.3446 | LR: 9.53e-05 | 7850.0ms/step | 2.5 samples/s +Step 8560/11700 | Loss: 2.4550 | LR: 9.48e-05 | 7836.0ms/step | 2.6 samples/s +Step 8570/11700 | Loss: 2.4470 | LR: 9.42e-05 | 7836.3ms/step | 2.6 samples/s +Step 8580/11700 | Loss: 2.4328 | LR: 9.36e-05 | 7839.3ms/step | 2.6 samples/s +Step 8590/11700 | Loss: 2.4029 | LR: 9.31e-05 | 7814.1ms/step | 2.6 samples/s +Step 8600/11700 | Loss: 2.4745 | LR: 9.25e-05 | 7818.3ms/step | 2.6 samples/s +Step 8610/11700 | Loss: 2.4489 | LR: 9.20e-05 | 7849.9ms/step | 2.5 samples/s +Step 8620/11700 | Loss: 2.5033 | LR: 9.14e-05 | 7853.1ms/step | 2.5 samples/s +Step 8630/11700 | Loss: 2.4756 | LR: 9.09e-05 | 7848.5ms/step | 2.5 samples/s +Step 8640/11700 | Loss: 2.4132 | LR: 9.03e-05 | 7851.5ms/step | 2.5 samples/s +Step 8650/11700 | Loss: 2.3711 | LR: 8.98e-05 | 7831.9ms/step | 2.6 samples/s +Step 8660/11700 | Loss: 2.4346 | LR: 8.92e-05 | 7826.4ms/step | 2.6 samples/s +Step 8670/11700 | Loss: 2.5258 | LR: 8.87e-05 | 7844.9ms/step | 2.5 samples/s +Step 8680/11700 | Loss: 2.4037 | LR: 8.81e-05 | 7841.5ms/step | 2.6 samples/s +Step 8690/11700 | Loss: 2.5263 | LR: 8.76e-05 | 7842.6ms/step | 2.6 samples/s +Step 8700/11700 | Loss: 2.4520 | LR: 8.70e-05 | 7841.8ms/step | 2.6 samples/s +Step 8710/11700 | Loss: 2.3174 | LR: 8.65e-05 | 7821.9ms/step | 2.6 samples/s +Step 8720/11700 | Loss: 2.3938 | LR: 8.59e-05 | 7814.0ms/step | 2.6 samples/s +Step 8730/11700 | Loss: 2.4405 | LR: 8.54e-05 | 7835.5ms/step | 2.6 samples/s +Step 8740/11700 | Loss: 2.4696 | LR: 8.49e-05 | 7839.7ms/step | 2.6 samples/s +Step 8750/11700 | Loss: 2.4759 | LR: 8.43e-05 | 7842.3ms/step | 2.6 samples/s +Validation at step 8750: Loss = 2.4600, Perplexity = 11.70 +Step 8760/11700 | Loss: 2.3842 | LR: 8.38e-05 | 19494.0ms/step | 1.0 samples/s +Step 8770/11700 | Loss: 2.4418 | LR: 8.33e-05 | 7824.6ms/step | 2.6 samples/s +Step 8780/11700 | Loss: 2.4403 | LR: 8.27e-05 | 7833.5ms/step | 2.6 samples/s +Step 8790/11700 | Loss: 2.4803 | LR: 8.22e-05 | 7835.2ms/step | 2.6 samples/s +Step 8800/11700 | Loss: 2.5085 | LR: 8.17e-05 | 7822.0ms/step | 2.6 samples/s +Step 8810/11700 | Loss: 2.3625 | LR: 8.11e-05 | 7844.5ms/step | 2.5 samples/s +Step 8820/11700 | Loss: 2.3582 | LR: 8.06e-05 | 7823.1ms/step | 2.6 samples/s +Step 8830/11700 | Loss: 2.4177 | LR: 8.01e-05 | 7823.8ms/step | 2.6 samples/s +Step 8840/11700 | Loss: 2.4982 | LR: 7.95e-05 | 7839.1ms/step | 2.6 samples/s +Step 8850/11700 | Loss: 2.4652 | LR: 7.90e-05 | 7845.4ms/step | 2.5 samples/s +Step 8860/11700 | Loss: 2.3449 | LR: 7.85e-05 | 7843.1ms/step | 2.6 samples/s +Step 8870/11700 | Loss: 2.4001 | LR: 7.80e-05 | 7841.8ms/step | 2.6 samples/s +Step 8880/11700 | Loss: 2.4856 | LR: 7.75e-05 | 7836.2ms/step | 2.6 samples/s +Step 8890/11700 | Loss: 2.4474 | LR: 7.69e-05 | 7818.8ms/step | 2.6 samples/s +Step 8900/11700 | Loss: 2.4257 | LR: 7.64e-05 | 7839.8ms/step | 2.6 samples/s +Step 8910/11700 | Loss: 2.5230 | LR: 7.59e-05 | 7839.1ms/step | 2.6 samples/s +Step 8920/11700 | Loss: 2.5016 | LR: 7.54e-05 | 7838.7ms/step | 2.6 samples/s +Step 8930/11700 | Loss: 2.3857 | LR: 7.49e-05 | 7837.3ms/step | 2.6 samples/s +Step 8940/11700 | Loss: 2.4854 | LR: 7.44e-05 | 7832.9ms/step | 2.6 samples/s +Step 8950/11700 | Loss: 2.4159 | LR: 7.39e-05 | 7824.4ms/step | 2.6 samples/s +Step 8960/11700 | Loss: 2.4243 | LR: 7.34e-05 | 7840.8ms/step | 2.6 samples/s +Step 8970/11700 | Loss: 2.4332 | LR: 7.28e-05 | 7843.6ms/step | 2.5 samples/s +Step 8980/11700 | Loss: 2.4225 | LR: 7.23e-05 | 7844.5ms/step | 2.5 samples/s +Step 8990/11700 | Loss: 2.4607 | LR: 7.18e-05 | 7840.6ms/step | 2.6 samples/s +Step 9000/11700 | Loss: 2.4372 | LR: 7.13e-05 | 7831.1ms/step | 2.6 samples/s +Validation at step 9000: Loss = 2.4578, Perplexity = 11.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-9000 +Step 9010/11700 | Loss: 2.5252 | LR: 7.08e-05 | 19756.1ms/step | 1.0 samples/s +Step 9020/11700 | Loss: 2.4386 | LR: 7.03e-05 | 7840.1ms/step | 2.6 samples/s +Step 9030/11700 | Loss: 2.4032 | LR: 6.98e-05 | 7839.9ms/step | 2.6 samples/s +Step 9040/11700 | Loss: 2.3762 | LR: 6.93e-05 | 7830.1ms/step | 2.6 samples/s +Step 9050/11700 | Loss: 2.4711 | LR: 6.88e-05 | 7816.1ms/step | 2.6 samples/s +Step 9060/11700 | Loss: 2.4854 | LR: 6.83e-05 | 7808.8ms/step | 2.6 samples/s +Step 9070/11700 | Loss: 2.4995 | LR: 6.79e-05 | 7819.8ms/step | 2.6 samples/s +Step 9080/11700 | Loss: 2.3963 | LR: 6.74e-05 | 7837.3ms/step | 2.6 samples/s +Step 9090/11700 | Loss: 2.3819 | LR: 6.69e-05 | 7813.3ms/step | 2.6 samples/s +Step 9100/11700 | Loss: 2.4695 | LR: 6.64e-05 | 7829.1ms/step | 2.6 samples/s +Step 9110/11700 | Loss: 2.4687 | LR: 6.59e-05 | 7804.9ms/step | 2.6 samples/s +Step 9120/11700 | Loss: 2.4992 | LR: 6.54e-05 | 7796.5ms/step | 2.6 samples/s +Step 9130/11700 | Loss: 2.3654 | LR: 6.49e-05 | 7824.6ms/step | 2.6 samples/s +Step 9140/11700 | Loss: 2.3425 | LR: 6.45e-05 | 7824.9ms/step | 2.6 samples/s +Step 9150/11700 | Loss: 2.4551 | LR: 6.40e-05 | 7826.9ms/step | 2.6 samples/s +Step 9160/11700 | Loss: 2.5439 | LR: 6.35e-05 | 7828.3ms/step | 2.6 samples/s +Step 9170/11700 | Loss: 2.4943 | LR: 6.30e-05 | 7814.7ms/step | 2.6 samples/s +Step 9180/11700 | Loss: 2.4300 | LR: 6.25e-05 | 7801.3ms/step | 2.6 samples/s +Step 9190/11700 | Loss: 2.4526 | LR: 6.21e-05 | 7820.7ms/step | 2.6 samples/s +Step 9200/11700 | Loss: 2.3352 | LR: 6.16e-05 | 7829.4ms/step | 2.6 samples/s +Step 9210/11700 | Loss: 2.4410 | LR: 6.11e-05 | 7833.1ms/step | 2.6 samples/s +Step 9220/11700 | Loss: 2.5236 | LR: 6.07e-05 | 7829.3ms/step | 2.6 samples/s +Step 9230/11700 | Loss: 2.4305 | LR: 6.02e-05 | 7817.5ms/step | 2.6 samples/s +Step 9240/11700 | Loss: 2.4172 | LR: 5.97e-05 | 7804.1ms/step | 2.6 samples/s +Step 9250/11700 | Loss: 2.4483 | LR: 5.93e-05 | 7811.2ms/step | 2.6 samples/s +Validation at step 9250: Loss = 2.4562, Perplexity = 11.66 +Step 9260/11700 | Loss: 2.3719 | LR: 5.88e-05 | 19454.7ms/step | 1.0 samples/s +Step 9270/11700 | Loss: 2.4650 | LR: 5.83e-05 | 7817.8ms/step | 2.6 samples/s +Step 9280/11700 | Loss: 2.3196 | LR: 5.79e-05 | 7806.8ms/step | 2.6 samples/s +Step 9290/11700 | Loss: 2.4079 | LR: 5.74e-05 | 7803.8ms/step | 2.6 samples/s +Step 9300/11700 | Loss: 2.4630 | LR: 5.70e-05 | 7820.1ms/step | 2.6 samples/s +Step 9310/11700 | Loss: 2.3573 | LR: 5.65e-05 | 7822.7ms/step | 2.6 samples/s +Step 9320/11700 | Loss: 2.3999 | LR: 5.61e-05 | 7831.6ms/step | 2.6 samples/s +Step 9330/11700 | Loss: 2.3964 | LR: 5.56e-05 | 7826.3ms/step | 2.6 samples/s +Step 9340/11700 | Loss: 2.5077 | LR: 5.52e-05 | 7813.2ms/step | 2.6 samples/s +Step 9350/11700 | Loss: 2.3960 | LR: 5.47e-05 | 7786.7ms/step | 2.6 samples/s +Step 9360/11700 | Loss: 2.4275 | LR: 5.43e-05 | 7813.5ms/step | 2.6 samples/s +Step 9370/11700 | Loss: 2.4449 | LR: 5.38e-05 | 7830.9ms/step | 2.6 samples/s +Step 9380/11700 | Loss: 2.5467 | LR: 5.34e-05 | 7829.0ms/step | 2.6 samples/s +Step 9390/11700 | Loss: 2.4543 | LR: 5.29e-05 | 7823.4ms/step | 2.6 samples/s +Step 9400/11700 | Loss: 2.4320 | LR: 5.25e-05 | 7832.8ms/step | 2.6 samples/s +Step 9410/11700 | Loss: 2.4471 | LR: 5.20e-05 | 7809.4ms/step | 2.6 samples/s +Step 9420/11700 | Loss: 2.5099 | LR: 5.16e-05 | 7786.3ms/step | 2.6 samples/s +Step 9430/11700 | Loss: 2.3209 | LR: 5.12e-05 | 7802.1ms/step | 2.6 samples/s +Step 9440/11700 | Loss: 2.5371 | LR: 5.07e-05 | 7826.2ms/step | 2.6 samples/s +Step 9450/11700 | Loss: 2.4793 | LR: 5.03e-05 | 7827.8ms/step | 2.6 samples/s +Step 9460/11700 | Loss: 2.4794 | LR: 4.99e-05 | 7821.2ms/step | 2.6 samples/s +Step 9470/11700 | Loss: 2.4543 | LR: 4.94e-05 | 7824.1ms/step | 2.6 samples/s +Step 9480/11700 | Loss: 2.4128 | LR: 4.90e-05 | 7801.3ms/step | 2.6 samples/s +Step 9490/11700 | Loss: 2.5642 | LR: 4.86e-05 | 7780.8ms/step | 2.6 samples/s +Step 9500/11700 | Loss: 2.3701 | LR: 4.82e-05 | 7802.5ms/step | 2.6 samples/s +Validation at step 9500: Loss = 2.4549, Perplexity = 11.65 +Step 9510/11700 | Loss: 2.4572 | LR: 4.77e-05 | 19479.3ms/step | 1.0 samples/s +Step 9520/11700 | Loss: 2.4111 | LR: 4.73e-05 | 7830.7ms/step | 2.6 samples/s +Step 9530/11700 | Loss: 2.4135 | LR: 4.69e-05 | 7827.3ms/step | 2.6 samples/s +Step 9540/11700 | Loss: 2.3756 | LR: 4.65e-05 | 7804.0ms/step | 2.6 samples/s +Step 9550/11700 | Loss: 2.5816 | LR: 4.61e-05 | 7786.5ms/step | 2.6 samples/s +Step 9560/11700 | Loss: 2.3608 | LR: 4.57e-05 | 7816.6ms/step | 2.6 samples/s +Step 9570/11700 | Loss: 2.3989 | LR: 4.52e-05 | 7824.6ms/step | 2.6 samples/s +Step 9580/11700 | Loss: 2.5185 | LR: 4.48e-05 | 7827.2ms/step | 2.6 samples/s +Step 9590/11700 | Loss: 2.4499 | LR: 4.44e-05 | 7840.9ms/step | 2.6 samples/s +Step 9600/11700 | Loss: 2.4595 | LR: 4.40e-05 | 7815.7ms/step | 2.6 samples/s +Step 9610/11700 | Loss: 2.4592 | LR: 4.36e-05 | 7799.0ms/step | 2.6 samples/s +Step 9620/11700 | Loss: 2.4717 | LR: 4.32e-05 | 7785.9ms/step | 2.6 samples/s +Step 9630/11700 | Loss: 2.4191 | LR: 4.28e-05 | 7817.8ms/step | 2.6 samples/s +Step 9640/11700 | Loss: 2.4690 | LR: 4.24e-05 | 7828.6ms/step | 2.6 samples/s +Step 9650/11700 | Loss: 2.5474 | LR: 4.20e-05 | 7832.4ms/step | 2.6 samples/s +Step 9660/11700 | Loss: 2.5480 | LR: 4.16e-05 | 7827.3ms/step | 2.6 samples/s +Step 9670/11700 | Loss: 2.4500 | LR: 4.12e-05 | 7825.9ms/step | 2.6 samples/s +Step 9680/11700 | Loss: 2.2714 | LR: 4.08e-05 | 7804.5ms/step | 2.6 samples/s +Step 9690/11700 | Loss: 2.5100 | LR: 4.04e-05 | 7795.9ms/step | 2.6 samples/s +Step 9700/11700 | Loss: 2.3795 | LR: 4.00e-05 | 7810.2ms/step | 2.6 samples/s +Step 9710/11700 | Loss: 2.4311 | LR: 3.97e-05 | 7811.8ms/step | 2.6 samples/s +Step 9720/11700 | Loss: 2.5051 | LR: 3.93e-05 | 7818.1ms/step | 2.6 samples/s +Step 9730/11700 | Loss: 2.3479 | LR: 3.89e-05 | 7811.2ms/step | 2.6 samples/s +Step 9740/11700 | Loss: 2.5195 | LR: 3.85e-05 | 7812.2ms/step | 2.6 samples/s +Step 9750/11700 | Loss: 2.4428 | LR: 3.81e-05 | 7797.0ms/step | 2.6 samples/s +Validation at step 9750: Loss = 2.4540, Perplexity = 11.64 +Step 9760/11700 | Loss: 2.4279 | LR: 3.77e-05 | 19405.4ms/step | 1.0 samples/s +Step 9770/11700 | Loss: 2.3718 | LR: 3.74e-05 | 7812.0ms/step | 2.6 samples/s +Step 9780/11700 | Loss: 2.5666 | LR: 3.70e-05 | 7809.7ms/step | 2.6 samples/s +Step 9790/11700 | Loss: 2.4250 | LR: 3.66e-05 | 7810.4ms/step | 2.6 samples/s +Step 9800/11700 | Loss: 2.3728 | LR: 3.62e-05 | 7807.4ms/step | 2.6 samples/s +Step 9810/11700 | Loss: 2.4898 | LR: 3.59e-05 | 7780.9ms/step | 2.6 samples/s +Step 9820/11700 | Loss: 2.3392 | LR: 3.55e-05 | 7803.6ms/step | 2.6 samples/s +Step 9830/11700 | Loss: 2.3605 | LR: 3.51e-05 | 7813.8ms/step | 2.6 samples/s +Step 9840/11700 | Loss: 2.3936 | LR: 3.48e-05 | 7820.1ms/step | 2.6 samples/s +Step 9850/11700 | Loss: 2.4693 | LR: 3.44e-05 | 7817.4ms/step | 2.6 samples/s +Step 9860/11700 | Loss: 2.4479 | LR: 3.40e-05 | 7828.5ms/step | 2.6 samples/s +Step 9870/11700 | Loss: 2.3956 | LR: 3.37e-05 | 7808.9ms/step | 2.6 samples/s +Step 9880/11700 | Loss: 2.4360 | LR: 3.33e-05 | 7799.8ms/step | 2.6 samples/s +Step 9890/11700 | Loss: 2.3433 | LR: 3.30e-05 | 7812.5ms/step | 2.6 samples/s +Step 9900/11700 | Loss: 2.4769 | LR: 3.26e-05 | 7809.2ms/step | 2.6 samples/s +Step 9910/11700 | Loss: 2.4580 | LR: 3.22e-05 | 7804.5ms/step | 2.6 samples/s +Step 9920/11700 | Loss: 2.4231 | LR: 3.19e-05 | 7804.8ms/step | 2.6 samples/s +Step 9930/11700 | Loss: 2.5058 | LR: 3.15e-05 | 7804.8ms/step | 2.6 samples/s +Step 9940/11700 | Loss: 2.3176 | LR: 3.12e-05 | 7792.5ms/step | 2.6 samples/s +Step 9950/11700 | Loss: 2.4275 | LR: 3.09e-05 | 7793.8ms/step | 2.6 samples/s +Step 9960/11700 | Loss: 2.3738 | LR: 3.05e-05 | 7813.8ms/step | 2.6 samples/s +Step 9970/11700 | Loss: 2.4504 | LR: 3.02e-05 | 7815.7ms/step | 2.6 samples/s +Step 9980/11700 | Loss: 2.3802 | LR: 2.98e-05 | 7815.7ms/step | 2.6 samples/s +Step 9990/11700 | Loss: 2.3340 | LR: 2.95e-05 | 7800.6ms/step | 2.6 samples/s +Step 10000/11700 | Loss: 2.4497 | LR: 2.92e-05 | 7803.8ms/step | 2.6 samples/s +Validation at step 10000: Loss = 2.4535, Perplexity = 11.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-10000 +Step 10010/11700 | Loss: 2.5456 | LR: 2.88e-05 | 19682.6ms/step | 1.0 samples/s +Step 10020/11700 | Loss: 2.5323 | LR: 2.85e-05 | 7814.9ms/step | 2.6 samples/s +Step 10030/11700 | Loss: 2.4306 | LR: 2.82e-05 | 7815.8ms/step | 2.6 samples/s +Step 10040/11700 | Loss: 2.4823 | LR: 2.78e-05 | 7818.1ms/step | 2.6 samples/s +Step 10050/11700 | Loss: 2.5193 | LR: 2.75e-05 | 7814.2ms/step | 2.6 samples/s +Step 10060/11700 | Loss: 2.3934 | LR: 2.72e-05 | 7793.2ms/step | 2.6 samples/s +Step 10070/11700 | Loss: 2.5112 | LR: 2.68e-05 | 7811.9ms/step | 2.6 samples/s +Step 10080/11700 | Loss: 2.3874 | LR: 2.65e-05 | 7816.0ms/step | 2.6 samples/s +Step 10090/11700 | Loss: 2.4573 | LR: 2.62e-05 | 7812.8ms/step | 2.6 samples/s +Step 10100/11700 | Loss: 2.4197 | LR: 2.59e-05 | 7817.8ms/step | 2.6 samples/s +Step 10110/11700 | Loss: 2.4912 | LR: 2.56e-05 | 7817.3ms/step | 2.6 samples/s +Step 10120/11700 | Loss: 2.3584 | LR: 2.52e-05 | 7802.4ms/step | 2.6 samples/s +Step 10130/11700 | Loss: 2.5266 | LR: 2.49e-05 | 7791.2ms/step | 2.6 samples/s +Step 10140/11700 | Loss: 2.5596 | LR: 2.46e-05 | 7806.7ms/step | 2.6 samples/s +Step 10150/11700 | Loss: 2.4701 | LR: 2.43e-05 | 7824.1ms/step | 2.6 samples/s +Step 10160/11700 | Loss: 2.4547 | LR: 2.40e-05 | 7825.3ms/step | 2.6 samples/s +Step 10170/11700 | Loss: 2.4399 | LR: 2.37e-05 | 7824.2ms/step | 2.6 samples/s +Step 10180/11700 | Loss: 2.4609 | LR: 2.34e-05 | 7826.2ms/step | 2.6 samples/s +Step 10190/11700 | Loss: 2.3682 | LR: 2.31e-05 | 7800.8ms/step | 2.6 samples/s +Step 10200/11700 | Loss: 2.3947 | LR: 2.28e-05 | 7784.5ms/step | 2.6 samples/s +Step 10210/11700 | Loss: 2.4897 | LR: 2.25e-05 | 7806.1ms/step | 2.6 samples/s +Step 10220/11700 | Loss: 2.4468 | LR: 2.22e-05 | 7831.9ms/step | 2.6 samples/s +Step 10230/11700 | Loss: 2.5497 | LR: 2.19e-05 | 7831.4ms/step | 2.6 samples/s +Step 10240/11700 | Loss: 2.5382 | LR: 2.16e-05 | 7821.0ms/step | 2.6 samples/s +Step 10250/11700 | Loss: 2.4871 | LR: 2.13e-05 | 7822.7ms/step | 2.6 samples/s +Validation at step 10250: Loss = 2.4531, Perplexity = 11.62 +Step 10260/11700 | Loss: 2.4944 | LR: 2.10e-05 | 19408.1ms/step | 1.0 samples/s +Step 10270/11700 | Loss: 2.3971 | LR: 2.07e-05 | 7834.7ms/step | 2.6 samples/s +Step 10280/11700 | Loss: 2.4948 | LR: 2.05e-05 | 7846.4ms/step | 2.5 samples/s +Step 10290/11700 | Loss: 2.4799 | LR: 2.02e-05 | 7829.6ms/step | 2.6 samples/s +Step 10300/11700 | Loss: 2.3335 | LR: 1.99e-05 | 7798.0ms/step | 2.6 samples/s +Step 10310/11700 | Loss: 2.4395 | LR: 1.96e-05 | 7797.0ms/step | 2.6 samples/s +Step 10320/11700 | Loss: 2.3861 | LR: 1.93e-05 | 7805.2ms/step | 2.6 samples/s +Step 10330/11700 | Loss: 2.3779 | LR: 1.91e-05 | 7809.0ms/step | 2.6 samples/s +Step 10340/11700 | Loss: 2.5466 | LR: 1.88e-05 | 7822.0ms/step | 2.6 samples/s +Step 10350/11700 | Loss: 2.4763 | LR: 1.85e-05 | 7814.6ms/step | 2.6 samples/s +Step 10360/11700 | Loss: 2.4718 | LR: 1.82e-05 | 7816.5ms/step | 2.6 samples/s +Step 10370/11700 | Loss: 2.4531 | LR: 1.80e-05 | 7802.3ms/step | 2.6 samples/s +Step 10380/11700 | Loss: 2.2895 | LR: 1.77e-05 | 7800.3ms/step | 2.6 samples/s +Step 10390/11700 | Loss: 2.5854 | LR: 1.75e-05 | 7816.6ms/step | 2.6 samples/s +Step 10400/11700 | Loss: 2.4952 | LR: 1.72e-05 | 7817.7ms/step | 2.6 samples/s +Step 10410/11700 | Loss: 2.4445 | LR: 1.69e-05 | 7817.3ms/step | 2.6 samples/s +Step 10420/11700 | Loss: 2.4225 | LR: 1.67e-05 | 7814.2ms/step | 2.6 samples/s +Step 10430/11700 | Loss: 2.4169 | LR: 1.64e-05 | 7804.5ms/step | 2.6 samples/s +Step 10440/11700 | Loss: 2.3892 | LR: 1.62e-05 | 7797.6ms/step | 2.6 samples/s +Step 10450/11700 | Loss: 2.4652 | LR: 1.59e-05 | 7810.2ms/step | 2.6 samples/s +Step 10460/11700 | Loss: 2.4902 | LR: 1.57e-05 | 7811.6ms/step | 2.6 samples/s +Step 10470/11700 | Loss: 2.3788 | LR: 1.54e-05 | 7824.4ms/step | 2.6 samples/s +Step 10480/11700 | Loss: 2.4171 | LR: 1.52e-05 | 7820.0ms/step | 2.6 samples/s +Step 10490/11700 | Loss: 2.3586 | LR: 1.49e-05 | 7802.7ms/step | 2.6 samples/s +Step 10500/11700 | Loss: 2.4476 | LR: 1.47e-05 | 7800.1ms/step | 2.6 samples/s +Validation at step 10500: Loss = 2.4530, Perplexity = 11.62 +Step 10510/11700 | Loss: 2.5713 | LR: 1.44e-05 | 19439.0ms/step | 1.0 samples/s +Step 10520/11700 | Loss: 2.4727 | LR: 1.42e-05 | 7810.5ms/step | 2.6 samples/s +Step 10530/11700 | Loss: 2.4588 | LR: 1.40e-05 | 7814.1ms/step | 2.6 samples/s +Step 10540/11700 | Loss: 2.3084 | LR: 1.37e-05 | 7798.2ms/step | 2.6 samples/s +Step 10550/11700 | Loss: 2.4850 | LR: 1.35e-05 | 7797.2ms/step | 2.6 samples/s +Step 10560/11700 | Loss: 2.5836 | LR: 1.33e-05 | 7816.0ms/step | 2.6 samples/s +Step 10570/11700 | Loss: 2.3408 | LR: 1.30e-05 | 7815.3ms/step | 2.6 samples/s +Step 10580/11700 | Loss: 2.5168 | LR: 1.28e-05 | 7812.5ms/step | 2.6 samples/s +Step 10590/11700 | Loss: 2.3799 | LR: 1.26e-05 | 7810.5ms/step | 2.6 samples/s +Step 10600/11700 | Loss: 2.4678 | LR: 1.23e-05 | 7799.4ms/step | 2.6 samples/s +Step 10610/11700 | Loss: 2.4375 | LR: 1.21e-05 | 7801.0ms/step | 2.6 samples/s +Step 10620/11700 | Loss: 2.3548 | LR: 1.19e-05 | 7813.3ms/step | 2.6 samples/s +Step 10630/11700 | Loss: 2.4255 | LR: 1.17e-05 | 7821.9ms/step | 2.6 samples/s +Step 10640/11700 | Loss: 2.4092 | LR: 1.15e-05 | 7842.8ms/step | 2.6 samples/s +Step 10650/11700 | Loss: 2.4631 | LR: 1.13e-05 | 7807.2ms/step | 2.6 samples/s +Step 10660/11700 | Loss: 2.4145 | LR: 1.10e-05 | 7801.7ms/step | 2.6 samples/s +Step 10670/11700 | Loss: 2.4016 | LR: 1.08e-05 | 7810.0ms/step | 2.6 samples/s +Step 10680/11700 | Loss: 2.4779 | LR: 1.06e-05 | 7815.9ms/step | 2.6 samples/s +Step 10690/11700 | Loss: 2.4662 | LR: 1.04e-05 | 7831.6ms/step | 2.6 samples/s +Step 10700/11700 | Loss: 2.4350 | LR: 1.02e-05 | 7806.8ms/step | 2.6 samples/s +Step 10710/11700 | Loss: 2.4678 | LR: 1.00e-05 | 7809.8ms/step | 2.6 samples/s +Step 10720/11700 | Loss: 2.4028 | LR: 9.82e-06 | 7798.6ms/step | 2.6 samples/s +Step 10730/11700 | Loss: 2.5046 | LR: 9.62e-06 | 7796.7ms/step | 2.6 samples/s +Step 10740/11700 | Loss: 2.3321 | LR: 9.42e-06 | 7813.9ms/step | 2.6 samples/s +Step 10750/11700 | Loss: 2.3613 | LR: 9.23e-06 | 7826.0ms/step | 2.6 samples/s +Validation at step 10750: Loss = 2.4529, Perplexity = 11.62 +Step 10760/11700 | Loss: 2.5294 | LR: 9.04e-06 | 19452.0ms/step | 1.0 samples/s +Step 10770/11700 | Loss: 2.5040 | LR: 8.85e-06 | 7793.2ms/step | 2.6 samples/s +Step 10780/11700 | Loss: 2.5327 | LR: 8.66e-06 | 7805.7ms/step | 2.6 samples/s +Step 10790/11700 | Loss: 2.4431 | LR: 8.47e-06 | 7827.0ms/step | 2.6 samples/s +Step 10800/11700 | Loss: 2.5364 | LR: 8.29e-06 | 7828.3ms/step | 2.6 samples/s +Step 10810/11700 | Loss: 2.4342 | LR: 8.11e-06 | 7834.6ms/step | 2.6 samples/s +Step 10820/11700 | Loss: 2.5222 | LR: 7.93e-06 | 7830.3ms/step | 2.6 samples/s +Step 10830/11700 | Loss: 2.4207 | LR: 7.75e-06 | 7808.6ms/step | 2.6 samples/s +Step 10840/11700 | Loss: 2.4038 | LR: 7.57e-06 | 7807.6ms/step | 2.6 samples/s +Step 10850/11700 | Loss: 2.3546 | LR: 7.40e-06 | 7812.2ms/step | 2.6 samples/s +Step 10860/11700 | Loss: 2.3549 | LR: 7.22e-06 | 7815.5ms/step | 2.6 samples/s +Step 10870/11700 | Loss: 2.3830 | LR: 7.05e-06 | 7812.9ms/step | 2.6 samples/s +Step 10880/11700 | Loss: 2.3626 | LR: 6.89e-06 | 7814.8ms/step | 2.6 samples/s +Step 10890/11700 | Loss: 2.3491 | LR: 6.72e-06 | 7800.4ms/step | 2.6 samples/s +Step 10900/11700 | Loss: 2.4538 | LR: 6.56e-06 | 7783.8ms/step | 2.6 samples/s +Step 10910/11700 | Loss: 2.4947 | LR: 6.39e-06 | 7801.9ms/step | 2.6 samples/s +Step 10920/11700 | Loss: 2.4177 | LR: 6.23e-06 | 7814.5ms/step | 2.6 samples/s +Step 10930/11700 | Loss: 2.3395 | LR: 6.08e-06 | 7813.6ms/step | 2.6 samples/s +Step 10940/11700 | Loss: 2.3622 | LR: 5.92e-06 | 7817.3ms/step | 2.6 samples/s +Step 10950/11700 | Loss: 2.3689 | LR: 5.77e-06 | 7828.4ms/step | 2.6 samples/s +Step 10960/11700 | Loss: 2.5181 | LR: 5.61e-06 | 7799.8ms/step | 2.6 samples/s +Step 10970/11700 | Loss: 2.4746 | LR: 5.46e-06 | 7788.4ms/step | 2.6 samples/s +Step 10980/11700 | Loss: 2.4712 | LR: 5.31e-06 | 7807.2ms/step | 2.6 samples/s +Step 10990/11700 | Loss: 2.4102 | LR: 5.17e-06 | 7812.9ms/step | 2.6 samples/s +Step 11000/11700 | Loss: 2.3902 | LR: 5.02e-06 | 7819.4ms/step | 2.6 samples/s +Validation at step 11000: Loss = 2.4528, Perplexity = 11.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11000 +Step 11010/11700 | Loss: 2.3791 | LR: 4.88e-06 | 19706.2ms/step | 1.0 samples/s +Step 11020/11700 | Loss: 2.4472 | LR: 4.74e-06 | 7786.0ms/step | 2.6 samples/s +Step 11030/11700 | Loss: 2.4814 | LR: 4.60e-06 | 7810.3ms/step | 2.6 samples/s +Step 11040/11700 | Loss: 2.4730 | LR: 4.47e-06 | 7827.5ms/step | 2.6 samples/s +Step 11050/11700 | Loss: 2.4961 | LR: 4.33e-06 | 7830.2ms/step | 2.6 samples/s +Step 11060/11700 | Loss: 2.4295 | LR: 4.20e-06 | 7840.8ms/step | 2.6 samples/s +Step 11070/11700 | Loss: 2.3912 | LR: 4.07e-06 | 7842.4ms/step | 2.6 samples/s +Step 11080/11700 | Loss: 2.4507 | LR: 3.94e-06 | 7811.8ms/step | 2.6 samples/s +Step 11090/11700 | Loss: 2.4363 | LR: 3.82e-06 | 7802.4ms/step | 2.6 samples/s +Step 11100/11700 | Loss: 2.4816 | LR: 3.69e-06 | 7829.0ms/step | 2.6 samples/s +Step 11110/11700 | Loss: 2.4271 | LR: 3.57e-06 | 7839.9ms/step | 2.6 samples/s +Step 11120/11700 | Loss: 2.4717 | LR: 3.45e-06 | 7841.2ms/step | 2.6 samples/s +Step 11130/11700 | Loss: 2.5182 | LR: 3.34e-06 | 7841.1ms/step | 2.6 samples/s +Step 11140/11700 | Loss: 2.4229 | LR: 3.22e-06 | 7836.4ms/step | 2.6 samples/s +Step 11150/11700 | Loss: 2.4447 | LR: 3.11e-06 | 7810.2ms/step | 2.6 samples/s +Step 11160/11700 | Loss: 2.3918 | LR: 2.99e-06 | 7835.1ms/step | 2.6 samples/s +Step 11170/11700 | Loss: 2.5040 | LR: 2.88e-06 | 7843.6ms/step | 2.5 samples/s +Step 11180/11700 | Loss: 2.4160 | LR: 2.78e-06 | 7841.7ms/step | 2.6 samples/s +Step 11190/11700 | Loss: 2.4543 | LR: 2.67e-06 | 7840.5ms/step | 2.6 samples/s +Step 11200/11700 | Loss: 2.4486 | LR: 2.57e-06 | 7830.9ms/step | 2.6 samples/s +Step 11210/11700 | Loss: 2.4977 | LR: 2.47e-06 | 7804.6ms/step | 2.6 samples/s +Step 11220/11700 | Loss: 2.3903 | LR: 2.37e-06 | 7833.4ms/step | 2.6 samples/s +Step 11230/11700 | Loss: 2.3175 | LR: 2.27e-06 | 7835.9ms/step | 2.6 samples/s +Step 11240/11700 | Loss: 2.3858 | LR: 2.17e-06 | 7830.1ms/step | 2.6 samples/s +Step 11250/11700 | Loss: 2.4011 | LR: 2.08e-06 | 7842.4ms/step | 2.6 samples/s +Validation at step 11250: Loss = 2.4528, Perplexity = 11.62 +Step 11260/11700 | Loss: 2.4335 | LR: 1.99e-06 | 19462.0ms/step | 1.0 samples/s +Step 11270/11700 | Loss: 2.4790 | LR: 1.90e-06 | 7836.4ms/step | 2.6 samples/s +Step 11280/11700 | Loss: 2.4196 | LR: 1.81e-06 | 7835.9ms/step | 2.6 samples/s +Step 11290/11700 | Loss: 2.4880 | LR: 1.73e-06 | 7839.4ms/step | 2.6 samples/s +Step 11300/11700 | Loss: 2.4554 | LR: 1.64e-06 | 7844.1ms/step | 2.5 samples/s +Step 11310/11700 | Loss: 2.4237 | LR: 1.56e-06 | 7815.3ms/step | 2.6 samples/s +Step 11320/11700 | Loss: 2.4546 | LR: 1.48e-06 | 7818.5ms/step | 2.6 samples/s +Step 11330/11700 | Loss: 2.3807 | LR: 1.41e-06 | 7836.7ms/step | 2.6 samples/s +Step 11340/11700 | Loss: 2.3584 | LR: 1.33e-06 | 7842.3ms/step | 2.6 samples/s +Step 11350/11700 | Loss: 2.4818 | LR: 1.26e-06 | 7842.9ms/step | 2.6 samples/s +Step 11360/11700 | Loss: 2.4741 | LR: 1.19e-06 | 7843.0ms/step | 2.6 samples/s +Step 11370/11700 | Loss: 2.4346 | LR: 1.12e-06 | 7822.3ms/step | 2.6 samples/s +Step 11380/11700 | Loss: 2.4208 | LR: 1.05e-06 | 7819.6ms/step | 2.6 samples/s +Step 11390/11700 | Loss: 2.4392 | LR: 9.88e-07 | 7839.1ms/step | 2.6 samples/s +Step 11400/11700 | Loss: 2.4661 | LR: 9.25e-07 | 7839.2ms/step | 2.6 samples/s +Step 11410/11700 | Loss: 2.4414 | LR: 8.65e-07 | 7839.8ms/step | 2.6 samples/s +Step 11420/11700 | Loss: 2.2899 | LR: 8.06e-07 | 7841.5ms/step | 2.6 samples/s +Step 11430/11700 | Loss: 2.3927 | LR: 7.50e-07 | 7829.4ms/step | 2.6 samples/s +Step 11440/11700 | Loss: 2.4829 | LR: 6.95e-07 | 7818.5ms/step | 2.6 samples/s +Step 11450/11700 | Loss: 2.3293 | LR: 6.43e-07 | 7843.5ms/step | 2.5 samples/s +Step 11460/11700 | Loss: 2.4548 | LR: 5.92e-07 | 7845.1ms/step | 2.5 samples/s +Step 11470/11700 | Loss: 2.4996 | LR: 5.44e-07 | 7841.1ms/step | 2.6 samples/s +Step 11480/11700 | Loss: 2.3698 | LR: 4.98e-07 | 7842.8ms/step | 2.6 samples/s +Step 11490/11700 | Loss: 2.4704 | LR: 4.54e-07 | 7831.0ms/step | 2.6 samples/s +Step 11500/11700 | Loss: 2.4046 | LR: 4.11e-07 | 7823.1ms/step | 2.6 samples/s +Validation at step 11500: Loss = 2.4528, Perplexity = 11.62 +Step 11510/11700 | Loss: 2.3907 | LR: 3.71e-07 | 19505.9ms/step | 1.0 samples/s +Step 11520/11700 | Loss: 2.4647 | LR: 3.33e-07 | 7838.0ms/step | 2.6 samples/s +Step 11530/11700 | Loss: 2.4093 | LR: 2.97e-07 | 7839.8ms/step | 2.6 samples/s +Step 11540/11700 | Loss: 2.3859 | LR: 2.63e-07 | 7811.4ms/step | 2.6 samples/s +Step 11550/11700 | Loss: 2.5074 | LR: 2.31e-07 | 7832.2ms/step | 2.6 samples/s +Step 11560/11700 | Loss: 2.4051 | LR: 2.02e-07 | 7838.6ms/step | 2.6 samples/s +Step 11570/11700 | Loss: 2.5180 | LR: 1.74e-07 | 7840.0ms/step | 2.6 samples/s +Step 11580/11700 | Loss: 2.3546 | LR: 1.48e-07 | 7837.6ms/step | 2.6 samples/s +Step 11590/11700 | Loss: 2.4715 | LR: 1.24e-07 | 7837.4ms/step | 2.6 samples/s +Step 11600/11700 | Loss: 2.4732 | LR: 1.03e-07 | 7807.9ms/step | 2.6 samples/s +Step 11610/11700 | Loss: 2.4744 | LR: 8.33e-08 | 7849.0ms/step | 2.5 samples/s +Step 11620/11700 | Loss: 2.3910 | LR: 6.58e-08 | 7851.8ms/step | 2.5 samples/s +Step 11630/11700 | Loss: 2.3381 | LR: 5.04e-08 | 7850.5ms/step | 2.5 samples/s +Step 11640/11700 | Loss: 2.4726 | LR: 3.70e-08 | 7838.2ms/step | 2.6 samples/s +Step 11650/11700 | Loss: 2.5243 | LR: 2.57e-08 | 7845.3ms/step | 2.5 samples/s +Step 11660/11700 | Loss: 2.2990 | LR: 1.65e-08 | 7830.4ms/step | 2.6 samples/s +Step 11670/11700 | Loss: 2.4536 | LR: 9.26e-09 | 7829.5ms/step | 2.6 samples/s +Step 11680/11700 | Loss: 2.4660 | LR: 4.12e-09 | 7845.8ms/step | 2.5 samples/s +Step 11690/11700 | Loss: 2.3698 | LR: 1.03e-09 | 7851.4ms/step | 2.5 samples/s +Step 11700/11700 | Loss: 2.4355 | LR: 0.00e+00 | 7843.0ms/step | 2.6 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11700 +wandb: updating run metadata +wandb: uploading config.yaml; uploading output.log; uploading wandb-summary.json +wandb: uploading config.yaml; uploading output.log +wandb: uploading history steps 1169-1169, summary, console lines 1229-1229 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▆▄▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▄▄▄▄▃▃▂▂▂▂▃▃▃▂▃▂▃▄▅▅▁▁▅▄▄▂▅▆██▇█▄▇█▆▃▅▃▇ +wandb: speed/samples_per_sec ███████▁████▁██████████▁████████████▁███ +wandb: speed/steps_per_sec █▁███████████████████████▁█████▁████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.45282 +wandb: eval/perplexity 11.62103 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 7843.02893 +wandb: speed/samples_per_sec 2.55004 +wandb: speed/steps_per_sec 0.1275 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m fixed s2 30k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/m10lf13v +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260707_035818-m10lf13v/logs +Training complete! diff --git a/epfl_final/logs/l30k365m_resampled.log b/epfl_final/logs/l30k365m_resampled.log new file mode 100644 index 0000000000000000000000000000000000000000..8b7e89830b350b134502f4434731b5c04567f0bc --- /dev/null +++ b/epfl_final/logs/l30k365m_resampled.log @@ -0,0 +1,1439 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized", + "val_path": null, + "max_seq_length": 30720, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 11700, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 20, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m resampled 30k nrb120", + "wandb_tags": [ + "qwen3", + "365m", + "30k", + "scaleladder", + "resampled" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 120, + "sparse_mask_pool_size": 1000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120", + "sparse_mask_seed": 0, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 120 + sparse_mask_pool_size: 1000 + sparse_max_seq_length: 30720 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 30720] -- -- True +├─Qwen3Model: 1-1 -- [1, 30720, 768] -- True +│ └─Embedding: 2-1 [1, 30720] [1, 30720, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 30720, 768] [1, 30720, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [1, 30720, 768] [1, 30720, 768] 768 True +├─Linear: 1-2 [1, 30720, 768] [1, 30720, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 475.65 +================================================================================================================================================================ +Input size (MB): 0.25 +Forward/backward pass size (MB): 58494.81 +Params size (MB): 951.29 +Estimated Total Size (MB): 59446.35 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized... +Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736) +Dataset split: 749299 train, 39437 validation +Training examples (packed, train split): 749299 +Validation examples (packed, val split): 39437 +Per-device batch size: 1 +World size (GPUs): 1 +Gradient accumulation steps: 20 +Effective batch size (seqs/step): 20 +Sequence length: 30720 +Tokens per optimizer step: 614,400 +Micro-batches per epoch (per rank): 749299 +Optimizer steps per epoch: 37464 +Total optimizer steps: 11700 (capped by max_steps; 0.31 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run sh1m3lmh +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035835-sh1m3lmh +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m resampled 30k nrb120 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/sh1m3lmh +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/11700 | Loss: 12.0479 | LR: 6.67e-06 | 11329.2ms/step | 1.8 samples/s +Step 20/11700 | Loss: 11.9647 | LR: 1.33e-05 | 7600.0ms/step | 2.6 samples/s +Step 30/11700 | Loss: 11.6988 | LR: 2.00e-05 | 7599.8ms/step | 2.6 samples/s +Step 40/11700 | Loss: 11.1859 | LR: 2.67e-05 | 7591.9ms/step | 2.6 samples/s +Step 50/11700 | Loss: 10.8574 | LR: 3.33e-05 | 7571.2ms/step | 2.6 samples/s +Step 60/11700 | Loss: 10.5733 | LR: 4.00e-05 | 7557.1ms/step | 2.6 samples/s +Step 70/11700 | Loss: 10.2937 | LR: 4.67e-05 | 7547.9ms/step | 2.6 samples/s +Step 80/11700 | Loss: 10.0250 | LR: 5.33e-05 | 7550.5ms/step | 2.6 samples/s +Step 90/11700 | Loss: 9.7320 | LR: 6.00e-05 | 7539.6ms/step | 2.7 samples/s +Step 100/11700 | Loss: 9.3457 | LR: 6.67e-05 | 7543.7ms/step | 2.7 samples/s +Step 110/11700 | Loss: 8.8785 | LR: 7.33e-05 | 7540.9ms/step | 2.7 samples/s +Step 120/11700 | Loss: 8.3278 | LR: 8.00e-05 | 7543.5ms/step | 2.7 samples/s +Step 130/11700 | Loss: 7.8522 | LR: 8.67e-05 | 7546.4ms/step | 2.7 samples/s +Step 140/11700 | Loss: 7.5728 | LR: 9.33e-05 | 7549.7ms/step | 2.6 samples/s +Step 150/11700 | Loss: 7.3827 | LR: 1.00e-04 | 7551.9ms/step | 2.6 samples/s +Step 160/11700 | Loss: 7.1803 | LR: 1.07e-04 | 7559.6ms/step | 2.6 samples/s +Step 170/11700 | Loss: 7.0660 | LR: 1.13e-04 | 7565.9ms/step | 2.6 samples/s +Step 180/11700 | Loss: 6.9225 | LR: 1.20e-04 | 7574.8ms/step | 2.6 samples/s +Step 190/11700 | Loss: 6.8360 | LR: 1.27e-04 | 7583.7ms/step | 2.6 samples/s +Step 200/11700 | Loss: 6.6722 | LR: 1.33e-04 | 7582.5ms/step | 2.6 samples/s +Step 210/11700 | Loss: 6.5912 | LR: 1.40e-04 | 7592.1ms/step | 2.6 samples/s +Step 220/11700 | Loss: 6.4450 | LR: 1.47e-04 | 7599.8ms/step | 2.6 samples/s +Step 230/11700 | Loss: 6.3792 | LR: 1.53e-04 | 7594.3ms/step | 2.6 samples/s +Step 240/11700 | Loss: 6.2398 | LR: 1.60e-04 | 7597.7ms/step | 2.6 samples/s +Step 250/11700 | Loss: 6.2632 | LR: 1.67e-04 | 7600.1ms/step | 2.6 samples/s +Validation at step 250: Loss = 6.1315, Perplexity = 460.10 +Step 260/11700 | Loss: 6.1782 | LR: 1.73e-04 | 21230.3ms/step | 0.9 samples/s +Step 270/11700 | Loss: 6.0482 | LR: 1.80e-04 | 7604.0ms/step | 2.6 samples/s +Step 280/11700 | Loss: 5.9825 | LR: 1.87e-04 | 7600.6ms/step | 2.6 samples/s +Step 290/11700 | Loss: 5.8524 | LR: 1.93e-04 | 7611.6ms/step | 2.6 samples/s +Step 300/11700 | Loss: 5.8357 | LR: 2.00e-04 | 7606.3ms/step | 2.6 samples/s +Step 310/11700 | Loss: 5.8035 | LR: 2.07e-04 | 7609.8ms/step | 2.6 samples/s +Step 320/11700 | Loss: 5.6966 | LR: 2.13e-04 | 7605.5ms/step | 2.6 samples/s +Step 330/11700 | Loss: 5.5916 | LR: 2.20e-04 | 7600.5ms/step | 2.6 samples/s +Step 340/11700 | Loss: 5.5418 | LR: 2.27e-04 | 7600.1ms/step | 2.6 samples/s +Step 350/11700 | Loss: 5.3863 | LR: 2.33e-04 | 7604.4ms/step | 2.6 samples/s +Step 360/11700 | Loss: 5.4568 | LR: 2.40e-04 | 7605.7ms/step | 2.6 samples/s +Step 370/11700 | Loss: 5.4807 | LR: 2.47e-04 | 7601.9ms/step | 2.6 samples/s +Step 380/11700 | Loss: 5.4711 | LR: 2.53e-04 | 7608.6ms/step | 2.6 samples/s +Step 390/11700 | Loss: 5.4323 | LR: 2.60e-04 | 7601.6ms/step | 2.6 samples/s +Step 400/11700 | Loss: 5.2830 | LR: 2.67e-04 | 7600.1ms/step | 2.6 samples/s +Step 410/11700 | Loss: 5.2953 | LR: 2.73e-04 | 7598.0ms/step | 2.6 samples/s +Step 420/11700 | Loss: 5.2572 | LR: 2.80e-04 | 7600.0ms/step | 2.6 samples/s +Step 430/11700 | Loss: 5.2149 | LR: 2.87e-04 | 7604.0ms/step | 2.6 samples/s +Step 440/11700 | Loss: 5.2148 | LR: 2.93e-04 | 7600.2ms/step | 2.6 samples/s +Step 450/11700 | Loss: 5.1298 | LR: 3.00e-04 | 7600.0ms/step | 2.6 samples/s +Step 460/11700 | Loss: 5.1517 | LR: 3.07e-04 | 7600.1ms/step | 2.6 samples/s +Step 470/11700 | Loss: 5.1186 | LR: 3.13e-04 | 7600.0ms/step | 2.6 samples/s +Step 480/11700 | Loss: 5.1114 | LR: 3.20e-04 | 7599.9ms/step | 2.6 samples/s +Step 490/11700 | Loss: 5.0286 | LR: 3.27e-04 | 7602.3ms/step | 2.6 samples/s +Step 500/11700 | Loss: 4.9817 | LR: 3.33e-04 | 7606.0ms/step | 2.6 samples/s +Validation at step 500: Loss = 4.9360, Perplexity = 139.22 +Step 510/11700 | Loss: 4.9004 | LR: 3.40e-04 | 18995.0ms/step | 1.1 samples/s +Step 520/11700 | Loss: 4.9012 | LR: 3.47e-04 | 7627.4ms/step | 2.6 samples/s +Step 530/11700 | Loss: 4.8322 | LR: 3.53e-04 | 7626.6ms/step | 2.6 samples/s +Step 540/11700 | Loss: 4.7308 | LR: 3.60e-04 | 7633.1ms/step | 2.6 samples/s +Step 550/11700 | Loss: 4.7559 | LR: 3.67e-04 | 7629.6ms/step | 2.6 samples/s +Step 560/11700 | Loss: 4.7494 | LR: 3.73e-04 | 7632.7ms/step | 2.6 samples/s +Step 570/11700 | Loss: 4.5046 | LR: 3.80e-04 | 7639.8ms/step | 2.6 samples/s +Step 580/11700 | Loss: 4.5455 | LR: 3.87e-04 | 7633.7ms/step | 2.6 samples/s +Step 590/11700 | Loss: 4.5063 | LR: 3.93e-04 | 7628.7ms/step | 2.6 samples/s +Step 600/11700 | Loss: 4.5620 | LR: 4.00e-04 | 7628.2ms/step | 2.6 samples/s +Step 610/11700 | Loss: 4.5280 | LR: 4.07e-04 | 7630.2ms/step | 2.6 samples/s +Step 620/11700 | Loss: 4.5137 | LR: 4.13e-04 | 7632.8ms/step | 2.6 samples/s +Step 630/11700 | Loss: 4.3595 | LR: 4.20e-04 | 7637.1ms/step | 2.6 samples/s +Step 640/11700 | Loss: 4.4466 | LR: 4.27e-04 | 7638.8ms/step | 2.6 samples/s +Step 650/11700 | Loss: 4.3167 | LR: 4.33e-04 | 7632.1ms/step | 2.6 samples/s +Step 660/11700 | Loss: 4.4288 | LR: 4.40e-04 | 7630.8ms/step | 2.6 samples/s +Step 670/11700 | Loss: 4.3230 | LR: 4.47e-04 | 7634.9ms/step | 2.6 samples/s +Step 680/11700 | Loss: 4.2359 | LR: 4.53e-04 | 7629.6ms/step | 2.6 samples/s +Step 690/11700 | Loss: 4.0769 | LR: 4.60e-04 | 7633.8ms/step | 2.6 samples/s +Step 700/11700 | Loss: 4.2452 | LR: 4.67e-04 | 7636.3ms/step | 2.6 samples/s +Step 710/11700 | Loss: 4.2274 | LR: 4.73e-04 | 7637.9ms/step | 2.6 samples/s +Step 720/11700 | Loss: 4.1034 | LR: 4.80e-04 | 7636.5ms/step | 2.6 samples/s +Step 730/11700 | Loss: 3.9870 | LR: 4.87e-04 | 7636.2ms/step | 2.6 samples/s +Step 740/11700 | Loss: 4.1412 | LR: 4.93e-04 | 7641.2ms/step | 2.6 samples/s +Step 750/11700 | Loss: 4.0230 | LR: 5.00e-04 | 7638.8ms/step | 2.6 samples/s +Validation at step 750: Loss = 3.9706, Perplexity = 53.02 +Step 760/11700 | Loss: 4.0455 | LR: 5.00e-04 | 19000.0ms/step | 1.1 samples/s +Step 770/11700 | Loss: 3.8142 | LR: 5.00e-04 | 7634.6ms/step | 2.6 samples/s +Step 780/11700 | Loss: 4.0223 | LR: 5.00e-04 | 7632.9ms/step | 2.6 samples/s +Step 790/11700 | Loss: 3.7921 | LR: 5.00e-04 | 7633.7ms/step | 2.6 samples/s +Step 800/11700 | Loss: 3.7885 | LR: 5.00e-04 | 7629.8ms/step | 2.6 samples/s +Step 810/11700 | Loss: 3.8295 | LR: 5.00e-04 | 7632.8ms/step | 2.6 samples/s +Step 820/11700 | Loss: 3.8057 | LR: 5.00e-04 | 7634.0ms/step | 2.6 samples/s +Step 830/11700 | Loss: 3.8012 | LR: 5.00e-04 | 7624.5ms/step | 2.6 samples/s +Step 840/11700 | Loss: 3.8010 | LR: 5.00e-04 | 7634.0ms/step | 2.6 samples/s +Step 850/11700 | Loss: 3.8501 | LR: 5.00e-04 | 7629.5ms/step | 2.6 samples/s +Step 860/11700 | Loss: 3.6847 | LR: 5.00e-04 | 7634.0ms/step | 2.6 samples/s +Step 870/11700 | Loss: 3.7236 | LR: 5.00e-04 | 7638.1ms/step | 2.6 samples/s +Step 880/11700 | Loss: 3.7636 | LR: 5.00e-04 | 7632.9ms/step | 2.6 samples/s +Step 890/11700 | Loss: 3.6095 | LR: 5.00e-04 | 7636.6ms/step | 2.6 samples/s +Step 900/11700 | Loss: 3.5485 | LR: 5.00e-04 | 7629.0ms/step | 2.6 samples/s +Step 910/11700 | Loss: 3.7111 | LR: 5.00e-04 | 7636.1ms/step | 2.6 samples/s +Step 920/11700 | Loss: 3.5776 | LR: 5.00e-04 | 7617.8ms/step | 2.6 samples/s +Step 930/11700 | Loss: 3.5592 | LR: 5.00e-04 | 7633.3ms/step | 2.6 samples/s +Step 940/11700 | Loss: 3.6484 | LR: 5.00e-04 | 7628.7ms/step | 2.6 samples/s +Step 950/11700 | Loss: 3.4823 | LR: 5.00e-04 | 7630.0ms/step | 2.6 samples/s +Step 960/11700 | Loss: 3.5535 | LR: 5.00e-04 | 7622.5ms/step | 2.6 samples/s +Step 970/11700 | Loss: 3.6103 | LR: 5.00e-04 | 7631.1ms/step | 2.6 samples/s +Step 980/11700 | Loss: 3.3337 | LR: 4.99e-04 | 7633.1ms/step | 2.6 samples/s +Step 990/11700 | Loss: 3.5460 | LR: 4.99e-04 | 7635.6ms/step | 2.6 samples/s +Step 1000/11700 | Loss: 3.4406 | LR: 4.99e-04 | 7624.1ms/step | 2.6 samples/s +Validation at step 1000: Loss = 3.4950, Perplexity = 32.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-1000 +Step 1010/11700 | Loss: 3.5669 | LR: 4.99e-04 | 19271.6ms/step | 1.0 samples/s +Step 1020/11700 | Loss: 3.4663 | LR: 4.99e-04 | 7600.2ms/step | 2.6 samples/s +Step 1030/11700 | Loss: 3.4848 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s +Step 1040/11700 | Loss: 3.5081 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s +Step 1050/11700 | Loss: 3.4249 | LR: 4.99e-04 | 7604.0ms/step | 2.6 samples/s +Step 1060/11700 | Loss: 3.4844 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s +Step 1070/11700 | Loss: 3.4368 | LR: 4.99e-04 | 7606.0ms/step | 2.6 samples/s +Step 1080/11700 | Loss: 3.3002 | LR: 4.99e-04 | 7600.1ms/step | 2.6 samples/s +Step 1090/11700 | Loss: 3.3574 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s +Step 1100/11700 | Loss: 3.5562 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s +Step 1110/11700 | Loss: 3.5005 | LR: 4.99e-04 | 7600.1ms/step | 2.6 samples/s +Step 1120/11700 | Loss: 3.3888 | LR: 4.99e-04 | 7599.9ms/step | 2.6 samples/s +Step 1130/11700 | Loss: 3.3930 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s +Step 1140/11700 | Loss: 3.1697 | LR: 4.98e-04 | 7600.2ms/step | 2.6 samples/s +Step 1150/11700 | Loss: 3.3267 | LR: 4.98e-04 | 7600.1ms/step | 2.6 samples/s +Step 1160/11700 | Loss: 3.3372 | LR: 4.98e-04 | 7610.0ms/step | 2.6 samples/s +Step 1170/11700 | Loss: 3.3013 | LR: 4.98e-04 | 7599.9ms/step | 2.6 samples/s +Step 1180/11700 | Loss: 3.2397 | LR: 4.98e-04 | 7626.6ms/step | 2.6 samples/s +Step 1190/11700 | Loss: 3.2126 | LR: 4.98e-04 | 7629.5ms/step | 2.6 samples/s +Step 1200/11700 | Loss: 3.1437 | LR: 4.98e-04 | 7634.4ms/step | 2.6 samples/s +Step 1210/11700 | Loss: 3.4685 | LR: 4.98e-04 | 7628.3ms/step | 2.6 samples/s +Step 1220/11700 | Loss: 3.2654 | LR: 4.98e-04 | 7638.9ms/step | 2.6 samples/s +Step 1230/11700 | Loss: 3.3039 | LR: 4.98e-04 | 7636.8ms/step | 2.6 samples/s +Step 1240/11700 | Loss: 3.2457 | LR: 4.98e-04 | 7630.8ms/step | 2.6 samples/s +Step 1250/11700 | Loss: 3.2235 | LR: 4.97e-04 | 7635.2ms/step | 2.6 samples/s +Validation at step 1250: Loss = 3.2701, Perplexity = 26.31 +Step 1260/11700 | Loss: 3.3010 | LR: 4.97e-04 | 18991.9ms/step | 1.1 samples/s +Step 1270/11700 | Loss: 3.1347 | LR: 4.97e-04 | 7634.7ms/step | 2.6 samples/s +Step 1280/11700 | Loss: 3.2432 | LR: 4.97e-04 | 7638.1ms/step | 2.6 samples/s +Step 1290/11700 | Loss: 3.2414 | LR: 4.97e-04 | 7639.6ms/step | 2.6 samples/s +Step 1300/11700 | Loss: 3.2508 | LR: 4.97e-04 | 7631.5ms/step | 2.6 samples/s +Step 1310/11700 | Loss: 3.3061 | LR: 4.97e-04 | 7640.9ms/step | 2.6 samples/s +Step 1320/11700 | Loss: 3.2569 | LR: 4.97e-04 | 7638.5ms/step | 2.6 samples/s +Step 1330/11700 | Loss: 3.1805 | LR: 4.97e-04 | 7635.2ms/step | 2.6 samples/s +Step 1340/11700 | Loss: 3.2266 | LR: 4.96e-04 | 7632.4ms/step | 2.6 samples/s +Step 1350/11700 | Loss: 3.2419 | LR: 4.96e-04 | 7630.5ms/step | 2.6 samples/s +Step 1360/11700 | Loss: 3.2054 | LR: 4.96e-04 | 7633.6ms/step | 2.6 samples/s +Step 1370/11700 | Loss: 3.1435 | LR: 4.96e-04 | 7636.3ms/step | 2.6 samples/s +Step 1380/11700 | Loss: 3.0078 | LR: 4.96e-04 | 7637.6ms/step | 2.6 samples/s +Step 1390/11700 | Loss: 3.1693 | LR: 4.96e-04 | 7631.2ms/step | 2.6 samples/s +Step 1400/11700 | Loss: 3.2617 | LR: 4.96e-04 | 7636.3ms/step | 2.6 samples/s +Step 1410/11700 | Loss: 3.1966 | LR: 4.96e-04 | 7638.8ms/step | 2.6 samples/s +Step 1420/11700 | Loss: 3.0909 | LR: 4.95e-04 | 7638.4ms/step | 2.6 samples/s +Step 1430/11700 | Loss: 3.1389 | LR: 4.95e-04 | 7637.6ms/step | 2.6 samples/s +Step 1440/11700 | Loss: 3.1319 | LR: 4.95e-04 | 7618.6ms/step | 2.6 samples/s +Step 1450/11700 | Loss: 3.0518 | LR: 4.95e-04 | 7611.5ms/step | 2.6 samples/s +Step 1460/11700 | Loss: 3.1102 | LR: 4.95e-04 | 7610.3ms/step | 2.6 samples/s +Step 1470/11700 | Loss: 3.1211 | LR: 4.95e-04 | 7611.9ms/step | 2.6 samples/s +Step 1480/11700 | Loss: 3.0520 | LR: 4.95e-04 | 7616.5ms/step | 2.6 samples/s +Step 1490/11700 | Loss: 3.2031 | LR: 4.94e-04 | 7615.6ms/step | 2.6 samples/s +Step 1500/11700 | Loss: 3.0442 | LR: 4.94e-04 | 7616.7ms/step | 2.6 samples/s +Validation at step 1500: Loss = 3.1245, Perplexity = 22.75 +Step 1510/11700 | Loss: 3.1192 | LR: 4.94e-04 | 18947.7ms/step | 1.1 samples/s +Step 1520/11700 | Loss: 3.1062 | LR: 4.94e-04 | 7608.0ms/step | 2.6 samples/s +Step 1530/11700 | Loss: 3.1092 | LR: 4.94e-04 | 7605.8ms/step | 2.6 samples/s +Step 1540/11700 | Loss: 3.2591 | LR: 4.94e-04 | 7606.1ms/step | 2.6 samples/s +Step 1550/11700 | Loss: 3.1102 | LR: 4.93e-04 | 7604.1ms/step | 2.6 samples/s +Step 1560/11700 | Loss: 3.1356 | LR: 4.93e-04 | 7606.2ms/step | 2.6 samples/s +Step 1570/11700 | Loss: 3.1269 | LR: 4.93e-04 | 7606.5ms/step | 2.6 samples/s +Step 1580/11700 | Loss: 3.0856 | LR: 4.93e-04 | 7601.3ms/step | 2.6 samples/s +Step 1590/11700 | Loss: 3.2140 | LR: 4.93e-04 | 7600.5ms/step | 2.6 samples/s +Step 1600/11700 | Loss: 3.1014 | LR: 4.93e-04 | 7610.0ms/step | 2.6 samples/s +Step 1610/11700 | Loss: 3.1492 | LR: 4.92e-04 | 7605.6ms/step | 2.6 samples/s +Step 1620/11700 | Loss: 3.0294 | LR: 4.92e-04 | 7604.3ms/step | 2.6 samples/s +Step 1630/11700 | Loss: 2.9351 | LR: 4.92e-04 | 7607.8ms/step | 2.6 samples/s +Step 1640/11700 | Loss: 3.1176 | LR: 4.92e-04 | 7607.5ms/step | 2.6 samples/s +Step 1650/11700 | Loss: 3.0504 | LR: 4.92e-04 | 7604.8ms/step | 2.6 samples/s +Step 1660/11700 | Loss: 3.0839 | LR: 4.92e-04 | 7601.3ms/step | 2.6 samples/s +Step 1670/11700 | Loss: 3.2062 | LR: 4.91e-04 | 7604.7ms/step | 2.6 samples/s +Step 1680/11700 | Loss: 3.0530 | LR: 4.91e-04 | 7603.2ms/step | 2.6 samples/s +Step 1690/11700 | Loss: 2.9155 | LR: 4.91e-04 | 7602.9ms/step | 2.6 samples/s +Step 1700/11700 | Loss: 2.9772 | LR: 4.91e-04 | 7604.0ms/step | 2.6 samples/s +Step 1710/11700 | Loss: 3.1062 | LR: 4.91e-04 | 7605.0ms/step | 2.6 samples/s +Step 1720/11700 | Loss: 3.0288 | LR: 4.90e-04 | 7603.2ms/step | 2.6 samples/s +Step 1730/11700 | Loss: 3.0550 | LR: 4.90e-04 | 7600.3ms/step | 2.6 samples/s +Step 1740/11700 | Loss: 2.9532 | LR: 4.90e-04 | 7603.7ms/step | 2.6 samples/s +Step 1750/11700 | Loss: 2.9292 | LR: 4.90e-04 | 7600.0ms/step | 2.6 samples/s +Validation at step 1750: Loss = 3.0491, Perplexity = 21.10 +Step 1760/11700 | Loss: 3.0268 | LR: 4.90e-04 | 18936.3ms/step | 1.1 samples/s +Step 1770/11700 | Loss: 3.0251 | LR: 4.89e-04 | 7602.7ms/step | 2.6 samples/s +Step 1780/11700 | Loss: 3.0488 | LR: 4.89e-04 | 7603.3ms/step | 2.6 samples/s +Step 1790/11700 | Loss: 3.0190 | LR: 4.89e-04 | 7602.2ms/step | 2.6 samples/s +Step 1800/11700 | Loss: 3.0347 | LR: 4.89e-04 | 7606.0ms/step | 2.6 samples/s +Step 1810/11700 | Loss: 3.0920 | LR: 4.89e-04 | 7606.3ms/step | 2.6 samples/s +Step 1820/11700 | Loss: 3.0454 | LR: 4.88e-04 | 7612.4ms/step | 2.6 samples/s +Step 1830/11700 | Loss: 3.0013 | LR: 4.88e-04 | 7606.7ms/step | 2.6 samples/s +Step 1840/11700 | Loss: 2.9907 | LR: 4.88e-04 | 7604.7ms/step | 2.6 samples/s +Step 1850/11700 | Loss: 3.0100 | LR: 4.88e-04 | 7599.9ms/step | 2.6 samples/s +Step 1860/11700 | Loss: 2.9925 | LR: 4.87e-04 | 7600.2ms/step | 2.6 samples/s +Step 1870/11700 | Loss: 2.9190 | LR: 4.87e-04 | 7600.1ms/step | 2.6 samples/s +Step 1880/11700 | Loss: 2.9439 | LR: 4.87e-04 | 7602.0ms/step | 2.6 samples/s +Step 1890/11700 | Loss: 3.0621 | LR: 4.87e-04 | 7600.0ms/step | 2.6 samples/s +Step 1900/11700 | Loss: 3.0059 | LR: 4.87e-04 | 7602.3ms/step | 2.6 samples/s +Step 1910/11700 | Loss: 3.0302 | LR: 4.86e-04 | 7600.1ms/step | 2.6 samples/s +Step 1920/11700 | Loss: 2.8230 | LR: 4.86e-04 | 7603.9ms/step | 2.6 samples/s +Step 1930/11700 | Loss: 3.0104 | LR: 4.86e-04 | 7602.1ms/step | 2.6 samples/s +Step 1940/11700 | Loss: 2.9483 | LR: 4.86e-04 | 7599.8ms/step | 2.6 samples/s +Step 1950/11700 | Loss: 2.8686 | LR: 4.85e-04 | 7600.3ms/step | 2.6 samples/s +Step 1960/11700 | Loss: 2.9357 | LR: 4.85e-04 | 7599.9ms/step | 2.6 samples/s +Step 1970/11700 | Loss: 2.9603 | LR: 4.85e-04 | 7599.8ms/step | 2.6 samples/s +Step 1980/11700 | Loss: 2.9403 | LR: 4.85e-04 | 7600.1ms/step | 2.6 samples/s +Step 1990/11700 | Loss: 2.8852 | LR: 4.84e-04 | 7602.6ms/step | 2.6 samples/s +Step 2000/11700 | Loss: 3.0944 | LR: 4.84e-04 | 7607.7ms/step | 2.6 samples/s +Validation at step 2000: Loss = 2.9636, Perplexity = 19.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-2000 +Step 2010/11700 | Loss: 2.9827 | LR: 4.84e-04 | 19260.5ms/step | 1.0 samples/s +Step 2020/11700 | Loss: 3.0175 | LR: 4.84e-04 | 7613.7ms/step | 2.6 samples/s +Step 2030/11700 | Loss: 2.9211 | LR: 4.83e-04 | 7616.1ms/step | 2.6 samples/s +Step 2040/11700 | Loss: 2.9654 | LR: 4.83e-04 | 7616.1ms/step | 2.6 samples/s +Step 2050/11700 | Loss: 2.8388 | LR: 4.83e-04 | 7621.4ms/step | 2.6 samples/s +Step 2060/11700 | Loss: 2.9127 | LR: 4.83e-04 | 7608.7ms/step | 2.6 samples/s +Step 2070/11700 | Loss: 2.9145 | LR: 4.82e-04 | 7608.3ms/step | 2.6 samples/s +Step 2080/11700 | Loss: 2.8878 | LR: 4.82e-04 | 7611.9ms/step | 2.6 samples/s +Step 2090/11700 | Loss: 2.9154 | LR: 4.82e-04 | 7611.8ms/step | 2.6 samples/s +Step 2100/11700 | Loss: 3.0062 | LR: 4.81e-04 | 7614.5ms/step | 2.6 samples/s +Step 2110/11700 | Loss: 3.0052 | LR: 4.81e-04 | 7609.7ms/step | 2.6 samples/s +Step 2120/11700 | Loss: 2.9525 | LR: 4.81e-04 | 7638.7ms/step | 2.6 samples/s +Step 2130/11700 | Loss: 2.9403 | LR: 4.81e-04 | 7627.2ms/step | 2.6 samples/s +Step 2140/11700 | Loss: 3.0496 | LR: 4.80e-04 | 7618.3ms/step | 2.6 samples/s +Step 2150/11700 | Loss: 2.9641 | LR: 4.80e-04 | 7616.1ms/step | 2.6 samples/s +Step 2160/11700 | Loss: 2.9237 | LR: 4.80e-04 | 7615.9ms/step | 2.6 samples/s +Step 2170/11700 | Loss: 2.8569 | LR: 4.80e-04 | 7610.7ms/step | 2.6 samples/s +Step 2180/11700 | Loss: 2.8931 | LR: 4.79e-04 | 7613.6ms/step | 2.6 samples/s +Step 2190/11700 | Loss: 2.8802 | LR: 4.79e-04 | 7608.1ms/step | 2.6 samples/s +Step 2200/11700 | Loss: 2.9422 | LR: 4.79e-04 | 7611.9ms/step | 2.6 samples/s +Step 2210/11700 | Loss: 2.9791 | LR: 4.78e-04 | 7604.0ms/step | 2.6 samples/s +Step 2220/11700 | Loss: 2.7883 | LR: 4.78e-04 | 7604.0ms/step | 2.6 samples/s +Step 2230/11700 | Loss: 2.8241 | LR: 4.78e-04 | 7602.0ms/step | 2.6 samples/s +Step 2240/11700 | Loss: 2.8108 | LR: 4.78e-04 | 7612.1ms/step | 2.6 samples/s +Step 2250/11700 | Loss: 3.0036 | LR: 4.77e-04 | 7610.2ms/step | 2.6 samples/s +Validation at step 2250: Loss = 2.9141, Perplexity = 18.43 +Step 2260/11700 | Loss: 2.7842 | LR: 4.77e-04 | 18947.9ms/step | 1.1 samples/s +Step 2270/11700 | Loss: 2.9975 | LR: 4.77e-04 | 7610.2ms/step | 2.6 samples/s +Step 2280/11700 | Loss: 2.9094 | LR: 4.76e-04 | 7614.0ms/step | 2.6 samples/s +Step 2290/11700 | Loss: 2.9258 | LR: 4.76e-04 | 7611.9ms/step | 2.6 samples/s +Step 2300/11700 | Loss: 2.8657 | LR: 4.76e-04 | 7608.3ms/step | 2.6 samples/s +Step 2310/11700 | Loss: 2.8510 | LR: 4.75e-04 | 7611.6ms/step | 2.6 samples/s +Step 2320/11700 | Loss: 3.0006 | LR: 4.75e-04 | 7612.7ms/step | 2.6 samples/s +Step 2330/11700 | Loss: 2.8261 | LR: 4.75e-04 | 7613.2ms/step | 2.6 samples/s +Step 2340/11700 | Loss: 2.8632 | LR: 4.74e-04 | 7612.4ms/step | 2.6 samples/s +Step 2350/11700 | Loss: 2.7935 | LR: 4.74e-04 | 7608.2ms/step | 2.6 samples/s +Step 2360/11700 | Loss: 2.8026 | LR: 4.74e-04 | 7608.0ms/step | 2.6 samples/s +Step 2370/11700 | Loss: 2.7833 | LR: 4.73e-04 | 7609.8ms/step | 2.6 samples/s +Step 2380/11700 | Loss: 2.7778 | LR: 4.73e-04 | 7616.0ms/step | 2.6 samples/s +Step 2390/11700 | Loss: 2.9199 | LR: 4.73e-04 | 7611.9ms/step | 2.6 samples/s +Step 2400/11700 | Loss: 2.8552 | LR: 4.73e-04 | 7610.3ms/step | 2.6 samples/s +Step 2410/11700 | Loss: 2.8733 | LR: 4.72e-04 | 7610.0ms/step | 2.6 samples/s +Step 2420/11700 | Loss: 2.7364 | LR: 4.72e-04 | 7611.3ms/step | 2.6 samples/s +Step 2430/11700 | Loss: 2.9123 | LR: 4.72e-04 | 7612.6ms/step | 2.6 samples/s +Step 2440/11700 | Loss: 2.9334 | LR: 4.71e-04 | 7612.2ms/step | 2.6 samples/s +Step 2450/11700 | Loss: 2.7649 | LR: 4.71e-04 | 7612.0ms/step | 2.6 samples/s +Step 2460/11700 | Loss: 2.8805 | LR: 4.71e-04 | 7608.6ms/step | 2.6 samples/s +Step 2470/11700 | Loss: 2.8871 | LR: 4.70e-04 | 7611.8ms/step | 2.6 samples/s +Step 2480/11700 | Loss: 2.8158 | LR: 4.70e-04 | 7608.0ms/step | 2.6 samples/s +Step 2490/11700 | Loss: 2.9086 | LR: 4.69e-04 | 7609.5ms/step | 2.6 samples/s +Step 2500/11700 | Loss: 2.7867 | LR: 4.69e-04 | 7610.2ms/step | 2.6 samples/s +Validation at step 2500: Loss = 2.8493, Perplexity = 17.28 +Step 2510/11700 | Loss: 2.7591 | LR: 4.69e-04 | 18952.3ms/step | 1.1 samples/s +Step 2520/11700 | Loss: 2.9264 | LR: 4.68e-04 | 7608.4ms/step | 2.6 samples/s +Step 2530/11700 | Loss: 2.8196 | LR: 4.68e-04 | 7612.4ms/step | 2.6 samples/s +Step 2540/11700 | Loss: 2.7508 | LR: 4.68e-04 | 7611.4ms/step | 2.6 samples/s +Step 2550/11700 | Loss: 2.9211 | LR: 4.67e-04 | 7608.3ms/step | 2.6 samples/s +Step 2560/11700 | Loss: 2.8610 | LR: 4.67e-04 | 7609.7ms/step | 2.6 samples/s +Step 2570/11700 | Loss: 2.8221 | LR: 4.67e-04 | 7610.4ms/step | 2.6 samples/s +Step 2580/11700 | Loss: 2.8414 | LR: 4.66e-04 | 7611.9ms/step | 2.6 samples/s +Step 2590/11700 | Loss: 2.7814 | LR: 4.66e-04 | 7609.1ms/step | 2.6 samples/s +Step 2600/11700 | Loss: 2.8094 | LR: 4.66e-04 | 7611.1ms/step | 2.6 samples/s +Step 2610/11700 | Loss: 2.7369 | LR: 4.65e-04 | 7615.8ms/step | 2.6 samples/s +Step 2620/11700 | Loss: 2.7949 | LR: 4.65e-04 | 7612.1ms/step | 2.6 samples/s +Step 2630/11700 | Loss: 2.7418 | LR: 4.65e-04 | 7610.1ms/step | 2.6 samples/s +Step 2640/11700 | Loss: 2.8498 | LR: 4.64e-04 | 7610.5ms/step | 2.6 samples/s +Step 2650/11700 | Loss: 2.9133 | LR: 4.64e-04 | 7605.9ms/step | 2.6 samples/s +Step 2660/11700 | Loss: 2.8431 | LR: 4.63e-04 | 7608.1ms/step | 2.6 samples/s +Step 2670/11700 | Loss: 2.7867 | LR: 4.63e-04 | 7609.9ms/step | 2.6 samples/s +Step 2680/11700 | Loss: 2.8282 | LR: 4.63e-04 | 7611.7ms/step | 2.6 samples/s +Step 2690/11700 | Loss: 2.9309 | LR: 4.62e-04 | 7608.5ms/step | 2.6 samples/s +Step 2700/11700 | Loss: 2.8279 | LR: 4.62e-04 | 7610.2ms/step | 2.6 samples/s +Step 2710/11700 | Loss: 2.8247 | LR: 4.62e-04 | 7611.9ms/step | 2.6 samples/s +Step 2720/11700 | Loss: 2.8291 | LR: 4.61e-04 | 7607.9ms/step | 2.6 samples/s +Step 2730/11700 | Loss: 2.7304 | LR: 4.61e-04 | 7610.0ms/step | 2.6 samples/s +Step 2740/11700 | Loss: 2.8146 | LR: 4.60e-04 | 7608.0ms/step | 2.6 samples/s +Step 2750/11700 | Loss: 2.8669 | LR: 4.60e-04 | 7614.1ms/step | 2.6 samples/s +Validation at step 2750: Loss = 2.8076, Perplexity = 16.57 +Step 2760/11700 | Loss: 2.6176 | LR: 4.60e-04 | 18948.5ms/step | 1.1 samples/s +Step 2770/11700 | Loss: 2.7700 | LR: 4.59e-04 | 7611.5ms/step | 2.6 samples/s +Step 2780/11700 | Loss: 2.7771 | LR: 4.59e-04 | 7606.3ms/step | 2.6 samples/s +Step 2790/11700 | Loss: 2.8256 | LR: 4.58e-04 | 7612.0ms/step | 2.6 samples/s +Step 2800/11700 | Loss: 2.7337 | LR: 4.58e-04 | 7607.8ms/step | 2.6 samples/s +Step 2810/11700 | Loss: 2.8341 | LR: 4.58e-04 | 7609.9ms/step | 2.6 samples/s +Step 2820/11700 | Loss: 2.8189 | LR: 4.57e-04 | 7606.6ms/step | 2.6 samples/s +Step 2830/11700 | Loss: 2.7889 | LR: 4.57e-04 | 7609.6ms/step | 2.6 samples/s +Step 2840/11700 | Loss: 2.8658 | LR: 4.56e-04 | 7610.8ms/step | 2.6 samples/s +Step 2850/11700 | Loss: 2.8974 | LR: 4.56e-04 | 7613.5ms/step | 2.6 samples/s +Step 2860/11700 | Loss: 2.7809 | LR: 4.56e-04 | 7610.2ms/step | 2.6 samples/s +Step 2870/11700 | Loss: 2.8514 | LR: 4.55e-04 | 7613.8ms/step | 2.6 samples/s +Step 2880/11700 | Loss: 2.5813 | LR: 4.55e-04 | 7610.0ms/step | 2.6 samples/s +Step 2890/11700 | Loss: 2.8308 | LR: 4.54e-04 | 7612.3ms/step | 2.6 samples/s +Step 2900/11700 | Loss: 2.7753 | LR: 4.54e-04 | 7608.1ms/step | 2.6 samples/s +Step 2910/11700 | Loss: 2.8491 | LR: 4.54e-04 | 7611.9ms/step | 2.6 samples/s +Step 2920/11700 | Loss: 2.7774 | LR: 4.53e-04 | 7611.0ms/step | 2.6 samples/s +Step 2930/11700 | Loss: 2.8587 | LR: 4.53e-04 | 7613.1ms/step | 2.6 samples/s +Step 2940/11700 | Loss: 2.7871 | LR: 4.52e-04 | 7610.5ms/step | 2.6 samples/s +Step 2950/11700 | Loss: 2.8360 | LR: 4.52e-04 | 7607.9ms/step | 2.6 samples/s +Step 2960/11700 | Loss: 2.7305 | LR: 4.51e-04 | 7608.6ms/step | 2.6 samples/s +Step 2970/11700 | Loss: 2.6809 | LR: 4.51e-04 | 7611.6ms/step | 2.6 samples/s +Step 2980/11700 | Loss: 2.7625 | LR: 4.51e-04 | 7611.5ms/step | 2.6 samples/s +Step 2990/11700 | Loss: 2.6966 | LR: 4.50e-04 | 7606.3ms/step | 2.6 samples/s +Step 3000/11700 | Loss: 2.7739 | LR: 4.50e-04 | 7612.8ms/step | 2.6 samples/s +Validation at step 3000: Loss = 2.7782, Perplexity = 16.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-3000 +Step 3010/11700 | Loss: 2.7432 | LR: 4.49e-04 | 19268.7ms/step | 1.0 samples/s +Step 3020/11700 | Loss: 2.8894 | LR: 4.49e-04 | 7612.2ms/step | 2.6 samples/s +Step 3030/11700 | Loss: 2.7245 | LR: 4.48e-04 | 7614.5ms/step | 2.6 samples/s +Step 3040/11700 | Loss: 2.8307 | LR: 4.48e-04 | 7616.1ms/step | 2.6 samples/s +Step 3050/11700 | Loss: 2.7633 | LR: 4.48e-04 | 7614.0ms/step | 2.6 samples/s +Step 3060/11700 | Loss: 2.9458 | LR: 4.47e-04 | 7609.8ms/step | 2.6 samples/s +Step 3070/11700 | Loss: 2.7247 | LR: 4.47e-04 | 7614.3ms/step | 2.6 samples/s +Step 3080/11700 | Loss: 2.7233 | LR: 4.46e-04 | 7616.7ms/step | 2.6 samples/s +Step 3090/11700 | Loss: 2.7289 | LR: 4.46e-04 | 7615.1ms/step | 2.6 samples/s +Step 3100/11700 | Loss: 2.8609 | LR: 4.45e-04 | 7612.2ms/step | 2.6 samples/s +Step 3110/11700 | Loss: 2.6425 | LR: 4.45e-04 | 7610.6ms/step | 2.6 samples/s +Step 3120/11700 | Loss: 2.7153 | LR: 4.44e-04 | 7611.3ms/step | 2.6 samples/s +Step 3130/11700 | Loss: 2.7524 | LR: 4.44e-04 | 7610.3ms/step | 2.6 samples/s +Step 3140/11700 | Loss: 2.7032 | LR: 4.43e-04 | 7608.6ms/step | 2.6 samples/s +Step 3150/11700 | Loss: 2.7226 | LR: 4.43e-04 | 7609.2ms/step | 2.6 samples/s +Step 3160/11700 | Loss: 2.7232 | LR: 4.43e-04 | 7607.6ms/step | 2.6 samples/s +Step 3170/11700 | Loss: 2.8159 | LR: 4.42e-04 | 7611.0ms/step | 2.6 samples/s +Step 3180/11700 | Loss: 2.7809 | LR: 4.42e-04 | 7609.9ms/step | 2.6 samples/s +Step 3190/11700 | Loss: 2.6536 | LR: 4.41e-04 | 7610.7ms/step | 2.6 samples/s +Step 3200/11700 | Loss: 2.6841 | LR: 4.41e-04 | 7613.1ms/step | 2.6 samples/s +Step 3210/11700 | Loss: 2.7078 | LR: 4.40e-04 | 7608.4ms/step | 2.6 samples/s +Step 3220/11700 | Loss: 2.7307 | LR: 4.40e-04 | 7612.4ms/step | 2.6 samples/s +Step 3230/11700 | Loss: 2.6818 | LR: 4.39e-04 | 7613.6ms/step | 2.6 samples/s +Step 3240/11700 | Loss: 2.7486 | LR: 4.39e-04 | 7611.8ms/step | 2.6 samples/s +Step 3250/11700 | Loss: 2.8363 | LR: 4.38e-04 | 7612.4ms/step | 2.6 samples/s +Validation at step 3250: Loss = 2.7420, Perplexity = 15.52 +Step 3260/11700 | Loss: 2.7500 | LR: 4.38e-04 | 18947.9ms/step | 1.1 samples/s +Step 3270/11700 | Loss: 2.6923 | LR: 4.37e-04 | 7610.1ms/step | 2.6 samples/s +Step 3280/11700 | Loss: 2.7506 | LR: 4.37e-04 | 7606.4ms/step | 2.6 samples/s +Step 3290/11700 | Loss: 2.6981 | LR: 4.37e-04 | 7607.9ms/step | 2.6 samples/s +Step 3300/11700 | Loss: 2.7601 | LR: 4.36e-04 | 7612.2ms/step | 2.6 samples/s +Step 3310/11700 | Loss: 2.7601 | LR: 4.36e-04 | 7609.8ms/step | 2.6 samples/s +Step 3320/11700 | Loss: 2.6774 | LR: 4.35e-04 | 7613.8ms/step | 2.6 samples/s +Step 3330/11700 | Loss: 2.6634 | LR: 4.35e-04 | 7610.2ms/step | 2.6 samples/s +Step 3340/11700 | Loss: 2.7992 | LR: 4.34e-04 | 7609.9ms/step | 2.6 samples/s +Step 3350/11700 | Loss: 2.7002 | LR: 4.34e-04 | 7606.3ms/step | 2.6 samples/s +Step 3360/11700 | Loss: 2.7788 | LR: 4.33e-04 | 7610.0ms/step | 2.6 samples/s +Step 3370/11700 | Loss: 2.6616 | LR: 4.33e-04 | 7610.6ms/step | 2.6 samples/s +Step 3380/11700 | Loss: 2.7292 | LR: 4.32e-04 | 7615.1ms/step | 2.6 samples/s +Step 3390/11700 | Loss: 2.6927 | LR: 4.32e-04 | 7610.3ms/step | 2.6 samples/s +Step 3400/11700 | Loss: 2.8020 | LR: 4.31e-04 | 7610.3ms/step | 2.6 samples/s +Step 3410/11700 | Loss: 2.7439 | LR: 4.31e-04 | 7613.8ms/step | 2.6 samples/s +Step 3420/11700 | Loss: 2.5693 | LR: 4.30e-04 | 7611.9ms/step | 2.6 samples/s +Step 3430/11700 | Loss: 2.7554 | LR: 4.30e-04 | 7611.2ms/step | 2.6 samples/s +Step 3440/11700 | Loss: 2.6121 | LR: 4.29e-04 | 7605.3ms/step | 2.6 samples/s +Step 3450/11700 | Loss: 2.7246 | LR: 4.29e-04 | 7611.5ms/step | 2.6 samples/s +Step 3460/11700 | Loss: 2.7085 | LR: 4.28e-04 | 7610.2ms/step | 2.6 samples/s +Step 3470/11700 | Loss: 2.5974 | LR: 4.28e-04 | 7612.2ms/step | 2.6 samples/s +Step 3480/11700 | Loss: 2.5218 | LR: 4.27e-04 | 7613.7ms/step | 2.6 samples/s +Step 3490/11700 | Loss: 2.8501 | LR: 4.27e-04 | 7612.0ms/step | 2.6 samples/s +Step 3500/11700 | Loss: 2.7622 | LR: 4.26e-04 | 7614.3ms/step | 2.6 samples/s +Validation at step 3500: Loss = 2.7115, Perplexity = 15.05 +Step 3510/11700 | Loss: 2.7423 | LR: 4.26e-04 | 18958.3ms/step | 1.1 samples/s +Step 3520/11700 | Loss: 2.7299 | LR: 4.25e-04 | 7617.2ms/step | 2.6 samples/s +Step 3530/11700 | Loss: 2.7402 | LR: 4.25e-04 | 7609.4ms/step | 2.6 samples/s +Step 3540/11700 | Loss: 2.7489 | LR: 4.24e-04 | 7615.1ms/step | 2.6 samples/s +Step 3550/11700 | Loss: 2.6690 | LR: 4.24e-04 | 7608.3ms/step | 2.6 samples/s +Step 3560/11700 | Loss: 2.8603 | LR: 4.23e-04 | 7607.9ms/step | 2.6 samples/s +Step 3570/11700 | Loss: 2.7303 | LR: 4.23e-04 | 7605.9ms/step | 2.6 samples/s +Step 3580/11700 | Loss: 2.7424 | LR: 4.22e-04 | 7606.1ms/step | 2.6 samples/s +Step 3590/11700 | Loss: 2.6399 | LR: 4.22e-04 | 7622.5ms/step | 2.6 samples/s +Step 3600/11700 | Loss: 2.6372 | LR: 4.21e-04 | 7611.4ms/step | 2.6 samples/s +Step 3610/11700 | Loss: 2.7694 | LR: 4.20e-04 | 7612.6ms/step | 2.6 samples/s +Step 3620/11700 | Loss: 2.6361 | LR: 4.20e-04 | 7619.8ms/step | 2.6 samples/s +Step 3630/11700 | Loss: 2.6453 | LR: 4.19e-04 | 7608.4ms/step | 2.6 samples/s +Step 3640/11700 | Loss: 2.5932 | LR: 4.19e-04 | 7609.8ms/step | 2.6 samples/s +Step 3650/11700 | Loss: 2.7232 | LR: 4.18e-04 | 7608.1ms/step | 2.6 samples/s +Step 3660/11700 | Loss: 2.6810 | LR: 4.18e-04 | 7611.7ms/step | 2.6 samples/s +Step 3670/11700 | Loss: 2.6434 | LR: 4.17e-04 | 7610.3ms/step | 2.6 samples/s +Step 3680/11700 | Loss: 2.7385 | LR: 4.17e-04 | 7610.6ms/step | 2.6 samples/s +Step 3690/11700 | Loss: 2.6741 | LR: 4.16e-04 | 7615.1ms/step | 2.6 samples/s +Step 3700/11700 | Loss: 2.7155 | LR: 4.16e-04 | 7611.5ms/step | 2.6 samples/s +Step 3710/11700 | Loss: 2.6129 | LR: 4.15e-04 | 7607.6ms/step | 2.6 samples/s +Step 3720/11700 | Loss: 2.6344 | LR: 4.15e-04 | 7605.3ms/step | 2.6 samples/s +Step 3730/11700 | Loss: 2.6931 | LR: 4.14e-04 | 7606.2ms/step | 2.6 samples/s +Step 3740/11700 | Loss: 2.6414 | LR: 4.14e-04 | 7608.1ms/step | 2.6 samples/s +Step 3750/11700 | Loss: 2.7151 | LR: 4.13e-04 | 7611.8ms/step | 2.6 samples/s +Validation at step 3750: Loss = 2.6834, Perplexity = 14.64 +Step 3760/11700 | Loss: 2.7246 | LR: 4.12e-04 | 18946.0ms/step | 1.1 samples/s +Step 3770/11700 | Loss: 2.7152 | LR: 4.12e-04 | 7602.4ms/step | 2.6 samples/s +Step 3780/11700 | Loss: 2.6642 | LR: 4.11e-04 | 7599.8ms/step | 2.6 samples/s +Step 3790/11700 | Loss: 2.8064 | LR: 4.11e-04 | 7599.9ms/step | 2.6 samples/s +Step 3800/11700 | Loss: 2.6964 | LR: 4.10e-04 | 7600.2ms/step | 2.6 samples/s +Step 3810/11700 | Loss: 2.6567 | LR: 4.10e-04 | 7603.9ms/step | 2.6 samples/s +Step 3820/11700 | Loss: 2.6447 | LR: 4.09e-04 | 7606.6ms/step | 2.6 samples/s +Step 3830/11700 | Loss: 2.6665 | LR: 4.09e-04 | 7603.7ms/step | 2.6 samples/s +Step 3840/11700 | Loss: 2.5959 | LR: 4.08e-04 | 7602.2ms/step | 2.6 samples/s +Step 3850/11700 | Loss: 2.7130 | LR: 4.07e-04 | 7601.9ms/step | 2.6 samples/s +Step 3860/11700 | Loss: 2.6326 | LR: 4.07e-04 | 7601.7ms/step | 2.6 samples/s +Step 3870/11700 | Loss: 2.6648 | LR: 4.06e-04 | 7600.2ms/step | 2.6 samples/s +Step 3880/11700 | Loss: 2.5469 | LR: 4.06e-04 | 7600.3ms/step | 2.6 samples/s +Step 3890/11700 | Loss: 2.7198 | LR: 4.05e-04 | 7599.9ms/step | 2.6 samples/s +Step 3900/11700 | Loss: 2.5934 | LR: 4.05e-04 | 7599.8ms/step | 2.6 samples/s +Step 3910/11700 | Loss: 2.6807 | LR: 4.04e-04 | 7600.1ms/step | 2.6 samples/s +Step 3920/11700 | Loss: 2.5899 | LR: 4.04e-04 | 7602.2ms/step | 2.6 samples/s +Step 3930/11700 | Loss: 2.6513 | LR: 4.03e-04 | 7600.4ms/step | 2.6 samples/s +Step 3940/11700 | Loss: 2.7888 | LR: 4.02e-04 | 7601.5ms/step | 2.6 samples/s +Step 3950/11700 | Loss: 2.6122 | LR: 4.02e-04 | 7602.2ms/step | 2.6 samples/s +Step 3960/11700 | Loss: 2.6588 | LR: 4.01e-04 | 7600.3ms/step | 2.6 samples/s +Step 3970/11700 | Loss: 2.6418 | LR: 4.01e-04 | 7603.9ms/step | 2.6 samples/s +Step 3980/11700 | Loss: 2.7049 | LR: 4.00e-04 | 7604.7ms/step | 2.6 samples/s +Step 3990/11700 | Loss: 2.6874 | LR: 4.00e-04 | 7603.4ms/step | 2.6 samples/s +Step 4000/11700 | Loss: 2.7379 | LR: 3.99e-04 | 7603.9ms/step | 2.6 samples/s +Validation at step 4000: Loss = 2.6632, Perplexity = 14.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-4000 +Step 4010/11700 | Loss: 2.5568 | LR: 3.98e-04 | 19278.4ms/step | 1.0 samples/s +Step 4020/11700 | Loss: 2.6559 | LR: 3.98e-04 | 7612.1ms/step | 2.6 samples/s +Step 4030/11700 | Loss: 2.5482 | LR: 3.97e-04 | 7613.7ms/step | 2.6 samples/s +Step 4040/11700 | Loss: 2.6874 | LR: 3.97e-04 | 7610.2ms/step | 2.6 samples/s +Step 4050/11700 | Loss: 2.6820 | LR: 3.96e-04 | 7610.8ms/step | 2.6 samples/s +Step 4060/11700 | Loss: 2.5577 | LR: 3.95e-04 | 7615.0ms/step | 2.6 samples/s +Step 4070/11700 | Loss: 2.7379 | LR: 3.95e-04 | 7616.5ms/step | 2.6 samples/s +Step 4080/11700 | Loss: 2.6271 | LR: 3.94e-04 | 7609.7ms/step | 2.6 samples/s +Step 4090/11700 | Loss: 2.6038 | LR: 3.94e-04 | 7612.3ms/step | 2.6 samples/s +Step 4100/11700 | Loss: 2.6451 | LR: 3.93e-04 | 7615.7ms/step | 2.6 samples/s +Step 4110/11700 | Loss: 2.6288 | LR: 3.93e-04 | 7610.4ms/step | 2.6 samples/s +Step 4120/11700 | Loss: 2.6967 | LR: 3.92e-04 | 7606.2ms/step | 2.6 samples/s +Step 4130/11700 | Loss: 2.5616 | LR: 3.91e-04 | 7619.8ms/step | 2.6 samples/s +Step 4140/11700 | Loss: 2.6132 | LR: 3.91e-04 | 7652.1ms/step | 2.6 samples/s +Step 4150/11700 | Loss: 2.5544 | LR: 3.90e-04 | 7618.1ms/step | 2.6 samples/s +Step 4160/11700 | Loss: 2.6283 | LR: 3.90e-04 | 7617.9ms/step | 2.6 samples/s +Step 4170/11700 | Loss: 2.6882 | LR: 3.89e-04 | 7616.1ms/step | 2.6 samples/s +Step 4180/11700 | Loss: 2.6284 | LR: 3.88e-04 | 7616.2ms/step | 2.6 samples/s +Step 4190/11700 | Loss: 2.6597 | LR: 3.88e-04 | 7626.3ms/step | 2.6 samples/s +Step 4200/11700 | Loss: 2.5740 | LR: 3.87e-04 | 7617.9ms/step | 2.6 samples/s +Step 4210/11700 | Loss: 2.6776 | LR: 3.87e-04 | 7615.7ms/step | 2.6 samples/s +Step 4220/11700 | Loss: 2.6415 | LR: 3.86e-04 | 7614.1ms/step | 2.6 samples/s +Step 4230/11700 | Loss: 2.5606 | LR: 3.85e-04 | 7615.4ms/step | 2.6 samples/s +Step 4240/11700 | Loss: 2.6703 | LR: 3.85e-04 | 7612.7ms/step | 2.6 samples/s +Step 4250/11700 | Loss: 2.7707 | LR: 3.84e-04 | 7615.7ms/step | 2.6 samples/s +Validation at step 4250: Loss = 2.6421, Perplexity = 14.04 +Step 4260/11700 | Loss: 2.7367 | LR: 3.84e-04 | 18952.5ms/step | 1.1 samples/s +Step 4270/11700 | Loss: 2.5886 | LR: 3.83e-04 | 7614.1ms/step | 2.6 samples/s +Step 4280/11700 | Loss: 2.6220 | LR: 3.82e-04 | 7612.4ms/step | 2.6 samples/s +Step 4290/11700 | Loss: 2.6274 | LR: 3.82e-04 | 7611.8ms/step | 2.6 samples/s +Step 4300/11700 | Loss: 2.6818 | LR: 3.81e-04 | 7613.6ms/step | 2.6 samples/s +Step 4310/11700 | Loss: 2.6037 | LR: 3.81e-04 | 7612.2ms/step | 2.6 samples/s +Step 4320/11700 | Loss: 2.5811 | LR: 3.80e-04 | 7612.1ms/step | 2.6 samples/s +Step 4330/11700 | Loss: 2.5921 | LR: 3.79e-04 | 7610.7ms/step | 2.6 samples/s +Step 4340/11700 | Loss: 2.6630 | LR: 3.79e-04 | 7610.2ms/step | 2.6 samples/s +Step 4350/11700 | Loss: 2.7361 | LR: 3.78e-04 | 7611.3ms/step | 2.6 samples/s +Step 4360/11700 | Loss: 2.5722 | LR: 3.77e-04 | 7610.0ms/step | 2.6 samples/s +Step 4370/11700 | Loss: 2.6058 | LR: 3.77e-04 | 7612.2ms/step | 2.6 samples/s +Step 4380/11700 | Loss: 2.5664 | LR: 3.76e-04 | 7614.0ms/step | 2.6 samples/s +Step 4390/11700 | Loss: 2.6413 | LR: 3.76e-04 | 7616.1ms/step | 2.6 samples/s +Step 4400/11700 | Loss: 2.6297 | LR: 3.75e-04 | 7613.6ms/step | 2.6 samples/s +Step 4410/11700 | Loss: 2.6474 | LR: 3.74e-04 | 7612.5ms/step | 2.6 samples/s +Step 4420/11700 | Loss: 2.6363 | LR: 3.74e-04 | 7614.0ms/step | 2.6 samples/s +Step 4430/11700 | Loss: 2.6625 | LR: 3.73e-04 | 7620.2ms/step | 2.6 samples/s +Step 4440/11700 | Loss: 2.6639 | LR: 3.73e-04 | 7618.3ms/step | 2.6 samples/s +Step 4450/11700 | Loss: 2.6055 | LR: 3.72e-04 | 7614.1ms/step | 2.6 samples/s +Step 4460/11700 | Loss: 2.7131 | LR: 3.71e-04 | 7615.8ms/step | 2.6 samples/s +Step 4470/11700 | Loss: 2.5277 | LR: 3.71e-04 | 7616.2ms/step | 2.6 samples/s +Step 4480/11700 | Loss: 2.5976 | LR: 3.70e-04 | 7613.9ms/step | 2.6 samples/s +Step 4490/11700 | Loss: 2.6095 | LR: 3.69e-04 | 7610.0ms/step | 2.6 samples/s +Step 4500/11700 | Loss: 2.6247 | LR: 3.69e-04 | 7611.8ms/step | 2.6 samples/s +Validation at step 4500: Loss = 2.6261, Perplexity = 13.82 +Step 4510/11700 | Loss: 2.5218 | LR: 3.68e-04 | 18949.1ms/step | 1.1 samples/s +Step 4520/11700 | Loss: 2.6445 | LR: 3.67e-04 | 7611.2ms/step | 2.6 samples/s +Step 4530/11700 | Loss: 2.7395 | LR: 3.67e-04 | 7610.3ms/step | 2.6 samples/s +Step 4540/11700 | Loss: 2.6643 | LR: 3.66e-04 | 7612.1ms/step | 2.6 samples/s +Step 4550/11700 | Loss: 2.5990 | LR: 3.66e-04 | 7611.9ms/step | 2.6 samples/s +Step 4560/11700 | Loss: 2.5714 | LR: 3.65e-04 | 7616.6ms/step | 2.6 samples/s +Step 4570/11700 | Loss: 2.5919 | LR: 3.64e-04 | 7613.8ms/step | 2.6 samples/s +Step 4580/11700 | Loss: 2.5569 | LR: 3.64e-04 | 7615.6ms/step | 2.6 samples/s +Step 4590/11700 | Loss: 2.7075 | LR: 3.63e-04 | 7613.6ms/step | 2.6 samples/s +Step 4600/11700 | Loss: 2.4525 | LR: 3.62e-04 | 7616.7ms/step | 2.6 samples/s +Step 4610/11700 | Loss: 2.5079 | LR: 3.62e-04 | 7614.5ms/step | 2.6 samples/s +Step 4620/11700 | Loss: 2.6193 | LR: 3.61e-04 | 7611.0ms/step | 2.6 samples/s +Step 4630/11700 | Loss: 2.6276 | LR: 3.60e-04 | 7612.4ms/step | 2.6 samples/s +Step 4640/11700 | Loss: 2.5736 | LR: 3.60e-04 | 7608.3ms/step | 2.6 samples/s +Step 4650/11700 | Loss: 2.5727 | LR: 3.59e-04 | 7613.9ms/step | 2.6 samples/s +Step 4660/11700 | Loss: 2.6427 | LR: 3.59e-04 | 7616.4ms/step | 2.6 samples/s +Step 4670/11700 | Loss: 2.5595 | LR: 3.58e-04 | 7618.0ms/step | 2.6 samples/s +Step 4680/11700 | Loss: 2.6462 | LR: 3.57e-04 | 7616.5ms/step | 2.6 samples/s +Step 4690/11700 | Loss: 2.5448 | LR: 3.57e-04 | 7613.3ms/step | 2.6 samples/s +Step 4700/11700 | Loss: 2.5456 | LR: 3.56e-04 | 7628.1ms/step | 2.6 samples/s +Step 4710/11700 | Loss: 2.7032 | LR: 3.55e-04 | 7620.7ms/step | 2.6 samples/s +Step 4720/11700 | Loss: 2.5688 | LR: 3.55e-04 | 7615.6ms/step | 2.6 samples/s +Step 4730/11700 | Loss: 2.6316 | LR: 3.54e-04 | 7613.7ms/step | 2.6 samples/s +Step 4740/11700 | Loss: 2.6416 | LR: 3.53e-04 | 7612.0ms/step | 2.6 samples/s +Step 4750/11700 | Loss: 2.5873 | LR: 3.53e-04 | 7614.5ms/step | 2.6 samples/s +Validation at step 4750: Loss = 2.6044, Perplexity = 13.52 +Step 4760/11700 | Loss: 2.6811 | LR: 3.52e-04 | 18960.5ms/step | 1.1 samples/s +Step 4770/11700 | Loss: 2.6368 | LR: 3.51e-04 | 7615.7ms/step | 2.6 samples/s +Step 4780/11700 | Loss: 2.6034 | LR: 3.51e-04 | 7613.9ms/step | 2.6 samples/s +Step 4790/11700 | Loss: 2.5793 | LR: 3.50e-04 | 7613.7ms/step | 2.6 samples/s +Step 4800/11700 | Loss: 2.5218 | LR: 3.49e-04 | 7614.6ms/step | 2.6 samples/s +Step 4810/11700 | Loss: 2.4933 | LR: 3.49e-04 | 7620.1ms/step | 2.6 samples/s +Step 4820/11700 | Loss: 2.5955 | LR: 3.48e-04 | 7618.0ms/step | 2.6 samples/s +Step 4830/11700 | Loss: 2.6151 | LR: 3.47e-04 | 7616.0ms/step | 2.6 samples/s +Step 4840/11700 | Loss: 2.6124 | LR: 3.47e-04 | 7612.1ms/step | 2.6 samples/s +Step 4850/11700 | Loss: 2.6185 | LR: 3.46e-04 | 7609.7ms/step | 2.6 samples/s +Step 4860/11700 | Loss: 2.5633 | LR: 3.45e-04 | 7610.9ms/step | 2.6 samples/s +Step 4870/11700 | Loss: 2.6896 | LR: 3.45e-04 | 7613.6ms/step | 2.6 samples/s +Step 4880/11700 | Loss: 2.6185 | LR: 3.44e-04 | 7616.0ms/step | 2.6 samples/s +Step 4890/11700 | Loss: 2.6802 | LR: 3.43e-04 | 7615.8ms/step | 2.6 samples/s +Step 4900/11700 | Loss: 2.6200 | LR: 3.43e-04 | 7613.3ms/step | 2.6 samples/s +Step 4910/11700 | Loss: 2.5694 | LR: 3.42e-04 | 7612.7ms/step | 2.6 samples/s +Step 4920/11700 | Loss: 2.5422 | LR: 3.41e-04 | 7614.5ms/step | 2.6 samples/s +Step 4930/11700 | Loss: 2.5080 | LR: 3.41e-04 | 7613.6ms/step | 2.6 samples/s +Step 4940/11700 | Loss: 2.6328 | LR: 3.40e-04 | 7607.2ms/step | 2.6 samples/s +Step 4950/11700 | Loss: 2.4516 | LR: 3.39e-04 | 7615.2ms/step | 2.6 samples/s +Step 4960/11700 | Loss: 2.6128 | LR: 3.39e-04 | 7614.4ms/step | 2.6 samples/s +Step 4970/11700 | Loss: 2.5995 | LR: 3.38e-04 | 7615.5ms/step | 2.6 samples/s +Step 4980/11700 | Loss: 2.7101 | LR: 3.37e-04 | 7612.4ms/step | 2.6 samples/s +Step 4990/11700 | Loss: 2.5288 | LR: 3.37e-04 | 7613.8ms/step | 2.6 samples/s +Step 5000/11700 | Loss: 2.6903 | LR: 3.36e-04 | 7614.7ms/step | 2.6 samples/s +Validation at step 5000: Loss = 2.5881, Perplexity = 13.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-5000 +Step 5010/11700 | Loss: 2.6881 | LR: 3.35e-04 | 19253.6ms/step | 1.0 samples/s +Step 5020/11700 | Loss: 2.6719 | LR: 3.35e-04 | 7608.5ms/step | 2.6 samples/s +Step 5030/11700 | Loss: 2.6653 | LR: 3.34e-04 | 7611.7ms/step | 2.6 samples/s +Step 5040/11700 | Loss: 2.4995 | LR: 3.33e-04 | 7611.9ms/step | 2.6 samples/s +Step 5050/11700 | Loss: 2.4390 | LR: 3.33e-04 | 7613.9ms/step | 2.6 samples/s +Step 5060/11700 | Loss: 2.6079 | LR: 3.32e-04 | 7617.5ms/step | 2.6 samples/s +Step 5070/11700 | Loss: 2.5679 | LR: 3.31e-04 | 7618.9ms/step | 2.6 samples/s +Step 5080/11700 | Loss: 2.7003 | LR: 3.31e-04 | 7621.9ms/step | 2.6 samples/s +Step 5090/11700 | Loss: 2.6001 | LR: 3.30e-04 | 7609.9ms/step | 2.6 samples/s +Step 5100/11700 | Loss: 2.6997 | LR: 3.29e-04 | 7610.4ms/step | 2.6 samples/s +Step 5110/11700 | Loss: 2.5994 | LR: 3.29e-04 | 7613.7ms/step | 2.6 samples/s +Step 5120/11700 | Loss: 2.5507 | LR: 3.28e-04 | 7613.9ms/step | 2.6 samples/s +Step 5130/11700 | Loss: 2.4661 | LR: 3.27e-04 | 7620.5ms/step | 2.6 samples/s +Step 5140/11700 | Loss: 2.5179 | LR: 3.27e-04 | 7614.0ms/step | 2.6 samples/s +Step 5150/11700 | Loss: 2.5856 | LR: 3.26e-04 | 7608.4ms/step | 2.6 samples/s +Step 5160/11700 | Loss: 2.5308 | LR: 3.25e-04 | 7613.7ms/step | 2.6 samples/s +Step 5170/11700 | Loss: 2.5376 | LR: 3.25e-04 | 7608.9ms/step | 2.6 samples/s +Step 5180/11700 | Loss: 2.5178 | LR: 3.24e-04 | 7613.8ms/step | 2.6 samples/s +Step 5190/11700 | Loss: 2.5916 | LR: 3.23e-04 | 7611.6ms/step | 2.6 samples/s +Step 5200/11700 | Loss: 2.4277 | LR: 3.22e-04 | 7609.6ms/step | 2.6 samples/s +Step 5210/11700 | Loss: 2.5445 | LR: 3.22e-04 | 7608.2ms/step | 2.6 samples/s +Step 5220/11700 | Loss: 2.5988 | LR: 3.21e-04 | 7606.7ms/step | 2.6 samples/s +Step 5230/11700 | Loss: 2.6751 | LR: 3.20e-04 | 7611.4ms/step | 2.6 samples/s +Step 5240/11700 | Loss: 2.5496 | LR: 3.20e-04 | 7612.2ms/step | 2.6 samples/s +Step 5250/11700 | Loss: 2.5894 | LR: 3.19e-04 | 7615.8ms/step | 2.6 samples/s +Validation at step 5250: Loss = 2.5715, Perplexity = 13.09 +Step 5260/11700 | Loss: 2.5818 | LR: 3.18e-04 | 18951.6ms/step | 1.1 samples/s +Step 5270/11700 | Loss: 2.5617 | LR: 3.18e-04 | 7616.6ms/step | 2.6 samples/s +Step 5280/11700 | Loss: 2.5183 | LR: 3.17e-04 | 7615.1ms/step | 2.6 samples/s +Step 5290/11700 | Loss: 2.5248 | LR: 3.16e-04 | 7613.1ms/step | 2.6 samples/s +Step 5300/11700 | Loss: 2.5409 | LR: 3.16e-04 | 7610.3ms/step | 2.6 samples/s +Step 5310/11700 | Loss: 2.5336 | LR: 3.15e-04 | 7612.8ms/step | 2.6 samples/s +Step 5320/11700 | Loss: 2.5087 | LR: 3.14e-04 | 7611.1ms/step | 2.6 samples/s +Step 5330/11700 | Loss: 2.4975 | LR: 3.13e-04 | 7611.6ms/step | 2.6 samples/s +Step 5340/11700 | Loss: 2.5320 | LR: 3.13e-04 | 7610.2ms/step | 2.6 samples/s +Step 5350/11700 | Loss: 2.5301 | LR: 3.12e-04 | 7613.9ms/step | 2.6 samples/s +Step 5360/11700 | Loss: 2.4581 | LR: 3.11e-04 | 7610.6ms/step | 2.6 samples/s +Step 5370/11700 | Loss: 2.5634 | LR: 3.11e-04 | 7610.5ms/step | 2.6 samples/s +Step 5380/11700 | Loss: 2.4609 | LR: 3.10e-04 | 7611.5ms/step | 2.6 samples/s +Step 5390/11700 | Loss: 2.6007 | LR: 3.09e-04 | 7611.8ms/step | 2.6 samples/s +Step 5400/11700 | Loss: 2.5092 | LR: 3.09e-04 | 7610.5ms/step | 2.6 samples/s +Step 5410/11700 | Loss: 2.4945 | LR: 3.08e-04 | 7607.9ms/step | 2.6 samples/s +Step 5420/11700 | Loss: 2.5610 | LR: 3.07e-04 | 7612.0ms/step | 2.6 samples/s +Step 5430/11700 | Loss: 2.6332 | LR: 3.07e-04 | 7612.4ms/step | 2.6 samples/s +Step 5440/11700 | Loss: 2.6385 | LR: 3.06e-04 | 7613.6ms/step | 2.6 samples/s +Step 5450/11700 | Loss: 2.5227 | LR: 3.05e-04 | 7613.9ms/step | 2.6 samples/s +Step 5460/11700 | Loss: 2.5622 | LR: 3.04e-04 | 7610.8ms/step | 2.6 samples/s +Step 5470/11700 | Loss: 2.4257 | LR: 3.04e-04 | 7613.6ms/step | 2.6 samples/s +Step 5480/11700 | Loss: 2.5297 | LR: 3.03e-04 | 7610.9ms/step | 2.6 samples/s +Step 5490/11700 | Loss: 2.6934 | LR: 3.02e-04 | 7613.1ms/step | 2.6 samples/s +Step 5500/11700 | Loss: 2.5976 | LR: 3.02e-04 | 7610.0ms/step | 2.6 samples/s +Validation at step 5500: Loss = 2.5588, Perplexity = 12.92 +Step 5510/11700 | Loss: 2.4423 | LR: 3.01e-04 | 18947.6ms/step | 1.1 samples/s +Step 5520/11700 | Loss: 2.4193 | LR: 3.00e-04 | 7610.8ms/step | 2.6 samples/s +Step 5530/11700 | Loss: 2.5742 | LR: 3.00e-04 | 7608.0ms/step | 2.6 samples/s +Step 5540/11700 | Loss: 2.5540 | LR: 2.99e-04 | 7610.2ms/step | 2.6 samples/s +Step 5550/11700 | Loss: 2.5597 | LR: 2.98e-04 | 7611.7ms/step | 2.6 samples/s +Step 5560/11700 | Loss: 2.6001 | LR: 2.97e-04 | 7610.3ms/step | 2.6 samples/s +Step 5570/11700 | Loss: 2.4737 | LR: 2.97e-04 | 7611.8ms/step | 2.6 samples/s +Step 5580/11700 | Loss: 2.5655 | LR: 2.96e-04 | 7613.4ms/step | 2.6 samples/s +Step 5590/11700 | Loss: 2.5908 | LR: 2.95e-04 | 7611.0ms/step | 2.6 samples/s +Step 5600/11700 | Loss: 2.5365 | LR: 2.95e-04 | 7609.8ms/step | 2.6 samples/s +Step 5610/11700 | Loss: 2.3945 | LR: 2.94e-04 | 7613.4ms/step | 2.6 samples/s +Step 5620/11700 | Loss: 2.5542 | LR: 2.93e-04 | 7608.6ms/step | 2.6 samples/s +Step 5630/11700 | Loss: 2.4460 | LR: 2.92e-04 | 7607.6ms/step | 2.6 samples/s +Step 5640/11700 | Loss: 2.5780 | LR: 2.92e-04 | 7606.5ms/step | 2.6 samples/s +Step 5650/11700 | Loss: 2.6303 | LR: 2.91e-04 | 7608.5ms/step | 2.6 samples/s +Step 5660/11700 | Loss: 2.5373 | LR: 2.90e-04 | 7614.2ms/step | 2.6 samples/s +Step 5670/11700 | Loss: 2.5207 | LR: 2.90e-04 | 7613.6ms/step | 2.6 samples/s +Step 5680/11700 | Loss: 2.5938 | LR: 2.89e-04 | 7612.2ms/step | 2.6 samples/s +Step 5690/11700 | Loss: 2.5069 | LR: 2.88e-04 | 7617.7ms/step | 2.6 samples/s +Step 5700/11700 | Loss: 2.5292 | LR: 2.88e-04 | 7614.3ms/step | 2.6 samples/s +Step 5710/11700 | Loss: 2.5033 | LR: 2.87e-04 | 7613.4ms/step | 2.6 samples/s +Step 5720/11700 | Loss: 2.4810 | LR: 2.86e-04 | 7615.3ms/step | 2.6 samples/s +Step 5730/11700 | Loss: 2.5955 | LR: 2.85e-04 | 7614.3ms/step | 2.6 samples/s +Step 5740/11700 | Loss: 2.4630 | LR: 2.85e-04 | 7611.3ms/step | 2.6 samples/s +Step 5750/11700 | Loss: 2.4985 | LR: 2.84e-04 | 7614.4ms/step | 2.6 samples/s +Validation at step 5750: Loss = 2.5464, Perplexity = 12.76 +Step 5760/11700 | Loss: 2.5372 | LR: 2.83e-04 | 18956.0ms/step | 1.1 samples/s +Step 5770/11700 | Loss: 2.6660 | LR: 2.83e-04 | 7612.8ms/step | 2.6 samples/s +Step 5780/11700 | Loss: 2.5048 | LR: 2.82e-04 | 7611.0ms/step | 2.6 samples/s +Step 5790/11700 | Loss: 2.6276 | LR: 2.81e-04 | 7614.8ms/step | 2.6 samples/s +Step 5800/11700 | Loss: 2.5949 | LR: 2.80e-04 | 7614.8ms/step | 2.6 samples/s +Step 5810/11700 | Loss: 2.5684 | LR: 2.80e-04 | 7610.6ms/step | 2.6 samples/s +Step 5820/11700 | Loss: 2.4634 | LR: 2.79e-04 | 7611.6ms/step | 2.6 samples/s +Step 5830/11700 | Loss: 2.5140 | LR: 2.78e-04 | 7618.2ms/step | 2.6 samples/s +Step 5840/11700 | Loss: 2.5053 | LR: 2.78e-04 | 7610.2ms/step | 2.6 samples/s +Step 5850/11700 | Loss: 2.5132 | LR: 2.77e-04 | 7608.2ms/step | 2.6 samples/s +Step 5860/11700 | Loss: 2.5853 | LR: 2.76e-04 | 7610.2ms/step | 2.6 samples/s +Step 5870/11700 | Loss: 2.5949 | LR: 2.75e-04 | 7612.2ms/step | 2.6 samples/s +Step 5880/11700 | Loss: 2.5736 | LR: 2.75e-04 | 7609.7ms/step | 2.6 samples/s +Step 5890/11700 | Loss: 2.5102 | LR: 2.74e-04 | 7610.0ms/step | 2.6 samples/s +Step 5900/11700 | Loss: 2.5212 | LR: 2.73e-04 | 7610.1ms/step | 2.6 samples/s +Step 5910/11700 | Loss: 2.5679 | LR: 2.73e-04 | 7605.6ms/step | 2.6 samples/s +Step 5920/11700 | Loss: 2.4775 | LR: 2.72e-04 | 7606.6ms/step | 2.6 samples/s +Step 5930/11700 | Loss: 2.5086 | LR: 2.71e-04 | 7605.9ms/step | 2.6 samples/s +Step 5940/11700 | Loss: 2.4383 | LR: 2.70e-04 | 7610.0ms/step | 2.6 samples/s +Step 5950/11700 | Loss: 2.6060 | LR: 2.70e-04 | 7606.2ms/step | 2.6 samples/s +Step 5960/11700 | Loss: 2.6309 | LR: 2.69e-04 | 7608.3ms/step | 2.6 samples/s +Step 5970/11700 | Loss: 2.5211 | LR: 2.68e-04 | 7606.2ms/step | 2.6 samples/s +Step 5980/11700 | Loss: 2.5405 | LR: 2.68e-04 | 7610.2ms/step | 2.6 samples/s +Step 5990/11700 | Loss: 2.5309 | LR: 2.67e-04 | 7611.4ms/step | 2.6 samples/s +Step 6000/11700 | Loss: 2.5256 | LR: 2.66e-04 | 7608.2ms/step | 2.6 samples/s +Validation at step 6000: Loss = 2.5346, Perplexity = 12.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-6000 +Step 6010/11700 | Loss: 2.5801 | LR: 2.65e-04 | 19240.2ms/step | 1.0 samples/s +Step 6020/11700 | Loss: 2.4423 | LR: 2.65e-04 | 7604.7ms/step | 2.6 samples/s +Step 6030/11700 | Loss: 2.5027 | LR: 2.64e-04 | 7611.0ms/step | 2.6 samples/s +Step 6040/11700 | Loss: 2.4667 | LR: 2.63e-04 | 7609.2ms/step | 2.6 samples/s +Step 6050/11700 | Loss: 2.4700 | LR: 2.63e-04 | 7613.4ms/step | 2.6 samples/s +Step 6060/11700 | Loss: 2.5714 | LR: 2.62e-04 | 7616.1ms/step | 2.6 samples/s +Step 6070/11700 | Loss: 2.5628 | LR: 2.61e-04 | 7617.8ms/step | 2.6 samples/s +Step 6080/11700 | Loss: 2.5917 | LR: 2.60e-04 | 7606.3ms/step | 2.6 samples/s +Step 6090/11700 | Loss: 2.4602 | LR: 2.60e-04 | 7609.8ms/step | 2.6 samples/s +Step 6100/11700 | Loss: 2.5336 | LR: 2.59e-04 | 7607.7ms/step | 2.6 samples/s +Step 6110/11700 | Loss: 2.4669 | LR: 2.58e-04 | 7604.4ms/step | 2.6 samples/s +Step 6120/11700 | Loss: 2.5934 | LR: 2.58e-04 | 7604.2ms/step | 2.6 samples/s +Step 6130/11700 | Loss: 2.5225 | LR: 2.57e-04 | 7603.7ms/step | 2.6 samples/s +Step 6140/11700 | Loss: 2.5335 | LR: 2.56e-04 | 7614.6ms/step | 2.6 samples/s +Step 6150/11700 | Loss: 2.5068 | LR: 2.55e-04 | 7609.6ms/step | 2.6 samples/s +Step 6160/11700 | Loss: 2.4444 | LR: 2.55e-04 | 7606.8ms/step | 2.6 samples/s +Step 6170/11700 | Loss: 2.4552 | LR: 2.54e-04 | 7607.4ms/step | 2.6 samples/s +Step 6180/11700 | Loss: 2.4243 | LR: 2.53e-04 | 7603.7ms/step | 2.6 samples/s +Step 6190/11700 | Loss: 2.4845 | LR: 2.53e-04 | 7604.4ms/step | 2.6 samples/s +Step 6200/11700 | Loss: 2.6105 | LR: 2.52e-04 | 7606.3ms/step | 2.6 samples/s +Step 6210/11700 | Loss: 2.5940 | LR: 2.51e-04 | 7609.3ms/step | 2.6 samples/s +Step 6220/11700 | Loss: 2.5409 | LR: 2.50e-04 | 7608.2ms/step | 2.6 samples/s +Step 6230/11700 | Loss: 2.6486 | LR: 2.50e-04 | 7609.1ms/step | 2.6 samples/s +Step 6240/11700 | Loss: 2.5075 | LR: 2.49e-04 | 7610.4ms/step | 2.6 samples/s +Step 6250/11700 | Loss: 2.5440 | LR: 2.48e-04 | 7609.1ms/step | 2.6 samples/s +Validation at step 6250: Loss = 2.5223, Perplexity = 12.46 +Step 6260/11700 | Loss: 2.4386 | LR: 2.47e-04 | 18944.2ms/step | 1.1 samples/s +Step 6270/11700 | Loss: 2.5351 | LR: 2.47e-04 | 7610.0ms/step | 2.6 samples/s +Step 6280/11700 | Loss: 2.5141 | LR: 2.46e-04 | 7605.9ms/step | 2.6 samples/s +Step 6290/11700 | Loss: 2.4930 | LR: 2.45e-04 | 7604.4ms/step | 2.6 samples/s +Step 6300/11700 | Loss: 2.5017 | LR: 2.45e-04 | 7605.2ms/step | 2.6 samples/s +Step 6310/11700 | Loss: 2.4942 | LR: 2.44e-04 | 7600.7ms/step | 2.6 samples/s +Step 6320/11700 | Loss: 2.5595 | LR: 2.43e-04 | 7603.8ms/step | 2.6 samples/s +Step 6330/11700 | Loss: 2.6230 | LR: 2.42e-04 | 7606.1ms/step | 2.6 samples/s +Step 6340/11700 | Loss: 2.5357 | LR: 2.42e-04 | 7607.9ms/step | 2.6 samples/s +Step 6350/11700 | Loss: 2.5446 | LR: 2.41e-04 | 7612.2ms/step | 2.6 samples/s +Step 6360/11700 | Loss: 2.4417 | LR: 2.40e-04 | 7607.9ms/step | 2.6 samples/s +Step 6370/11700 | Loss: 2.4258 | LR: 2.40e-04 | 7612.2ms/step | 2.6 samples/s +Step 6380/11700 | Loss: 2.3967 | LR: 2.39e-04 | 7617.9ms/step | 2.6 samples/s +Step 6390/11700 | Loss: 2.5286 | LR: 2.38e-04 | 7614.7ms/step | 2.6 samples/s +Step 6400/11700 | Loss: 2.5045 | LR: 2.37e-04 | 7609.5ms/step | 2.6 samples/s +Step 6410/11700 | Loss: 2.4336 | LR: 2.37e-04 | 7608.1ms/step | 2.6 samples/s +Step 6420/11700 | Loss: 2.4319 | LR: 2.36e-04 | 7610.4ms/step | 2.6 samples/s +Step 6430/11700 | Loss: 2.3275 | LR: 2.35e-04 | 7610.5ms/step | 2.6 samples/s +Step 6440/11700 | Loss: 2.4991 | LR: 2.35e-04 | 7609.7ms/step | 2.6 samples/s +Step 6450/11700 | Loss: 2.4399 | LR: 2.34e-04 | 7608.1ms/step | 2.6 samples/s +Step 6460/11700 | Loss: 2.5711 | LR: 2.33e-04 | 7609.5ms/step | 2.6 samples/s +Step 6470/11700 | Loss: 2.4356 | LR: 2.32e-04 | 7610.3ms/step | 2.6 samples/s +Step 6480/11700 | Loss: 2.6864 | LR: 2.32e-04 | 7605.7ms/step | 2.6 samples/s +Step 6490/11700 | Loss: 2.4923 | LR: 2.31e-04 | 7607.9ms/step | 2.6 samples/s +Step 6500/11700 | Loss: 2.5258 | LR: 2.30e-04 | 7600.5ms/step | 2.6 samples/s +Validation at step 6500: Loss = 2.5129, Perplexity = 12.34 +Step 6510/11700 | Loss: 2.5046 | LR: 2.30e-04 | 18946.0ms/step | 1.1 samples/s +Step 6520/11700 | Loss: 2.5058 | LR: 2.29e-04 | 7606.2ms/step | 2.6 samples/s +Step 6530/11700 | Loss: 2.4228 | LR: 2.28e-04 | 7607.9ms/step | 2.6 samples/s +Step 6540/11700 | Loss: 2.5749 | LR: 2.27e-04 | 7608.2ms/step | 2.6 samples/s +Step 6550/11700 | Loss: 2.4751 | LR: 2.27e-04 | 7608.3ms/step | 2.6 samples/s +Step 6560/11700 | Loss: 2.4266 | LR: 2.26e-04 | 7607.6ms/step | 2.6 samples/s +Step 6570/11700 | Loss: 2.3825 | LR: 2.25e-04 | 7604.2ms/step | 2.6 samples/s +Step 6580/11700 | Loss: 2.5420 | LR: 2.25e-04 | 7609.9ms/step | 2.6 samples/s +Step 6590/11700 | Loss: 2.5226 | LR: 2.24e-04 | 7612.7ms/step | 2.6 samples/s +Step 6600/11700 | Loss: 2.5358 | LR: 2.23e-04 | 7610.5ms/step | 2.6 samples/s +Step 6610/11700 | Loss: 2.6076 | LR: 2.22e-04 | 7606.7ms/step | 2.6 samples/s +Step 6620/11700 | Loss: 2.4078 | LR: 2.22e-04 | 7610.6ms/step | 2.6 samples/s +Step 6630/11700 | Loss: 2.5149 | LR: 2.21e-04 | 7612.1ms/step | 2.6 samples/s +Step 6640/11700 | Loss: 2.5226 | LR: 2.20e-04 | 7611.9ms/step | 2.6 samples/s +Step 6650/11700 | Loss: 2.4554 | LR: 2.20e-04 | 7610.1ms/step | 2.6 samples/s +Step 6660/11700 | Loss: 2.4788 | LR: 2.19e-04 | 7612.5ms/step | 2.6 samples/s +Step 6670/11700 | Loss: 2.4729 | LR: 2.18e-04 | 7607.7ms/step | 2.6 samples/s +Step 6680/11700 | Loss: 2.4699 | LR: 2.17e-04 | 7613.3ms/step | 2.6 samples/s +Step 6690/11700 | Loss: 2.5599 | LR: 2.17e-04 | 7608.4ms/step | 2.6 samples/s +Step 6700/11700 | Loss: 2.4988 | LR: 2.16e-04 | 7608.2ms/step | 2.6 samples/s +Step 6710/11700 | Loss: 2.5112 | LR: 2.15e-04 | 7607.6ms/step | 2.6 samples/s +Step 6720/11700 | Loss: 2.3947 | LR: 2.15e-04 | 7600.6ms/step | 2.6 samples/s +Step 6730/11700 | Loss: 2.4528 | LR: 2.14e-04 | 7604.1ms/step | 2.6 samples/s +Step 6740/11700 | Loss: 2.4702 | LR: 2.13e-04 | 7608.0ms/step | 2.6 samples/s +Step 6750/11700 | Loss: 2.4526 | LR: 2.12e-04 | 7608.3ms/step | 2.6 samples/s +Validation at step 6750: Loss = 2.5040, Perplexity = 12.23 +Step 6760/11700 | Loss: 2.5270 | LR: 2.12e-04 | 18949.7ms/step | 1.1 samples/s +Step 6770/11700 | Loss: 2.4926 | LR: 2.11e-04 | 7614.3ms/step | 2.6 samples/s +Step 6780/11700 | Loss: 2.4326 | LR: 2.10e-04 | 7613.5ms/step | 2.6 samples/s +Step 6790/11700 | Loss: 2.4363 | LR: 2.10e-04 | 7614.3ms/step | 2.6 samples/s +Step 6800/11700 | Loss: 2.5190 | LR: 2.09e-04 | 7616.1ms/step | 2.6 samples/s +Step 6810/11700 | Loss: 2.6866 | LR: 2.08e-04 | 7612.3ms/step | 2.6 samples/s +Step 6820/11700 | Loss: 2.5277 | LR: 2.08e-04 | 7614.5ms/step | 2.6 samples/s +Step 6830/11700 | Loss: 2.4387 | LR: 2.07e-04 | 7611.5ms/step | 2.6 samples/s +Step 6840/11700 | Loss: 2.4580 | LR: 2.06e-04 | 7608.3ms/step | 2.6 samples/s +Step 6850/11700 | Loss: 2.5311 | LR: 2.05e-04 | 7611.9ms/step | 2.6 samples/s +Step 6860/11700 | Loss: 2.4716 | LR: 2.05e-04 | 7620.3ms/step | 2.6 samples/s +Step 6870/11700 | Loss: 2.4594 | LR: 2.04e-04 | 7637.7ms/step | 2.6 samples/s +Step 6880/11700 | Loss: 2.4605 | LR: 2.03e-04 | 7612.0ms/step | 2.6 samples/s +Step 6890/11700 | Loss: 2.4814 | LR: 2.03e-04 | 7610.6ms/step | 2.6 samples/s +Step 6900/11700 | Loss: 2.6037 | LR: 2.02e-04 | 7631.3ms/step | 2.6 samples/s +Step 6910/11700 | Loss: 2.5723 | LR: 2.01e-04 | 7626.3ms/step | 2.6 samples/s +Step 6920/11700 | Loss: 2.4288 | LR: 2.00e-04 | 7616.0ms/step | 2.6 samples/s +Step 6930/11700 | Loss: 2.3774 | LR: 2.00e-04 | 7611.8ms/step | 2.6 samples/s +Step 6940/11700 | Loss: 2.5882 | LR: 1.99e-04 | 7606.3ms/step | 2.6 samples/s +Step 6950/11700 | Loss: 2.5361 | LR: 1.98e-04 | 7614.2ms/step | 2.6 samples/s +Step 6960/11700 | Loss: 2.4846 | LR: 1.98e-04 | 7613.6ms/step | 2.6 samples/s +Step 6970/11700 | Loss: 2.3327 | LR: 1.97e-04 | 7614.0ms/step | 2.6 samples/s +Step 6980/11700 | Loss: 2.4929 | LR: 1.96e-04 | 7614.3ms/step | 2.6 samples/s +Step 6990/11700 | Loss: 2.5604 | LR: 1.96e-04 | 7611.8ms/step | 2.6 samples/s +Step 7000/11700 | Loss: 2.4927 | LR: 1.95e-04 | 7610.0ms/step | 2.6 samples/s +Validation at step 7000: Loss = 2.4958, Perplexity = 12.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-7000 +Step 7010/11700 | Loss: 2.5957 | LR: 1.94e-04 | 19260.2ms/step | 1.0 samples/s +Step 7020/11700 | Loss: 2.4771 | LR: 1.93e-04 | 7610.3ms/step | 2.6 samples/s +Step 7030/11700 | Loss: 2.5690 | LR: 1.93e-04 | 7611.8ms/step | 2.6 samples/s +Step 7040/11700 | Loss: 2.5455 | LR: 1.92e-04 | 7610.2ms/step | 2.6 samples/s +Step 7050/11700 | Loss: 2.4176 | LR: 1.91e-04 | 7611.8ms/step | 2.6 samples/s +Step 7060/11700 | Loss: 2.5393 | LR: 1.91e-04 | 7612.4ms/step | 2.6 samples/s +Step 7070/11700 | Loss: 2.4671 | LR: 1.90e-04 | 7609.4ms/step | 2.6 samples/s +Step 7080/11700 | Loss: 2.4823 | LR: 1.89e-04 | 7610.7ms/step | 2.6 samples/s +Step 7090/11700 | Loss: 2.4852 | LR: 1.89e-04 | 7608.1ms/step | 2.6 samples/s +Step 7100/11700 | Loss: 2.5228 | LR: 1.88e-04 | 7615.2ms/step | 2.6 samples/s +Step 7110/11700 | Loss: 2.4640 | LR: 1.87e-04 | 7617.3ms/step | 2.6 samples/s +Step 7120/11700 | Loss: 2.4083 | LR: 1.87e-04 | 7612.9ms/step | 2.6 samples/s +Step 7130/11700 | Loss: 2.4064 | LR: 1.86e-04 | 7610.9ms/step | 2.6 samples/s +Step 7140/11700 | Loss: 2.4667 | LR: 1.85e-04 | 7605.6ms/step | 2.6 samples/s +Step 7150/11700 | Loss: 2.5480 | LR: 1.84e-04 | 7610.5ms/step | 2.6 samples/s +Step 7160/11700 | Loss: 2.4613 | LR: 1.84e-04 | 7611.8ms/step | 2.6 samples/s +Step 7170/11700 | Loss: 2.4995 | LR: 1.83e-04 | 7606.1ms/step | 2.6 samples/s +Step 7180/11700 | Loss: 2.4060 | LR: 1.82e-04 | 7617.0ms/step | 2.6 samples/s +Step 7190/11700 | Loss: 2.4091 | LR: 1.82e-04 | 7611.9ms/step | 2.6 samples/s +Step 7200/11700 | Loss: 2.3739 | LR: 1.81e-04 | 7611.4ms/step | 2.6 samples/s +Step 7210/11700 | Loss: 2.3184 | LR: 1.80e-04 | 7607.6ms/step | 2.6 samples/s +Step 7220/11700 | Loss: 2.4571 | LR: 1.80e-04 | 7612.6ms/step | 2.6 samples/s +Step 7230/11700 | Loss: 2.5005 | LR: 1.79e-04 | 7609.8ms/step | 2.6 samples/s +Step 7240/11700 | Loss: 2.4589 | LR: 1.78e-04 | 7620.0ms/step | 2.6 samples/s +Step 7250/11700 | Loss: 2.3302 | LR: 1.78e-04 | 7612.0ms/step | 2.6 samples/s +Validation at step 7250: Loss = 2.4883, Perplexity = 12.04 +Step 7260/11700 | Loss: 2.4484 | LR: 1.77e-04 | 18932.0ms/step | 1.1 samples/s +Step 7270/11700 | Loss: 2.5020 | LR: 1.76e-04 | 7602.1ms/step | 2.6 samples/s +Step 7280/11700 | Loss: 2.5069 | LR: 1.75e-04 | 7600.0ms/step | 2.6 samples/s +Step 7290/11700 | Loss: 2.5024 | LR: 1.75e-04 | 7600.2ms/step | 2.6 samples/s +Step 7300/11700 | Loss: 2.4244 | LR: 1.74e-04 | 7599.9ms/step | 2.6 samples/s +Step 7310/11700 | Loss: 2.5489 | LR: 1.73e-04 | 7600.0ms/step | 2.6 samples/s +Step 7320/11700 | Loss: 2.4468 | LR: 1.73e-04 | 7600.0ms/step | 2.6 samples/s +Step 7330/11700 | Loss: 2.5159 | LR: 1.72e-04 | 7600.2ms/step | 2.6 samples/s +Step 7340/11700 | Loss: 2.4547 | LR: 1.71e-04 | 7599.9ms/step | 2.6 samples/s +Step 7350/11700 | Loss: 2.4385 | LR: 1.71e-04 | 7600.1ms/step | 2.6 samples/s +Step 7360/11700 | Loss: 2.3919 | LR: 1.70e-04 | 7600.2ms/step | 2.6 samples/s +Step 7370/11700 | Loss: 2.4497 | LR: 1.69e-04 | 7601.8ms/step | 2.6 samples/s +Step 7380/11700 | Loss: 2.4888 | LR: 1.69e-04 | 7600.0ms/step | 2.6 samples/s +Step 7390/11700 | Loss: 2.4555 | LR: 1.68e-04 | 7601.4ms/step | 2.6 samples/s +Step 7400/11700 | Loss: 2.4899 | LR: 1.67e-04 | 7600.7ms/step | 2.6 samples/s +Step 7410/11700 | Loss: 2.4119 | LR: 1.67e-04 | 7600.1ms/step | 2.6 samples/s +Step 7420/11700 | Loss: 2.3953 | LR: 1.66e-04 | 7606.8ms/step | 2.6 samples/s +Step 7430/11700 | Loss: 2.4076 | LR: 1.65e-04 | 7601.4ms/step | 2.6 samples/s +Step 7440/11700 | Loss: 2.5110 | LR: 1.65e-04 | 7599.9ms/step | 2.6 samples/s +Step 7450/11700 | Loss: 2.4912 | LR: 1.64e-04 | 7599.9ms/step | 2.6 samples/s +Step 7460/11700 | Loss: 2.4984 | LR: 1.63e-04 | 7600.3ms/step | 2.6 samples/s +Step 7470/11700 | Loss: 2.4511 | LR: 1.63e-04 | 7600.2ms/step | 2.6 samples/s +Step 7480/11700 | Loss: 2.5837 | LR: 1.62e-04 | 7599.8ms/step | 2.6 samples/s +Step 7490/11700 | Loss: 2.4172 | LR: 1.61e-04 | 7600.0ms/step | 2.6 samples/s +Step 7500/11700 | Loss: 2.4074 | LR: 1.61e-04 | 7600.2ms/step | 2.6 samples/s +Validation at step 7500: Loss = 2.4824, Perplexity = 11.97 +Step 7510/11700 | Loss: 2.4924 | LR: 1.60e-04 | 18952.5ms/step | 1.1 samples/s +Step 7520/11700 | Loss: 2.5156 | LR: 1.59e-04 | 7601.8ms/step | 2.6 samples/s +Step 7530/11700 | Loss: 2.4196 | LR: 1.59e-04 | 7602.0ms/step | 2.6 samples/s +Step 7540/11700 | Loss: 2.4979 | LR: 1.58e-04 | 7600.2ms/step | 2.6 samples/s +Step 7550/11700 | Loss: 2.4442 | LR: 1.57e-04 | 7608.3ms/step | 2.6 samples/s +Step 7560/11700 | Loss: 2.5041 | LR: 1.57e-04 | 7607.7ms/step | 2.6 samples/s +Step 7570/11700 | Loss: 2.4641 | LR: 1.56e-04 | 7616.2ms/step | 2.6 samples/s +Step 7580/11700 | Loss: 2.4518 | LR: 1.55e-04 | 7622.0ms/step | 2.6 samples/s +Step 7590/11700 | Loss: 2.4324 | LR: 1.55e-04 | 7626.3ms/step | 2.6 samples/s +Step 7600/11700 | Loss: 2.4897 | LR: 1.54e-04 | 7623.0ms/step | 2.6 samples/s +Step 7610/11700 | Loss: 2.5839 | LR: 1.53e-04 | 7624.8ms/step | 2.6 samples/s +Step 7620/11700 | Loss: 2.5503 | LR: 1.53e-04 | 7621.8ms/step | 2.6 samples/s +Step 7630/11700 | Loss: 2.4257 | LR: 1.52e-04 | 7619.3ms/step | 2.6 samples/s +Step 7640/11700 | Loss: 2.4594 | LR: 1.51e-04 | 7621.3ms/step | 2.6 samples/s +Step 7650/11700 | Loss: 2.4954 | LR: 1.51e-04 | 7615.8ms/step | 2.6 samples/s +Step 7660/11700 | Loss: 2.4436 | LR: 1.50e-04 | 7614.2ms/step | 2.6 samples/s +Step 7670/11700 | Loss: 2.3815 | LR: 1.49e-04 | 7613.6ms/step | 2.6 samples/s +Step 7680/11700 | Loss: 2.4435 | LR: 1.49e-04 | 7614.8ms/step | 2.6 samples/s +Step 7690/11700 | Loss: 2.5233 | LR: 1.48e-04 | 7610.2ms/step | 2.6 samples/s +Step 7700/11700 | Loss: 2.5875 | LR: 1.47e-04 | 7611.5ms/step | 2.6 samples/s +Step 7710/11700 | Loss: 2.3897 | LR: 1.47e-04 | 7614.3ms/step | 2.6 samples/s +Step 7720/11700 | Loss: 2.4910 | LR: 1.46e-04 | 7610.0ms/step | 2.6 samples/s +Step 7730/11700 | Loss: 2.3856 | LR: 1.45e-04 | 7614.1ms/step | 2.6 samples/s +Step 7740/11700 | Loss: 2.5447 | LR: 1.45e-04 | 7612.1ms/step | 2.6 samples/s +Step 7750/11700 | Loss: 2.4608 | LR: 1.44e-04 | 7612.9ms/step | 2.6 samples/s +Validation at step 7750: Loss = 2.4757, Perplexity = 11.89 +Step 7760/11700 | Loss: 2.5773 | LR: 1.43e-04 | 18972.8ms/step | 1.1 samples/s +Step 7770/11700 | Loss: 2.4405 | LR: 1.43e-04 | 7612.4ms/step | 2.6 samples/s +Step 7780/11700 | Loss: 2.5363 | LR: 1.42e-04 | 7613.9ms/step | 2.6 samples/s +Step 7790/11700 | Loss: 2.4366 | LR: 1.41e-04 | 7611.9ms/step | 2.6 samples/s +Step 7800/11700 | Loss: 2.5219 | LR: 1.41e-04 | 7606.3ms/step | 2.6 samples/s +Step 7810/11700 | Loss: 2.3639 | LR: 1.40e-04 | 7606.2ms/step | 2.6 samples/s +Step 7820/11700 | Loss: 2.4076 | LR: 1.40e-04 | 7607.9ms/step | 2.6 samples/s +Step 7830/11700 | Loss: 2.4099 | LR: 1.39e-04 | 7608.5ms/step | 2.6 samples/s +Step 7840/11700 | Loss: 2.4722 | LR: 1.38e-04 | 7613.9ms/step | 2.6 samples/s +Step 7850/11700 | Loss: 2.4925 | LR: 1.38e-04 | 7613.8ms/step | 2.6 samples/s +Step 7860/11700 | Loss: 2.4905 | LR: 1.37e-04 | 7615.5ms/step | 2.6 samples/s +Step 7870/11700 | Loss: 2.5307 | LR: 1.36e-04 | 7612.6ms/step | 2.6 samples/s +Step 7880/11700 | Loss: 2.4855 | LR: 1.36e-04 | 7607.0ms/step | 2.6 samples/s +Step 7890/11700 | Loss: 2.4431 | LR: 1.35e-04 | 7607.5ms/step | 2.6 samples/s +Step 7900/11700 | Loss: 2.5452 | LR: 1.34e-04 | 7616.2ms/step | 2.6 samples/s +Step 7910/11700 | Loss: 2.3239 | LR: 1.34e-04 | 7621.4ms/step | 2.6 samples/s +Step 7920/11700 | Loss: 2.4277 | LR: 1.33e-04 | 7617.4ms/step | 2.6 samples/s +Step 7930/11700 | Loss: 2.4371 | LR: 1.33e-04 | 7625.0ms/step | 2.6 samples/s +Step 7940/11700 | Loss: 2.4883 | LR: 1.32e-04 | 7621.8ms/step | 2.6 samples/s +Step 7950/11700 | Loss: 2.5104 | LR: 1.31e-04 | 7621.1ms/step | 2.6 samples/s +Step 7960/11700 | Loss: 2.4369 | LR: 1.31e-04 | 7617.2ms/step | 2.6 samples/s +Step 7970/11700 | Loss: 2.3847 | LR: 1.30e-04 | 7618.1ms/step | 2.6 samples/s +Step 7980/11700 | Loss: 2.5393 | LR: 1.29e-04 | 7606.0ms/step | 2.6 samples/s +Step 7990/11700 | Loss: 2.5148 | LR: 1.29e-04 | 7608.4ms/step | 2.6 samples/s +Step 8000/11700 | Loss: 2.3985 | LR: 1.28e-04 | 7613.7ms/step | 2.6 samples/s +Validation at step 8000: Loss = 2.4711, Perplexity = 11.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-8000 +Step 8010/11700 | Loss: 2.4858 | LR: 1.27e-04 | 19285.9ms/step | 1.0 samples/s +Step 8020/11700 | Loss: 2.3606 | LR: 1.27e-04 | 7604.2ms/step | 2.6 samples/s +Step 8030/11700 | Loss: 2.4512 | LR: 1.26e-04 | 7604.0ms/step | 2.6 samples/s +Step 8040/11700 | Loss: 2.4588 | LR: 1.26e-04 | 7606.0ms/step | 2.6 samples/s +Step 8050/11700 | Loss: 2.4208 | LR: 1.25e-04 | 7602.0ms/step | 2.6 samples/s +Step 8060/11700 | Loss: 2.4477 | LR: 1.24e-04 | 7604.2ms/step | 2.6 samples/s +Step 8070/11700 | Loss: 2.5032 | LR: 1.24e-04 | 7608.3ms/step | 2.6 samples/s +Step 8080/11700 | Loss: 2.5011 | LR: 1.23e-04 | 7612.1ms/step | 2.6 samples/s +Step 8090/11700 | Loss: 2.4319 | LR: 1.23e-04 | 7607.8ms/step | 2.6 samples/s +Step 8100/11700 | Loss: 2.4687 | LR: 1.22e-04 | 7606.7ms/step | 2.6 samples/s +Step 8110/11700 | Loss: 2.4155 | LR: 1.21e-04 | 7607.7ms/step | 2.6 samples/s +Step 8120/11700 | Loss: 2.3963 | LR: 1.21e-04 | 7609.9ms/step | 2.6 samples/s +Step 8130/11700 | Loss: 2.3743 | LR: 1.20e-04 | 7608.2ms/step | 2.6 samples/s +Step 8140/11700 | Loss: 2.4296 | LR: 1.19e-04 | 7610.2ms/step | 2.6 samples/s +Step 8150/11700 | Loss: 2.3140 | LR: 1.19e-04 | 7615.5ms/step | 2.6 samples/s +Step 8160/11700 | Loss: 2.5428 | LR: 1.18e-04 | 7616.4ms/step | 2.6 samples/s +Step 8170/11700 | Loss: 2.4384 | LR: 1.18e-04 | 7619.7ms/step | 2.6 samples/s +Step 8180/11700 | Loss: 2.4834 | LR: 1.17e-04 | 7614.6ms/step | 2.6 samples/s +Step 8190/11700 | Loss: 2.4023 | LR: 1.16e-04 | 7609.9ms/step | 2.6 samples/s +Step 8200/11700 | Loss: 2.4544 | LR: 1.16e-04 | 7609.1ms/step | 2.6 samples/s +Step 8210/11700 | Loss: 2.4875 | LR: 1.15e-04 | 7609.1ms/step | 2.6 samples/s +Step 8220/11700 | Loss: 2.4880 | LR: 1.15e-04 | 7610.0ms/step | 2.6 samples/s +Step 8230/11700 | Loss: 2.4703 | LR: 1.14e-04 | 7610.4ms/step | 2.6 samples/s +Step 8240/11700 | Loss: 2.4412 | LR: 1.13e-04 | 7613.4ms/step | 2.6 samples/s +Step 8250/11700 | Loss: 2.4716 | LR: 1.13e-04 | 7610.6ms/step | 2.6 samples/s +Validation at step 8250: Loss = 2.4672, Perplexity = 11.79 +Step 8260/11700 | Loss: 2.4073 | LR: 1.12e-04 | 18951.8ms/step | 1.1 samples/s +Step 8270/11700 | Loss: 2.4175 | LR: 1.12e-04 | 7612.1ms/step | 2.6 samples/s +Step 8280/11700 | Loss: 2.5658 | LR: 1.11e-04 | 7642.0ms/step | 2.6 samples/s +Step 8290/11700 | Loss: 2.4277 | LR: 1.10e-04 | 7621.8ms/step | 2.6 samples/s +Step 8300/11700 | Loss: 2.4798 | LR: 1.10e-04 | 7618.0ms/step | 2.6 samples/s +Step 8310/11700 | Loss: 2.4124 | LR: 1.09e-04 | 7616.5ms/step | 2.6 samples/s +Step 8320/11700 | Loss: 2.4976 | LR: 1.09e-04 | 7616.0ms/step | 2.6 samples/s +Step 8330/11700 | Loss: 2.5052 | LR: 1.08e-04 | 7614.1ms/step | 2.6 samples/s +Step 8340/11700 | Loss: 2.4697 | LR: 1.07e-04 | 7614.9ms/step | 2.6 samples/s +Step 8350/11700 | Loss: 2.4827 | LR: 1.07e-04 | 7613.1ms/step | 2.6 samples/s +Step 8360/11700 | Loss: 2.4249 | LR: 1.06e-04 | 7614.5ms/step | 2.6 samples/s +Step 8370/11700 | Loss: 2.4146 | LR: 1.06e-04 | 7613.6ms/step | 2.6 samples/s +Step 8380/11700 | Loss: 2.4473 | LR: 1.05e-04 | 7615.2ms/step | 2.6 samples/s +Step 8390/11700 | Loss: 2.4543 | LR: 1.05e-04 | 7608.9ms/step | 2.6 samples/s +Step 8400/11700 | Loss: 2.5145 | LR: 1.04e-04 | 7611.9ms/step | 2.6 samples/s +Step 8410/11700 | Loss: 2.6366 | LR: 1.03e-04 | 7610.2ms/step | 2.6 samples/s +Step 8420/11700 | Loss: 2.3940 | LR: 1.03e-04 | 7614.1ms/step | 2.6 samples/s +Step 8430/11700 | Loss: 2.5150 | LR: 1.02e-04 | 7613.4ms/step | 2.6 samples/s +Step 8440/11700 | Loss: 2.5124 | LR: 1.02e-04 | 7615.5ms/step | 2.6 samples/s +Step 8450/11700 | Loss: 2.4560 | LR: 1.01e-04 | 7614.7ms/step | 2.6 samples/s +Step 8460/11700 | Loss: 2.4455 | LR: 1.00e-04 | 7614.5ms/step | 2.6 samples/s +Step 8470/11700 | Loss: 2.4419 | LR: 9.99e-05 | 7614.1ms/step | 2.6 samples/s +Step 8480/11700 | Loss: 2.4938 | LR: 9.93e-05 | 7615.8ms/step | 2.6 samples/s +Step 8490/11700 | Loss: 2.4452 | LR: 9.87e-05 | 7612.2ms/step | 2.6 samples/s +Step 8500/11700 | Loss: 2.3090 | LR: 9.82e-05 | 7612.2ms/step | 2.6 samples/s +Validation at step 8500: Loss = 2.4631, Perplexity = 11.74 +Step 8510/11700 | Loss: 2.4117 | LR: 9.76e-05 | 18960.3ms/step | 1.1 samples/s +Step 8520/11700 | Loss: 2.4581 | LR: 9.70e-05 | 7615.7ms/step | 2.6 samples/s +Step 8530/11700 | Loss: 2.4832 | LR: 9.65e-05 | 7612.4ms/step | 2.6 samples/s +Step 8540/11700 | Loss: 2.4542 | LR: 9.59e-05 | 7611.7ms/step | 2.6 samples/s +Step 8550/11700 | Loss: 2.3445 | LR: 9.53e-05 | 7612.1ms/step | 2.6 samples/s +Step 8560/11700 | Loss: 2.4550 | LR: 9.48e-05 | 7605.8ms/step | 2.6 samples/s +Step 8570/11700 | Loss: 2.4472 | LR: 9.42e-05 | 7611.9ms/step | 2.6 samples/s +Step 8580/11700 | Loss: 2.4328 | LR: 9.36e-05 | 7612.2ms/step | 2.6 samples/s +Step 8590/11700 | Loss: 2.4031 | LR: 9.31e-05 | 7609.7ms/step | 2.6 samples/s +Step 8600/11700 | Loss: 2.4752 | LR: 9.25e-05 | 7608.3ms/step | 2.6 samples/s +Step 8610/11700 | Loss: 2.4492 | LR: 9.20e-05 | 7606.8ms/step | 2.6 samples/s +Step 8620/11700 | Loss: 2.5041 | LR: 9.14e-05 | 7613.9ms/step | 2.6 samples/s +Step 8630/11700 | Loss: 2.4762 | LR: 9.09e-05 | 7613.3ms/step | 2.6 samples/s +Step 8640/11700 | Loss: 2.4130 | LR: 9.03e-05 | 7607.9ms/step | 2.6 samples/s +Step 8650/11700 | Loss: 2.3716 | LR: 8.98e-05 | 7610.4ms/step | 2.6 samples/s +Step 8660/11700 | Loss: 2.4344 | LR: 8.92e-05 | 7616.1ms/step | 2.6 samples/s +Step 8670/11700 | Loss: 2.5263 | LR: 8.87e-05 | 7615.9ms/step | 2.6 samples/s +Step 8680/11700 | Loss: 2.4038 | LR: 8.81e-05 | 7616.2ms/step | 2.6 samples/s +Step 8690/11700 | Loss: 2.5273 | LR: 8.76e-05 | 7614.3ms/step | 2.6 samples/s +Step 8700/11700 | Loss: 2.4522 | LR: 8.70e-05 | 7614.4ms/step | 2.6 samples/s +Step 8710/11700 | Loss: 2.3183 | LR: 8.65e-05 | 7611.4ms/step | 2.6 samples/s +Step 8720/11700 | Loss: 2.3937 | LR: 8.59e-05 | 7609.9ms/step | 2.6 samples/s +Step 8730/11700 | Loss: 2.4416 | LR: 8.54e-05 | 7608.0ms/step | 2.6 samples/s +Step 8740/11700 | Loss: 2.4696 | LR: 8.49e-05 | 7615.2ms/step | 2.6 samples/s +Step 8750/11700 | Loss: 2.4761 | LR: 8.43e-05 | 7615.0ms/step | 2.6 samples/s +Validation at step 8750: Loss = 2.4606, Perplexity = 11.71 +Step 8760/11700 | Loss: 2.3846 | LR: 8.38e-05 | 18936.0ms/step | 1.1 samples/s +Step 8770/11700 | Loss: 2.4421 | LR: 8.33e-05 | 7602.5ms/step | 2.6 samples/s +Step 8780/11700 | Loss: 2.4413 | LR: 8.27e-05 | 7599.6ms/step | 2.6 samples/s +Step 8790/11700 | Loss: 2.4803 | LR: 8.22e-05 | 7600.1ms/step | 2.6 samples/s +Step 8800/11700 | Loss: 2.5086 | LR: 8.17e-05 | 7600.1ms/step | 2.6 samples/s +Step 8810/11700 | Loss: 2.3626 | LR: 8.11e-05 | 7602.2ms/step | 2.6 samples/s +Step 8820/11700 | Loss: 2.3586 | LR: 8.06e-05 | 7602.4ms/step | 2.6 samples/s +Step 8830/11700 | Loss: 2.4181 | LR: 8.01e-05 | 7601.9ms/step | 2.6 samples/s +Step 8840/11700 | Loss: 2.4983 | LR: 7.95e-05 | 7601.8ms/step | 2.6 samples/s +Step 8850/11700 | Loss: 2.4658 | LR: 7.90e-05 | 7604.2ms/step | 2.6 samples/s +Step 8860/11700 | Loss: 2.3445 | LR: 7.85e-05 | 7604.2ms/step | 2.6 samples/s +Step 8870/11700 | Loss: 2.4008 | LR: 7.80e-05 | 7605.9ms/step | 2.6 samples/s +Step 8880/11700 | Loss: 2.4861 | LR: 7.75e-05 | 7606.7ms/step | 2.6 samples/s +Step 8890/11700 | Loss: 2.4481 | LR: 7.69e-05 | 7603.1ms/step | 2.6 samples/s +Step 8900/11700 | Loss: 2.4261 | LR: 7.64e-05 | 7602.4ms/step | 2.6 samples/s +Step 8910/11700 | Loss: 2.5236 | LR: 7.59e-05 | 7603.9ms/step | 2.6 samples/s +Step 8920/11700 | Loss: 2.5018 | LR: 7.54e-05 | 7599.8ms/step | 2.6 samples/s +Step 8930/11700 | Loss: 2.3859 | LR: 7.49e-05 | 7600.2ms/step | 2.6 samples/s +Step 8940/11700 | Loss: 2.4857 | LR: 7.44e-05 | 7600.7ms/step | 2.6 samples/s +Step 8950/11700 | Loss: 2.4159 | LR: 7.39e-05 | 7603.2ms/step | 2.6 samples/s +Step 8960/11700 | Loss: 2.4252 | LR: 7.34e-05 | 7600.5ms/step | 2.6 samples/s +Step 8970/11700 | Loss: 2.4335 | LR: 7.28e-05 | 7607.8ms/step | 2.6 samples/s +Step 8980/11700 | Loss: 2.4233 | LR: 7.23e-05 | 7605.1ms/step | 2.6 samples/s +Step 8990/11700 | Loss: 2.4610 | LR: 7.18e-05 | 7605.3ms/step | 2.6 samples/s +Step 9000/11700 | Loss: 2.4376 | LR: 7.13e-05 | 7601.7ms/step | 2.6 samples/s +Validation at step 9000: Loss = 2.4583, Perplexity = 11.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-9000 +Step 9010/11700 | Loss: 2.5255 | LR: 7.08e-05 | 19298.3ms/step | 1.0 samples/s +Step 9020/11700 | Loss: 2.4385 | LR: 7.03e-05 | 7614.7ms/step | 2.6 samples/s +Step 9030/11700 | Loss: 2.4030 | LR: 6.98e-05 | 7613.4ms/step | 2.6 samples/s +Step 9040/11700 | Loss: 2.3766 | LR: 6.93e-05 | 7616.0ms/step | 2.6 samples/s +Step 9050/11700 | Loss: 2.4717 | LR: 6.88e-05 | 7614.4ms/step | 2.6 samples/s +Step 9060/11700 | Loss: 2.4862 | LR: 6.83e-05 | 7616.2ms/step | 2.6 samples/s +Step 9070/11700 | Loss: 2.4996 | LR: 6.79e-05 | 7618.0ms/step | 2.6 samples/s +Step 9080/11700 | Loss: 2.3957 | LR: 6.74e-05 | 7618.2ms/step | 2.6 samples/s +Step 9090/11700 | Loss: 2.3826 | LR: 6.69e-05 | 7617.7ms/step | 2.6 samples/s +Step 9100/11700 | Loss: 2.4701 | LR: 6.64e-05 | 7616.4ms/step | 2.6 samples/s +Step 9110/11700 | Loss: 2.4690 | LR: 6.59e-05 | 7613.7ms/step | 2.6 samples/s +Step 9120/11700 | Loss: 2.4997 | LR: 6.54e-05 | 7613.8ms/step | 2.6 samples/s +Step 9130/11700 | Loss: 2.3659 | LR: 6.49e-05 | 7612.2ms/step | 2.6 samples/s +Step 9140/11700 | Loss: 2.3443 | LR: 6.45e-05 | 7612.2ms/step | 2.6 samples/s +Step 9150/11700 | Loss: 2.4553 | LR: 6.40e-05 | 7615.0ms/step | 2.6 samples/s +Step 9160/11700 | Loss: 2.5446 | LR: 6.35e-05 | 7614.7ms/step | 2.6 samples/s +Step 9170/11700 | Loss: 2.4946 | LR: 6.30e-05 | 7615.8ms/step | 2.6 samples/s +Step 9180/11700 | Loss: 2.4301 | LR: 6.25e-05 | 7618.4ms/step | 2.6 samples/s +Step 9190/11700 | Loss: 2.4538 | LR: 6.21e-05 | 7614.9ms/step | 2.6 samples/s +Step 9200/11700 | Loss: 2.3359 | LR: 6.16e-05 | 7618.9ms/step | 2.6 samples/s +Step 9210/11700 | Loss: 2.4414 | LR: 6.11e-05 | 7612.6ms/step | 2.6 samples/s +Step 9220/11700 | Loss: 2.5241 | LR: 6.07e-05 | 7615.9ms/step | 2.6 samples/s +Step 9230/11700 | Loss: 2.4303 | LR: 6.02e-05 | 7612.9ms/step | 2.6 samples/s +Step 9240/11700 | Loss: 2.4171 | LR: 5.97e-05 | 7609.4ms/step | 2.6 samples/s +Step 9250/11700 | Loss: 2.4482 | LR: 5.93e-05 | 7607.8ms/step | 2.6 samples/s +Validation at step 9250: Loss = 2.4567, Perplexity = 11.67 +Step 9260/11700 | Loss: 2.3713 | LR: 5.88e-05 | 18959.1ms/step | 1.1 samples/s +Step 9270/11700 | Loss: 2.4650 | LR: 5.83e-05 | 7611.1ms/step | 2.6 samples/s +Step 9280/11700 | Loss: 2.3205 | LR: 5.79e-05 | 7631.1ms/step | 2.6 samples/s +Step 9290/11700 | Loss: 2.4082 | LR: 5.74e-05 | 7638.9ms/step | 2.6 samples/s +Step 9300/11700 | Loss: 2.4636 | LR: 5.70e-05 | 7608.1ms/step | 2.6 samples/s +Step 9310/11700 | Loss: 2.3581 | LR: 5.65e-05 | 7625.6ms/step | 2.6 samples/s +Step 9320/11700 | Loss: 2.4004 | LR: 5.61e-05 | 7610.4ms/step | 2.6 samples/s +Step 9330/11700 | Loss: 2.3972 | LR: 5.56e-05 | 7616.8ms/step | 2.6 samples/s +Step 9340/11700 | Loss: 2.5079 | LR: 5.52e-05 | 7631.1ms/step | 2.6 samples/s +Step 9350/11700 | Loss: 2.3959 | LR: 5.47e-05 | 7621.0ms/step | 2.6 samples/s +Step 9360/11700 | Loss: 2.4284 | LR: 5.43e-05 | 7607.1ms/step | 2.6 samples/s +Step 9370/11700 | Loss: 2.4451 | LR: 5.38e-05 | 7606.1ms/step | 2.6 samples/s +Step 9380/11700 | Loss: 2.5476 | LR: 5.34e-05 | 7615.8ms/step | 2.6 samples/s +Step 9390/11700 | Loss: 2.4552 | LR: 5.29e-05 | 7616.5ms/step | 2.6 samples/s +Step 9400/11700 | Loss: 2.4323 | LR: 5.25e-05 | 7630.1ms/step | 2.6 samples/s +Step 9410/11700 | Loss: 2.4474 | LR: 5.20e-05 | 7612.1ms/step | 2.6 samples/s +Step 9420/11700 | Loss: 2.5106 | LR: 5.16e-05 | 7617.3ms/step | 2.6 samples/s +Step 9430/11700 | Loss: 2.3208 | LR: 5.12e-05 | 7612.7ms/step | 2.6 samples/s +Step 9440/11700 | Loss: 2.5372 | LR: 5.07e-05 | 7607.6ms/step | 2.6 samples/s +Step 9450/11700 | Loss: 2.4803 | LR: 5.03e-05 | 7613.5ms/step | 2.6 samples/s +Step 9460/11700 | Loss: 2.4792 | LR: 4.99e-05 | 7620.6ms/step | 2.6 samples/s +Step 9470/11700 | Loss: 2.4554 | LR: 4.94e-05 | 7600.1ms/step | 2.6 samples/s +Step 9480/11700 | Loss: 2.4132 | LR: 4.90e-05 | 7618.2ms/step | 2.6 samples/s +Step 9490/11700 | Loss: 2.5626 | LR: 4.86e-05 | 7614.1ms/step | 2.6 samples/s +Step 9500/11700 | Loss: 2.3702 | LR: 4.82e-05 | 7613.9ms/step | 2.6 samples/s +Validation at step 9500: Loss = 2.4554, Perplexity = 11.65 +Step 9510/11700 | Loss: 2.4578 | LR: 4.77e-05 | 18939.9ms/step | 1.1 samples/s +Step 9520/11700 | Loss: 2.4109 | LR: 4.73e-05 | 7601.9ms/step | 2.6 samples/s +Step 9530/11700 | Loss: 2.4140 | LR: 4.69e-05 | 7600.1ms/step | 2.6 samples/s +Step 9540/11700 | Loss: 2.3761 | LR: 4.65e-05 | 7600.2ms/step | 2.6 samples/s +Step 9550/11700 | Loss: 2.5817 | LR: 4.61e-05 | 7599.9ms/step | 2.6 samples/s +Step 9560/11700 | Loss: 2.3609 | LR: 4.57e-05 | 7600.1ms/step | 2.6 samples/s +Step 9570/11700 | Loss: 2.3992 | LR: 4.52e-05 | 7600.1ms/step | 2.6 samples/s +Step 9580/11700 | Loss: 2.5193 | LR: 4.48e-05 | 7600.1ms/step | 2.6 samples/s +Step 9590/11700 | Loss: 2.4503 | LR: 4.44e-05 | 7599.9ms/step | 2.6 samples/s +Step 9600/11700 | Loss: 2.4596 | LR: 4.40e-05 | 7600.0ms/step | 2.6 samples/s +Step 9610/11700 | Loss: 2.4602 | LR: 4.36e-05 | 7600.1ms/step | 2.6 samples/s +Step 9620/11700 | Loss: 2.4718 | LR: 4.32e-05 | 7600.0ms/step | 2.6 samples/s +Step 9630/11700 | Loss: 2.4188 | LR: 4.28e-05 | 7600.1ms/step | 2.6 samples/s +Step 9640/11700 | Loss: 2.4699 | LR: 4.24e-05 | 7600.1ms/step | 2.6 samples/s +Step 9650/11700 | Loss: 2.5473 | LR: 4.20e-05 | 7600.3ms/step | 2.6 samples/s +Step 9660/11700 | Loss: 2.5540 | LR: 4.16e-05 | 7599.8ms/step | 2.6 samples/s +Step 9670/11700 | Loss: 2.4500 | LR: 4.12e-05 | 7600.1ms/step | 2.6 samples/s +Step 9680/11700 | Loss: 2.2710 | LR: 4.08e-05 | 7600.1ms/step | 2.6 samples/s +Step 9690/11700 | Loss: 2.5106 | LR: 4.04e-05 | 7602.0ms/step | 2.6 samples/s +Step 9700/11700 | Loss: 2.3795 | LR: 4.00e-05 | 7600.1ms/step | 2.6 samples/s +Step 9710/11700 | Loss: 2.4320 | LR: 3.97e-05 | 7600.0ms/step | 2.6 samples/s +Step 9720/11700 | Loss: 2.5059 | LR: 3.93e-05 | 7602.4ms/step | 2.6 samples/s +Step 9730/11700 | Loss: 2.3480 | LR: 3.89e-05 | 7599.7ms/step | 2.6 samples/s +Step 9740/11700 | Loss: 2.5204 | LR: 3.85e-05 | 7600.1ms/step | 2.6 samples/s +Step 9750/11700 | Loss: 2.4430 | LR: 3.81e-05 | 7602.0ms/step | 2.6 samples/s +Validation at step 9750: Loss = 2.4544, Perplexity = 11.64 +Step 9760/11700 | Loss: 2.4282 | LR: 3.77e-05 | 18937.9ms/step | 1.1 samples/s +Step 9770/11700 | Loss: 2.3716 | LR: 3.74e-05 | 7600.5ms/step | 2.6 samples/s +Step 9780/11700 | Loss: 2.5664 | LR: 3.70e-05 | 7603.6ms/step | 2.6 samples/s +Step 9790/11700 | Loss: 2.4246 | LR: 3.66e-05 | 7602.2ms/step | 2.6 samples/s +Step 9800/11700 | Loss: 2.3735 | LR: 3.62e-05 | 7602.0ms/step | 2.6 samples/s +Step 9810/11700 | Loss: 2.4897 | LR: 3.59e-05 | 7600.0ms/step | 2.6 samples/s +Step 9820/11700 | Loss: 2.3394 | LR: 3.55e-05 | 7602.1ms/step | 2.6 samples/s +Step 9830/11700 | Loss: 2.3603 | LR: 3.51e-05 | 7601.9ms/step | 2.6 samples/s +Step 9840/11700 | Loss: 2.3932 | LR: 3.48e-05 | 7600.2ms/step | 2.6 samples/s +Step 9850/11700 | Loss: 2.4696 | LR: 3.44e-05 | 7601.9ms/step | 2.6 samples/s +Step 9860/11700 | Loss: 2.4482 | LR: 3.40e-05 | 7600.3ms/step | 2.6 samples/s +Step 9870/11700 | Loss: 2.3962 | LR: 3.37e-05 | 7600.0ms/step | 2.6 samples/s +Step 9880/11700 | Loss: 2.4365 | LR: 3.33e-05 | 7600.1ms/step | 2.6 samples/s +Step 9890/11700 | Loss: 2.3439 | LR: 3.30e-05 | 7602.2ms/step | 2.6 samples/s +Step 9900/11700 | Loss: 2.4766 | LR: 3.26e-05 | 7603.7ms/step | 2.6 samples/s +Step 9910/11700 | Loss: 2.4588 | LR: 3.22e-05 | 7599.9ms/step | 2.6 samples/s +Step 9920/11700 | Loss: 2.4233 | LR: 3.19e-05 | 7602.4ms/step | 2.6 samples/s +Step 9930/11700 | Loss: 2.5065 | LR: 3.15e-05 | 7607.9ms/step | 2.6 samples/s +Step 9940/11700 | Loss: 2.3181 | LR: 3.12e-05 | 7600.1ms/step | 2.6 samples/s +Step 9950/11700 | Loss: 2.4278 | LR: 3.09e-05 | 7600.0ms/step | 2.6 samples/s +Step 9960/11700 | Loss: 2.3746 | LR: 3.05e-05 | 7599.9ms/step | 2.6 samples/s +Step 9970/11700 | Loss: 2.4510 | LR: 3.02e-05 | 7600.2ms/step | 2.6 samples/s +Step 9980/11700 | Loss: 2.3806 | LR: 2.98e-05 | 7599.8ms/step | 2.6 samples/s +Step 9990/11700 | Loss: 2.3347 | LR: 2.95e-05 | 7604.2ms/step | 2.6 samples/s +Step 10000/11700 | Loss: 2.4492 | LR: 2.92e-05 | 7600.1ms/step | 2.6 samples/s +Validation at step 10000: Loss = 2.4540, Perplexity = 11.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-10000 +Step 10010/11700 | Loss: 2.5458 | LR: 2.88e-05 | 19284.1ms/step | 1.0 samples/s +Step 10020/11700 | Loss: 2.5321 | LR: 2.85e-05 | 7600.0ms/step | 2.6 samples/s +Step 10030/11700 | Loss: 2.4313 | LR: 2.82e-05 | 7599.9ms/step | 2.6 samples/s +Step 10040/11700 | Loss: 2.4829 | LR: 2.78e-05 | 7600.1ms/step | 2.6 samples/s +Step 10050/11700 | Loss: 2.5191 | LR: 2.75e-05 | 7600.9ms/step | 2.6 samples/s +Step 10060/11700 | Loss: 2.3937 | LR: 2.72e-05 | 7603.5ms/step | 2.6 samples/s +Step 10070/11700 | Loss: 2.5121 | LR: 2.68e-05 | 7599.9ms/step | 2.6 samples/s +Step 10080/11700 | Loss: 2.3880 | LR: 2.65e-05 | 7600.9ms/step | 2.6 samples/s +Step 10090/11700 | Loss: 2.4580 | LR: 2.62e-05 | 7601.4ms/step | 2.6 samples/s +Step 10100/11700 | Loss: 2.4204 | LR: 2.59e-05 | 7599.9ms/step | 2.6 samples/s +Step 10110/11700 | Loss: 2.4916 | LR: 2.56e-05 | 7601.1ms/step | 2.6 samples/s +Step 10120/11700 | Loss: 2.3591 | LR: 2.52e-05 | 7601.0ms/step | 2.6 samples/s +Step 10130/11700 | Loss: 2.5268 | LR: 2.49e-05 | 7602.2ms/step | 2.6 samples/s +Step 10140/11700 | Loss: 2.5598 | LR: 2.46e-05 | 7600.0ms/step | 2.6 samples/s +Step 10150/11700 | Loss: 2.4696 | LR: 2.43e-05 | 7600.0ms/step | 2.6 samples/s +Step 10160/11700 | Loss: 2.4546 | LR: 2.40e-05 | 7600.0ms/step | 2.6 samples/s +Step 10170/11700 | Loss: 2.4404 | LR: 2.37e-05 | 7600.1ms/step | 2.6 samples/s +Step 10180/11700 | Loss: 2.4619 | LR: 2.34e-05 | 7600.1ms/step | 2.6 samples/s +Step 10190/11700 | Loss: 2.3682 | LR: 2.31e-05 | 7599.9ms/step | 2.6 samples/s +Step 10200/11700 | Loss: 2.3946 | LR: 2.28e-05 | 7600.2ms/step | 2.6 samples/s +Step 10210/11700 | Loss: 2.4902 | LR: 2.25e-05 | 7604.2ms/step | 2.6 samples/s +Step 10220/11700 | Loss: 2.4472 | LR: 2.22e-05 | 7599.7ms/step | 2.6 samples/s +Step 10230/11700 | Loss: 2.5511 | LR: 2.19e-05 | 7600.3ms/step | 2.6 samples/s +Step 10240/11700 | Loss: 2.5383 | LR: 2.16e-05 | 7599.9ms/step | 2.6 samples/s +Step 10250/11700 | Loss: 2.4876 | LR: 2.13e-05 | 7600.2ms/step | 2.6 samples/s +Validation at step 10250: Loss = 2.4537, Perplexity = 11.63 +Step 10260/11700 | Loss: 2.4953 | LR: 2.10e-05 | 18930.2ms/step | 1.1 samples/s +Step 10270/11700 | Loss: 2.3981 | LR: 2.07e-05 | 7601.9ms/step | 2.6 samples/s +Step 10280/11700 | Loss: 2.4959 | LR: 2.05e-05 | 7599.9ms/step | 2.6 samples/s +Step 10290/11700 | Loss: 2.4799 | LR: 2.02e-05 | 7602.1ms/step | 2.6 samples/s +Step 10300/11700 | Loss: 2.3334 | LR: 1.99e-05 | 7602.0ms/step | 2.6 samples/s +Step 10310/11700 | Loss: 2.4394 | LR: 1.96e-05 | 7600.0ms/step | 2.6 samples/s +Step 10320/11700 | Loss: 2.3862 | LR: 1.93e-05 | 7604.1ms/step | 2.6 samples/s +Step 10330/11700 | Loss: 2.3776 | LR: 1.91e-05 | 7600.1ms/step | 2.6 samples/s +Step 10340/11700 | Loss: 2.5479 | LR: 1.88e-05 | 7600.1ms/step | 2.6 samples/s +Step 10350/11700 | Loss: 2.4774 | LR: 1.85e-05 | 7602.1ms/step | 2.6 samples/s +Step 10360/11700 | Loss: 2.4729 | LR: 1.82e-05 | 7599.8ms/step | 2.6 samples/s +Step 10370/11700 | Loss: 2.4533 | LR: 1.80e-05 | 7602.1ms/step | 2.6 samples/s +Step 10380/11700 | Loss: 2.2901 | LR: 1.77e-05 | 7600.0ms/step | 2.6 samples/s +Step 10390/11700 | Loss: 2.5865 | LR: 1.75e-05 | 7600.3ms/step | 2.6 samples/s +Step 10400/11700 | Loss: 2.4950 | LR: 1.72e-05 | 7599.7ms/step | 2.6 samples/s +Step 10410/11700 | Loss: 2.4454 | LR: 1.69e-05 | 7602.1ms/step | 2.6 samples/s +Step 10420/11700 | Loss: 2.4234 | LR: 1.67e-05 | 7600.6ms/step | 2.6 samples/s +Step 10430/11700 | Loss: 2.4170 | LR: 1.64e-05 | 7601.4ms/step | 2.6 samples/s +Step 10440/11700 | Loss: 2.3904 | LR: 1.62e-05 | 7600.2ms/step | 2.6 samples/s +Step 10450/11700 | Loss: 2.4663 | LR: 1.59e-05 | 7599.8ms/step | 2.6 samples/s +Step 10460/11700 | Loss: 2.4901 | LR: 1.57e-05 | 7600.5ms/step | 2.6 samples/s +Step 10470/11700 | Loss: 2.3774 | LR: 1.54e-05 | 7600.5ms/step | 2.6 samples/s +Step 10480/11700 | Loss: 2.4175 | LR: 1.52e-05 | 7609.4ms/step | 2.6 samples/s +Step 10490/11700 | Loss: 2.3597 | LR: 1.49e-05 | 7600.3ms/step | 2.6 samples/s +Step 10500/11700 | Loss: 2.4478 | LR: 1.47e-05 | 7600.1ms/step | 2.6 samples/s +Validation at step 10500: Loss = 2.4536, Perplexity = 11.63 +Step 10510/11700 | Loss: 2.5715 | LR: 1.44e-05 | 18946.2ms/step | 1.1 samples/s +Step 10520/11700 | Loss: 2.4730 | LR: 1.42e-05 | 7608.0ms/step | 2.6 samples/s +Step 10530/11700 | Loss: 2.4598 | LR: 1.40e-05 | 7618.8ms/step | 2.6 samples/s +Step 10540/11700 | Loss: 2.3088 | LR: 1.37e-05 | 7611.3ms/step | 2.6 samples/s +Step 10550/11700 | Loss: 2.4844 | LR: 1.35e-05 | 7615.1ms/step | 2.6 samples/s +Step 10560/11700 | Loss: 2.5850 | LR: 1.33e-05 | 7603.0ms/step | 2.6 samples/s +Step 10570/11700 | Loss: 2.3412 | LR: 1.30e-05 | 7604.3ms/step | 2.6 samples/s +Step 10580/11700 | Loss: 2.5177 | LR: 1.28e-05 | 7607.9ms/step | 2.6 samples/s +Step 10590/11700 | Loss: 2.3795 | LR: 1.26e-05 | 7607.7ms/step | 2.6 samples/s +Step 10600/11700 | Loss: 2.4693 | LR: 1.23e-05 | 7612.3ms/step | 2.6 samples/s +Step 10610/11700 | Loss: 2.4381 | LR: 1.21e-05 | 7605.9ms/step | 2.6 samples/s +Step 10620/11700 | Loss: 2.3561 | LR: 1.19e-05 | 7604.8ms/step | 2.6 samples/s +Step 10630/11700 | Loss: 2.4259 | LR: 1.17e-05 | 7607.5ms/step | 2.6 samples/s +Step 10640/11700 | Loss: 2.4098 | LR: 1.15e-05 | 7601.9ms/step | 2.6 samples/s +Step 10650/11700 | Loss: 2.4633 | LR: 1.13e-05 | 7606.1ms/step | 2.6 samples/s +Step 10660/11700 | Loss: 2.4150 | LR: 1.10e-05 | 7604.2ms/step | 2.6 samples/s +Step 10670/11700 | Loss: 2.4020 | LR: 1.08e-05 | 7607.8ms/step | 2.6 samples/s +Step 10680/11700 | Loss: 2.4774 | LR: 1.06e-05 | 7604.2ms/step | 2.6 samples/s +Step 10690/11700 | Loss: 2.4662 | LR: 1.04e-05 | 7607.8ms/step | 2.6 samples/s +Step 10700/11700 | Loss: 2.4347 | LR: 1.02e-05 | 7600.2ms/step | 2.6 samples/s +Step 10710/11700 | Loss: 2.4674 | LR: 1.00e-05 | 7604.2ms/step | 2.6 samples/s +Step 10720/11700 | Loss: 2.4029 | LR: 9.82e-06 | 7604.0ms/step | 2.6 samples/s +Step 10730/11700 | Loss: 2.5051 | LR: 9.62e-06 | 7606.1ms/step | 2.6 samples/s +Step 10740/11700 | Loss: 2.3322 | LR: 9.42e-06 | 7611.8ms/step | 2.6 samples/s +Step 10750/11700 | Loss: 2.3614 | LR: 9.23e-06 | 7608.2ms/step | 2.6 samples/s +Validation at step 10750: Loss = 2.4533, Perplexity = 11.63 +Step 10760/11700 | Loss: 2.5300 | LR: 9.04e-06 | 18932.2ms/step | 1.1 samples/s +Step 10770/11700 | Loss: 2.5045 | LR: 8.85e-06 | 7604.1ms/step | 2.6 samples/s +Step 10780/11700 | Loss: 2.5332 | LR: 8.66e-06 | 7600.0ms/step | 2.6 samples/s +Step 10790/11700 | Loss: 2.4433 | LR: 8.47e-06 | 7601.8ms/step | 2.6 samples/s +Step 10800/11700 | Loss: 2.5369 | LR: 8.29e-06 | 7604.4ms/step | 2.6 samples/s +Step 10810/11700 | Loss: 2.4351 | LR: 8.11e-06 | 7607.7ms/step | 2.6 samples/s +Step 10820/11700 | Loss: 2.5222 | LR: 7.93e-06 | 7601.9ms/step | 2.6 samples/s +Step 10830/11700 | Loss: 2.4210 | LR: 7.75e-06 | 7600.1ms/step | 2.6 samples/s +Step 10840/11700 | Loss: 2.4038 | LR: 7.57e-06 | 7600.0ms/step | 2.6 samples/s +Step 10850/11700 | Loss: 2.3539 | LR: 7.40e-06 | 7600.1ms/step | 2.6 samples/s +Step 10860/11700 | Loss: 2.3557 | LR: 7.22e-06 | 7600.2ms/step | 2.6 samples/s +Step 10870/11700 | Loss: 2.3831 | LR: 7.05e-06 | 7599.9ms/step | 2.6 samples/s +Step 10880/11700 | Loss: 2.3627 | LR: 6.89e-06 | 7602.3ms/step | 2.6 samples/s +Step 10890/11700 | Loss: 2.3494 | LR: 6.72e-06 | 7600.2ms/step | 2.6 samples/s +Step 10900/11700 | Loss: 2.4546 | LR: 6.56e-06 | 7601.8ms/step | 2.6 samples/s +Step 10910/11700 | Loss: 2.4948 | LR: 6.39e-06 | 7604.0ms/step | 2.6 samples/s +Step 10920/11700 | Loss: 2.4178 | LR: 6.23e-06 | 7608.1ms/step | 2.6 samples/s +Step 10930/11700 | Loss: 2.3399 | LR: 6.08e-06 | 7600.0ms/step | 2.6 samples/s +Step 10940/11700 | Loss: 2.3624 | LR: 5.92e-06 | 7600.0ms/step | 2.6 samples/s +Step 10950/11700 | Loss: 2.3693 | LR: 5.77e-06 | 7600.2ms/step | 2.6 samples/s +Step 10960/11700 | Loss: 2.5184 | LR: 5.61e-06 | 7600.0ms/step | 2.6 samples/s +Step 10970/11700 | Loss: 2.4757 | LR: 5.46e-06 | 7606.2ms/step | 2.6 samples/s +Step 10980/11700 | Loss: 2.4721 | LR: 5.31e-06 | 7599.9ms/step | 2.6 samples/s +Step 10990/11700 | Loss: 2.4103 | LR: 5.17e-06 | 7600.0ms/step | 2.6 samples/s +Step 11000/11700 | Loss: 2.3903 | LR: 5.02e-06 | 7600.1ms/step | 2.6 samples/s +Validation at step 11000: Loss = 2.4533, Perplexity = 11.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11000 +Step 11010/11700 | Loss: 2.3799 | LR: 4.88e-06 | 19262.2ms/step | 1.0 samples/s +Step 11020/11700 | Loss: 2.4480 | LR: 4.74e-06 | 7606.2ms/step | 2.6 samples/s +Step 11030/11700 | Loss: 2.4817 | LR: 4.60e-06 | 7606.0ms/step | 2.6 samples/s +Step 11040/11700 | Loss: 2.4731 | LR: 4.47e-06 | 7611.9ms/step | 2.6 samples/s +Step 11050/11700 | Loss: 2.4967 | LR: 4.33e-06 | 7604.8ms/step | 2.6 samples/s +Step 11060/11700 | Loss: 2.4293 | LR: 4.20e-06 | 7607.4ms/step | 2.6 samples/s +Step 11070/11700 | Loss: 2.3917 | LR: 4.07e-06 | 7604.3ms/step | 2.6 samples/s +Step 11080/11700 | Loss: 2.4512 | LR: 3.94e-06 | 7609.9ms/step | 2.6 samples/s +Step 11090/11700 | Loss: 2.4358 | LR: 3.82e-06 | 7608.0ms/step | 2.6 samples/s +Step 11100/11700 | Loss: 2.4819 | LR: 3.69e-06 | 7608.2ms/step | 2.6 samples/s +Step 11110/11700 | Loss: 2.4274 | LR: 3.57e-06 | 7608.7ms/step | 2.6 samples/s +Step 11120/11700 | Loss: 2.4722 | LR: 3.45e-06 | 7607.2ms/step | 2.6 samples/s +Step 11130/11700 | Loss: 2.5189 | LR: 3.34e-06 | 7610.1ms/step | 2.6 samples/s +Step 11140/11700 | Loss: 2.4225 | LR: 3.22e-06 | 7605.8ms/step | 2.6 samples/s +Step 11150/11700 | Loss: 2.4454 | LR: 3.11e-06 | 7606.3ms/step | 2.6 samples/s +Step 11160/11700 | Loss: 2.3913 | LR: 2.99e-06 | 7608.1ms/step | 2.6 samples/s +Step 11170/11700 | Loss: 2.5045 | LR: 2.88e-06 | 7607.9ms/step | 2.6 samples/s +Step 11180/11700 | Loss: 2.4156 | LR: 2.78e-06 | 7602.3ms/step | 2.6 samples/s +Step 11190/11700 | Loss: 2.4551 | LR: 2.67e-06 | 7600.0ms/step | 2.6 samples/s +Step 11200/11700 | Loss: 2.4489 | LR: 2.57e-06 | 7608.1ms/step | 2.6 samples/s +Step 11210/11700 | Loss: 2.4983 | LR: 2.47e-06 | 7603.6ms/step | 2.6 samples/s +Step 11220/11700 | Loss: 2.3905 | LR: 2.37e-06 | 7600.2ms/step | 2.6 samples/s +Step 11230/11700 | Loss: 2.3179 | LR: 2.27e-06 | 7600.1ms/step | 2.6 samples/s +Step 11240/11700 | Loss: 2.3857 | LR: 2.17e-06 | 7608.0ms/step | 2.6 samples/s +Step 11250/11700 | Loss: 2.4019 | LR: 2.08e-06 | 7600.4ms/step | 2.6 samples/s +Validation at step 11250: Loss = 2.4531, Perplexity = 11.62 +Step 11260/11700 | Loss: 2.4338 | LR: 1.99e-06 | 18931.7ms/step | 1.1 samples/s +Step 11270/11700 | Loss: 2.4790 | LR: 1.90e-06 | 7601.9ms/step | 2.6 samples/s +Step 11280/11700 | Loss: 2.4201 | LR: 1.81e-06 | 7600.3ms/step | 2.6 samples/s +Step 11290/11700 | Loss: 2.4886 | LR: 1.73e-06 | 7600.1ms/step | 2.6 samples/s +Step 11300/11700 | Loss: 2.4558 | LR: 1.64e-06 | 7599.9ms/step | 2.6 samples/s +Step 11310/11700 | Loss: 2.4240 | LR: 1.56e-06 | 7600.2ms/step | 2.6 samples/s +Step 11320/11700 | Loss: 2.4550 | LR: 1.48e-06 | 7602.3ms/step | 2.6 samples/s +Step 11330/11700 | Loss: 2.3810 | LR: 1.41e-06 | 7600.1ms/step | 2.6 samples/s +Step 11340/11700 | Loss: 2.3584 | LR: 1.33e-06 | 7601.7ms/step | 2.6 samples/s +Step 11350/11700 | Loss: 2.4822 | LR: 1.26e-06 | 7606.1ms/step | 2.6 samples/s +Step 11360/11700 | Loss: 2.4748 | LR: 1.19e-06 | 7600.1ms/step | 2.6 samples/s +Step 11370/11700 | Loss: 2.4345 | LR: 1.12e-06 | 7602.4ms/step | 2.6 samples/s +Step 11380/11700 | Loss: 2.4208 | LR: 1.05e-06 | 7601.6ms/step | 2.6 samples/s +Step 11390/11700 | Loss: 2.4389 | LR: 9.88e-07 | 7600.1ms/step | 2.6 samples/s +Step 11400/11700 | Loss: 2.4666 | LR: 9.25e-07 | 7600.0ms/step | 2.6 samples/s +Step 11410/11700 | Loss: 2.4413 | LR: 8.65e-07 | 7604.2ms/step | 2.6 samples/s +Step 11420/11700 | Loss: 2.2909 | LR: 8.06e-07 | 7604.8ms/step | 2.6 samples/s +Step 11430/11700 | Loss: 2.3937 | LR: 7.50e-07 | 7605.4ms/step | 2.6 samples/s +Step 11440/11700 | Loss: 2.4832 | LR: 6.95e-07 | 7604.2ms/step | 2.6 samples/s +Step 11450/11700 | Loss: 2.3296 | LR: 6.43e-07 | 7599.9ms/step | 2.6 samples/s +Step 11460/11700 | Loss: 2.4556 | LR: 5.92e-07 | 7599.9ms/step | 2.6 samples/s +Step 11470/11700 | Loss: 2.4999 | LR: 5.44e-07 | 7602.0ms/step | 2.6 samples/s +Step 11480/11700 | Loss: 2.3700 | LR: 4.98e-07 | 7602.1ms/step | 2.6 samples/s +Step 11490/11700 | Loss: 2.4707 | LR: 4.54e-07 | 7600.1ms/step | 2.6 samples/s +Step 11500/11700 | Loss: 2.4053 | LR: 4.11e-07 | 7604.0ms/step | 2.6 samples/s +Validation at step 11500: Loss = 2.4533, Perplexity = 11.63 +Step 11510/11700 | Loss: 2.3911 | LR: 3.71e-07 | 18934.2ms/step | 1.1 samples/s +Step 11520/11700 | Loss: 2.4649 | LR: 3.33e-07 | 7600.3ms/step | 2.6 samples/s +Step 11530/11700 | Loss: 2.4093 | LR: 2.97e-07 | 7601.8ms/step | 2.6 samples/s +Step 11540/11700 | Loss: 2.3863 | LR: 2.63e-07 | 7599.9ms/step | 2.6 samples/s +Step 11550/11700 | Loss: 2.5006 | LR: 2.31e-07 | 7600.2ms/step | 2.6 samples/s +Step 11560/11700 | Loss: 2.4048 | LR: 2.02e-07 | 7599.9ms/step | 2.6 samples/s +Step 11570/11700 | Loss: 2.5185 | LR: 1.74e-07 | 7603.9ms/step | 2.6 samples/s +Step 11580/11700 | Loss: 2.3550 | LR: 1.48e-07 | 7602.2ms/step | 2.6 samples/s +Step 11590/11700 | Loss: 2.4716 | LR: 1.24e-07 | 7600.1ms/step | 2.6 samples/s +Step 11600/11700 | Loss: 2.4740 | LR: 1.03e-07 | 7600.0ms/step | 2.6 samples/s +Step 11610/11700 | Loss: 2.4743 | LR: 8.33e-08 | 7602.1ms/step | 2.6 samples/s +Step 11620/11700 | Loss: 2.3911 | LR: 6.58e-08 | 7602.1ms/step | 2.6 samples/s +Step 11630/11700 | Loss: 2.3382 | LR: 5.04e-08 | 7606.9ms/step | 2.6 samples/s +Step 11640/11700 | Loss: 2.4731 | LR: 3.70e-08 | 7603.7ms/step | 2.6 samples/s +Step 11650/11700 | Loss: 2.5252 | LR: 2.57e-08 | 7603.7ms/step | 2.6 samples/s +Step 11660/11700 | Loss: 2.2996 | LR: 1.65e-08 | 7599.9ms/step | 2.6 samples/s +Step 11670/11700 | Loss: 2.4539 | LR: 9.26e-09 | 7604.3ms/step | 2.6 samples/s +Step 11680/11700 | Loss: 2.4658 | LR: 4.12e-09 | 7602.2ms/step | 2.6 samples/s +Step 11690/11700 | Loss: 2.3702 | LR: 1.03e-09 | 7604.0ms/step | 2.6 samples/s +Step 11700/11700 | Loss: 2.4349 | LR: 0.00e+00 | 7599.5ms/step | 2.6 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11700 +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▆▄▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▃▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec █▁██████████████████████████████████████ +wandb: speed/steps_per_sec ██████████████████████████████████████▁█ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.45334 +wandb: eval/perplexity 11.6271 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 7599.50585 +wandb: speed/samples_per_sec 2.63175 +wandb: speed/steps_per_sec 0.13159 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m resampled 30k nrb120 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/sh1m3lmh +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260707_035835-sh1m3lmh/logs +Training complete! diff --git a/epfl_final/logs/l8k365m_dense.log b/epfl_final/logs/l8k365m_dense.log new file mode 100644 index 0000000000000000000000000000000000000000..301fa19e01671fc722a8504c741fd6ca68e7fbd8 --- /dev/null +++ b/epfl_final/logs/l8k365m_dense.log @@ -0,0 +1,1614 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 8k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "dense" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_034612-qlqhc3gy +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m dense 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qlqhc3gy +Starting training... +Switching attention mode for step 1: dense (dense step interval) +Step 10/44000 | Loss: 12.0394 | LR: 6.67e-06 | 5987.4ms/step | 3.3 samples/s +Step 20/44000 | Loss: 11.9274 | LR: 1.33e-05 | 1357.0ms/step | 14.7 samples/s +Step 30/44000 | Loss: 11.5664 | LR: 2.00e-05 | 1355.8ms/step | 14.8 samples/s +Step 40/44000 | Loss: 11.0213 | LR: 2.67e-05 | 1356.0ms/step | 14.7 samples/s +Step 50/44000 | Loss: 10.7365 | LR: 3.33e-05 | 1353.5ms/step | 14.8 samples/s +Step 60/44000 | Loss: 10.4922 | LR: 4.00e-05 | 1353.2ms/step | 14.8 samples/s +Step 70/44000 | Loss: 10.2575 | LR: 4.67e-05 | 1351.5ms/step | 14.8 samples/s +Step 80/44000 | Loss: 9.9604 | LR: 5.33e-05 | 1350.3ms/step | 14.8 samples/s +Step 90/44000 | Loss: 9.6066 | LR: 6.00e-05 | 1351.2ms/step | 14.8 samples/s +Step 100/44000 | Loss: 9.1528 | LR: 6.67e-05 | 1355.5ms/step | 14.8 samples/s +Step 110/44000 | Loss: 8.6193 | LR: 7.33e-05 | 1357.1ms/step | 14.7 samples/s +Step 120/44000 | Loss: 8.1614 | LR: 8.00e-05 | 1361.8ms/step | 14.7 samples/s +Step 130/44000 | Loss: 7.7791 | LR: 8.67e-05 | 1362.4ms/step | 14.7 samples/s +Step 140/44000 | Loss: 7.5464 | LR: 9.33e-05 | 1365.2ms/step | 14.6 samples/s +Step 150/44000 | Loss: 7.3587 | LR: 1.00e-04 | 1367.6ms/step | 14.6 samples/s +Step 160/44000 | Loss: 7.2631 | LR: 1.07e-04 | 1369.7ms/step | 14.6 samples/s +Step 170/44000 | Loss: 7.1473 | LR: 1.13e-04 | 1371.8ms/step | 14.6 samples/s +Step 180/44000 | Loss: 7.0524 | LR: 1.20e-04 | 1372.1ms/step | 14.6 samples/s +Step 190/44000 | Loss: 6.9434 | LR: 1.27e-04 | 1373.1ms/step | 14.6 samples/s +Step 200/44000 | Loss: 6.8854 | LR: 1.33e-04 | 1375.4ms/step | 14.5 samples/s +Step 210/44000 | Loss: 6.8164 | LR: 1.40e-04 | 1376.6ms/step | 14.5 samples/s +Step 220/44000 | Loss: 6.7069 | LR: 1.47e-04 | 1378.5ms/step | 14.5 samples/s +Step 230/44000 | Loss: 6.5976 | LR: 1.53e-04 | 1379.3ms/step | 14.5 samples/s +Step 240/44000 | Loss: 6.5419 | LR: 1.60e-04 | 1376.5ms/step | 14.5 samples/s +Step 250/44000 | Loss: 6.4535 | LR: 1.67e-04 | 1381.3ms/step | 14.5 samples/s +Validation at step 250: Loss = 6.4426, Perplexity = 628.03 +Step 260/44000 | Loss: 6.4211 | LR: 1.73e-04 | 5258.8ms/step | 3.8 samples/s +Step 270/44000 | Loss: 6.3512 | LR: 1.80e-04 | 1378.5ms/step | 14.5 samples/s +Step 280/44000 | Loss: 6.3225 | LR: 1.87e-04 | 1379.3ms/step | 14.5 samples/s +Step 290/44000 | Loss: 6.3089 | LR: 1.93e-04 | 1382.3ms/step | 14.5 samples/s +Step 300/44000 | Loss: 6.2370 | LR: 2.00e-04 | 1379.8ms/step | 14.5 samples/s +Step 310/44000 | Loss: 6.2013 | LR: 2.07e-04 | 1377.2ms/step | 14.5 samples/s +Step 320/44000 | Loss: 6.1476 | LR: 2.13e-04 | 1375.9ms/step | 14.5 samples/s +Step 330/44000 | Loss: 6.0887 | LR: 2.20e-04 | 1379.3ms/step | 14.5 samples/s +Step 340/44000 | Loss: 6.0659 | LR: 2.27e-04 | 1382.3ms/step | 14.5 samples/s +Step 350/44000 | Loss: 5.9950 | LR: 2.33e-04 | 1382.7ms/step | 14.5 samples/s +Step 360/44000 | Loss: 5.9560 | LR: 2.40e-04 | 1381.6ms/step | 14.5 samples/s +Step 370/44000 | Loss: 5.9293 | LR: 2.47e-04 | 1381.9ms/step | 14.5 samples/s +Step 380/44000 | Loss: 5.9120 | LR: 2.53e-04 | 1384.1ms/step | 14.5 samples/s +Step 390/44000 | Loss: 5.8310 | LR: 2.60e-04 | 1384.7ms/step | 14.4 samples/s +Step 400/44000 | Loss: 5.8416 | LR: 2.67e-04 | 1384.4ms/step | 14.4 samples/s +Step 410/44000 | Loss: 5.7956 | LR: 2.73e-04 | 1383.4ms/step | 14.5 samples/s +Step 420/44000 | Loss: 5.8088 | LR: 2.80e-04 | 1386.5ms/step | 14.4 samples/s +Step 430/44000 | Loss: 5.7413 | LR: 2.87e-04 | 1385.6ms/step | 14.4 samples/s +Step 440/44000 | Loss: 5.7087 | LR: 2.93e-04 | 1388.0ms/step | 14.4 samples/s +Step 450/44000 | Loss: 5.6676 | LR: 3.00e-04 | 1386.9ms/step | 14.4 samples/s +Step 460/44000 | Loss: 5.6018 | LR: 3.07e-04 | 1384.8ms/step | 14.4 samples/s +Step 470/44000 | Loss: 5.6114 | LR: 3.13e-04 | 1386.5ms/step | 14.4 samples/s +Step 480/44000 | Loss: 5.5595 | LR: 3.20e-04 | 1386.1ms/step | 14.4 samples/s +Step 490/44000 | Loss: 5.5107 | LR: 3.27e-04 | 1386.7ms/step | 14.4 samples/s +Step 500/44000 | Loss: 5.4917 | LR: 3.33e-04 | 1389.2ms/step | 14.4 samples/s +Validation at step 500: Loss = 5.4876, Perplexity = 241.67 +Step 510/44000 | Loss: 5.4544 | LR: 3.40e-04 | 3584.0ms/step | 5.6 samples/s +Step 520/44000 | Loss: 5.4116 | LR: 3.47e-04 | 1389.5ms/step | 14.4 samples/s +Step 530/44000 | Loss: 5.4363 | LR: 3.53e-04 | 1388.3ms/step | 14.4 samples/s +Step 540/44000 | Loss: 5.3456 | LR: 3.60e-04 | 1392.9ms/step | 14.4 samples/s +Step 550/44000 | Loss: 5.3251 | LR: 3.67e-04 | 1391.2ms/step | 14.4 samples/s +Step 560/44000 | Loss: 5.2952 | LR: 3.73e-04 | 1391.6ms/step | 14.4 samples/s +Step 570/44000 | Loss: 5.2918 | LR: 3.80e-04 | 1389.9ms/step | 14.4 samples/s +Step 580/44000 | Loss: 5.2537 | LR: 3.87e-04 | 1391.0ms/step | 14.4 samples/s +Step 590/44000 | Loss: 5.1966 | LR: 3.93e-04 | 1391.5ms/step | 14.4 samples/s +Step 600/44000 | Loss: 5.1900 | LR: 4.00e-04 | 1388.6ms/step | 14.4 samples/s +Step 610/44000 | Loss: 5.1557 | LR: 4.07e-04 | 1391.5ms/step | 14.4 samples/s +Step 620/44000 | Loss: 5.1425 | LR: 4.13e-04 | 1391.3ms/step | 14.4 samples/s +Step 630/44000 | Loss: 5.0581 | LR: 4.20e-04 | 1390.6ms/step | 14.4 samples/s +Step 640/44000 | Loss: 5.0405 | LR: 4.27e-04 | 1390.2ms/step | 14.4 samples/s +Step 650/44000 | Loss: 5.0108 | LR: 4.33e-04 | 1391.9ms/step | 14.4 samples/s +Step 660/44000 | Loss: 4.9989 | LR: 4.40e-04 | 1392.8ms/step | 14.4 samples/s +Step 670/44000 | Loss: 4.9670 | LR: 4.47e-04 | 1390.6ms/step | 14.4 samples/s +Step 680/44000 | Loss: 4.9135 | LR: 4.53e-04 | 1384.6ms/step | 14.4 samples/s +Step 690/44000 | Loss: 4.9026 | LR: 4.60e-04 | 1385.8ms/step | 14.4 samples/s +Step 700/44000 | Loss: 4.8811 | LR: 4.67e-04 | 1384.5ms/step | 14.4 samples/s +Step 710/44000 | Loss: 4.8263 | LR: 4.73e-04 | 1393.8ms/step | 14.3 samples/s +Step 720/44000 | Loss: 4.7619 | LR: 4.80e-04 | 1389.2ms/step | 14.4 samples/s +Step 730/44000 | Loss: 4.8002 | LR: 4.87e-04 | 1387.7ms/step | 14.4 samples/s +Step 740/44000 | Loss: 4.7366 | LR: 4.93e-04 | 1386.6ms/step | 14.4 samples/s +Step 750/44000 | Loss: 4.7343 | LR: 5.00e-04 | 1387.6ms/step | 14.4 samples/s +Validation at step 750: Loss = 4.7154, Perplexity = 111.66 +Step 760/44000 | Loss: 4.7005 | LR: 5.00e-04 | 3586.7ms/step | 5.6 samples/s +Step 770/44000 | Loss: 4.6582 | LR: 5.00e-04 | 1388.1ms/step | 14.4 samples/s +Step 780/44000 | Loss: 4.6512 | LR: 5.00e-04 | 1386.7ms/step | 14.4 samples/s +Step 790/44000 | Loss: 4.6425 | LR: 5.00e-04 | 1387.6ms/step | 14.4 samples/s +Step 800/44000 | Loss: 4.5857 | LR: 5.00e-04 | 1387.4ms/step | 14.4 samples/s +Step 810/44000 | Loss: 4.5617 | LR: 5.00e-04 | 1385.7ms/step | 14.4 samples/s +Step 820/44000 | Loss: 4.5411 | LR: 5.00e-04 | 1388.3ms/step | 14.4 samples/s +Step 830/44000 | Loss: 4.5262 | LR: 5.00e-04 | 1390.4ms/step | 14.4 samples/s +Step 840/44000 | Loss: 4.4996 | LR: 5.00e-04 | 1389.5ms/step | 14.4 samples/s +Step 850/44000 | Loss: 4.4903 | LR: 5.00e-04 | 1388.6ms/step | 14.4 samples/s +Step 860/44000 | Loss: 4.5136 | LR: 5.00e-04 | 1389.2ms/step | 14.4 samples/s +Step 870/44000 | Loss: 4.4263 | LR: 5.00e-04 | 1390.4ms/step | 14.4 samples/s +Step 880/44000 | Loss: 4.4365 | LR: 5.00e-04 | 1390.0ms/step | 14.4 samples/s +Step 890/44000 | Loss: 4.4160 | LR: 5.00e-04 | 1387.9ms/step | 14.4 samples/s +Step 900/44000 | Loss: 4.3905 | LR: 5.00e-04 | 1387.4ms/step | 14.4 samples/s +Step 910/44000 | Loss: 4.3626 | LR: 5.00e-04 | 1387.2ms/step | 14.4 samples/s +Step 920/44000 | Loss: 4.3622 | LR: 5.00e-04 | 1385.7ms/step | 14.4 samples/s +Step 930/44000 | Loss: 4.3329 | LR: 5.00e-04 | 1388.3ms/step | 14.4 samples/s +Step 940/44000 | Loss: 4.3264 | LR: 5.00e-04 | 1387.6ms/step | 14.4 samples/s +Step 950/44000 | Loss: 4.3147 | LR: 5.00e-04 | 1384.5ms/step | 14.4 samples/s +Step 960/44000 | Loss: 4.2723 | LR: 5.00e-04 | 1383.6ms/step | 14.5 samples/s +Step 970/44000 | Loss: 4.3061 | LR: 5.00e-04 | 1384.7ms/step | 14.4 samples/s +Step 980/44000 | Loss: 4.2832 | LR: 5.00e-04 | 1385.4ms/step | 14.4 samples/s +Step 990/44000 | Loss: 4.2603 | LR: 5.00e-04 | 1385.9ms/step | 14.4 samples/s +Step 1000/44000 | Loss: 4.2566 | LR: 5.00e-04 | 1384.0ms/step | 14.5 samples/s +Validation at step 1000: Loss = 4.2727, Perplexity = 71.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-1000 +Step 1010/44000 | Loss: 4.2262 | LR: 5.00e-04 | 3869.9ms/step | 5.2 samples/s +Step 1020/44000 | Loss: 4.2278 | LR: 5.00e-04 | 1386.1ms/step | 14.4 samples/s +Step 1030/44000 | Loss: 4.1752 | LR: 5.00e-04 | 1386.8ms/step | 14.4 samples/s +Step 1040/44000 | Loss: 4.1853 | LR: 5.00e-04 | 1387.3ms/step | 14.4 samples/s +Step 1050/44000 | Loss: 4.1858 | LR: 5.00e-04 | 1388.2ms/step | 14.4 samples/s +Step 1060/44000 | Loss: 4.1794 | LR: 5.00e-04 | 1388.5ms/step | 14.4 samples/s +Step 1070/44000 | Loss: 4.1601 | LR: 5.00e-04 | 1387.2ms/step | 14.4 samples/s +Step 1080/44000 | Loss: 4.1453 | LR: 5.00e-04 | 1386.3ms/step | 14.4 samples/s +Step 1090/44000 | Loss: 4.1712 | LR: 5.00e-04 | 1389.5ms/step | 14.4 samples/s +Step 1100/44000 | Loss: 4.1413 | LR: 5.00e-04 | 1389.8ms/step | 14.4 samples/s +Step 1110/44000 | Loss: 4.1205 | LR: 5.00e-04 | 1389.4ms/step | 14.4 samples/s +Step 1120/44000 | Loss: 4.1080 | LR: 5.00e-04 | 1386.9ms/step | 14.4 samples/s +Step 1130/44000 | Loss: 4.0828 | LR: 5.00e-04 | 1388.9ms/step | 14.4 samples/s +Step 1140/44000 | Loss: 4.1073 | LR: 5.00e-04 | 1389.1ms/step | 14.4 samples/s +Step 1150/44000 | Loss: 4.0566 | LR: 5.00e-04 | 1388.2ms/step | 14.4 samples/s +Step 1160/44000 | Loss: 4.0566 | LR: 5.00e-04 | 1389.5ms/step | 14.4 samples/s +Step 1170/44000 | Loss: 4.0645 | LR: 5.00e-04 | 1387.7ms/step | 14.4 samples/s +Step 1180/44000 | Loss: 4.0477 | LR: 5.00e-04 | 1388.4ms/step | 14.4 samples/s +Step 1190/44000 | Loss: 4.0649 | LR: 5.00e-04 | 1386.7ms/step | 14.4 samples/s +Step 1200/44000 | Loss: 4.0559 | LR: 5.00e-04 | 1391.9ms/step | 14.4 samples/s +Step 1210/44000 | Loss: 4.0694 | LR: 5.00e-04 | 1392.7ms/step | 14.4 samples/s +Step 1220/44000 | Loss: 4.0156 | LR: 5.00e-04 | 1387.2ms/step | 14.4 samples/s +Step 1230/44000 | Loss: 4.0097 | LR: 5.00e-04 | 1386.1ms/step | 14.4 samples/s +Step 1240/44000 | Loss: 4.0126 | LR: 5.00e-04 | 1386.4ms/step | 14.4 samples/s +Step 1250/44000 | Loss: 4.0173 | LR: 5.00e-04 | 1391.4ms/step | 14.4 samples/s +Validation at step 1250: Loss = 4.0106, Perplexity = 55.18 +Step 1260/44000 | Loss: 3.9850 | LR: 5.00e-04 | 3585.8ms/step | 5.6 samples/s +Step 1270/44000 | Loss: 3.9854 | LR: 5.00e-04 | 1388.3ms/step | 14.4 samples/s +Step 1280/44000 | Loss: 3.9977 | LR: 5.00e-04 | 1386.9ms/step | 14.4 samples/s +Step 1290/44000 | Loss: 3.9420 | LR: 5.00e-04 | 1388.6ms/step | 14.4 samples/s +Step 1300/44000 | Loss: 3.9909 | LR: 5.00e-04 | 1389.2ms/step | 14.4 samples/s +Step 1310/44000 | Loss: 3.9302 | LR: 5.00e-04 | 1387.3ms/step | 14.4 samples/s +Step 1320/44000 | Loss: 3.9604 | LR: 5.00e-04 | 1389.9ms/step | 14.4 samples/s +Step 1330/44000 | Loss: 3.9675 | LR: 5.00e-04 | 1391.2ms/step | 14.4 samples/s +Step 1340/44000 | Loss: 3.9262 | LR: 5.00e-04 | 1391.3ms/step | 14.4 samples/s +Step 1350/44000 | Loss: 3.9306 | LR: 5.00e-04 | 1389.0ms/step | 14.4 samples/s +Step 1360/44000 | Loss: 3.9253 | LR: 5.00e-04 | 1390.5ms/step | 14.4 samples/s +Step 1370/44000 | Loss: 3.9045 | LR: 5.00e-04 | 1387.6ms/step | 14.4 samples/s +Step 1380/44000 | Loss: 3.9125 | LR: 5.00e-04 | 1389.3ms/step | 14.4 samples/s +Step 1390/44000 | Loss: 3.9127 | LR: 5.00e-04 | 1390.9ms/step | 14.4 samples/s +Step 1400/44000 | Loss: 3.9093 | LR: 5.00e-04 | 1390.2ms/step | 14.4 samples/s +Step 1410/44000 | Loss: 3.9005 | LR: 5.00e-04 | 1387.6ms/step | 14.4 samples/s +Step 1420/44000 | Loss: 3.8938 | LR: 5.00e-04 | 1388.2ms/step | 14.4 samples/s +Step 1430/44000 | Loss: 3.9109 | LR: 5.00e-04 | 1389.4ms/step | 14.4 samples/s +Step 1440/44000 | Loss: 3.9035 | LR: 5.00e-04 | 1388.8ms/step | 14.4 samples/s +Step 1450/44000 | Loss: 3.8905 | LR: 5.00e-04 | 1389.3ms/step | 14.4 samples/s +Step 1460/44000 | Loss: 3.8884 | LR: 5.00e-04 | 1389.4ms/step | 14.4 samples/s +Step 1470/44000 | Loss: 3.8994 | LR: 5.00e-04 | 1392.0ms/step | 14.4 samples/s +Step 1480/44000 | Loss: 3.8891 | LR: 5.00e-04 | 1390.9ms/step | 14.4 samples/s +Step 1490/44000 | Loss: 3.8863 | LR: 5.00e-04 | 1388.4ms/step | 14.4 samples/s +Step 1500/44000 | Loss: 3.8653 | LR: 5.00e-04 | 1392.9ms/step | 14.4 samples/s +Validation at step 1500: Loss = 3.8648, Perplexity = 47.70 +Step 1510/44000 | Loss: 3.8755 | LR: 5.00e-04 | 3588.8ms/step | 5.6 samples/s +Step 1520/44000 | Loss: 3.8768 | LR: 5.00e-04 | 1388.3ms/step | 14.4 samples/s +Step 1530/44000 | Loss: 3.8464 | LR: 5.00e-04 | 1389.9ms/step | 14.4 samples/s +Step 1540/44000 | Loss: 3.8589 | LR: 5.00e-04 | 1389.5ms/step | 14.4 samples/s +Step 1550/44000 | Loss: 3.8200 | LR: 5.00e-04 | 1389.4ms/step | 14.4 samples/s +Step 1560/44000 | Loss: 3.8124 | LR: 5.00e-04 | 1390.9ms/step | 14.4 samples/s +Step 1570/44000 | Loss: 3.8036 | LR: 5.00e-04 | 1391.4ms/step | 14.4 samples/s +Step 1580/44000 | Loss: 3.8268 | LR: 5.00e-04 | 1387.7ms/step | 14.4 samples/s +Step 1590/44000 | Loss: 3.7999 | LR: 5.00e-04 | 1392.7ms/step | 14.4 samples/s +Step 1600/44000 | Loss: 3.8075 | LR: 5.00e-04 | 1389.8ms/step | 14.4 samples/s +Step 1610/44000 | Loss: 3.7884 | LR: 5.00e-04 | 1390.5ms/step | 14.4 samples/s +Step 1620/44000 | Loss: 3.7717 | LR: 5.00e-04 | 1389.9ms/step | 14.4 samples/s +Step 1630/44000 | Loss: 3.7875 | LR: 4.99e-04 | 1388.2ms/step | 14.4 samples/s +Step 1640/44000 | Loss: 3.7908 | LR: 4.99e-04 | 1389.7ms/step | 14.4 samples/s +Step 1650/44000 | Loss: 3.7661 | LR: 4.99e-04 | 1387.6ms/step | 14.4 samples/s +Step 1660/44000 | Loss: 3.7715 | LR: 4.99e-04 | 1388.5ms/step | 14.4 samples/s +Step 1670/44000 | Loss: 3.7619 | LR: 4.99e-04 | 1386.8ms/step | 14.4 samples/s +Step 1680/44000 | Loss: 3.7667 | LR: 4.99e-04 | 1389.3ms/step | 14.4 samples/s +Step 1690/44000 | Loss: 3.7547 | LR: 4.99e-04 | 1388.1ms/step | 14.4 samples/s +Step 1700/44000 | Loss: 3.7930 | LR: 4.99e-04 | 1387.4ms/step | 14.4 samples/s +Step 1710/44000 | Loss: 3.7448 | LR: 4.99e-04 | 1386.4ms/step | 14.4 samples/s +Step 1720/44000 | Loss: 3.7523 | LR: 4.99e-04 | 1384.3ms/step | 14.4 samples/s +Step 1730/44000 | Loss: 3.7528 | LR: 4.99e-04 | 1388.1ms/step | 14.4 samples/s +Step 1740/44000 | Loss: 3.7188 | LR: 4.99e-04 | 1392.6ms/step | 14.4 samples/s +Step 1750/44000 | Loss: 3.7390 | LR: 4.99e-04 | 1388.0ms/step | 14.4 samples/s +Validation at step 1750: Loss = 3.7551, Perplexity = 42.74 +Step 1760/44000 | Loss: 3.7234 | LR: 4.99e-04 | 3590.3ms/step | 5.6 samples/s +Step 1770/44000 | Loss: 3.7230 | LR: 4.99e-04 | 1389.0ms/step | 14.4 samples/s +Step 1780/44000 | Loss: 3.7600 | LR: 4.99e-04 | 1388.1ms/step | 14.4 samples/s +Step 1790/44000 | Loss: 3.7204 | LR: 4.99e-04 | 1388.7ms/step | 14.4 samples/s +Step 1800/44000 | Loss: 3.7204 | LR: 4.99e-04 | 1388.6ms/step | 14.4 samples/s +Step 1810/44000 | Loss: 3.7360 | LR: 4.99e-04 | 1387.9ms/step | 14.4 samples/s +Step 1820/44000 | Loss: 3.7282 | LR: 4.99e-04 | 1388.1ms/step | 14.4 samples/s +Step 1830/44000 | Loss: 3.7158 | LR: 4.99e-04 | 1388.2ms/step | 14.4 samples/s +Step 1840/44000 | Loss: 3.6989 | LR: 4.99e-04 | 1390.3ms/step | 14.4 samples/s +Step 1850/44000 | Loss: 3.7270 | LR: 4.99e-04 | 1388.3ms/step | 14.4 samples/s +Step 1860/44000 | Loss: 3.7127 | LR: 4.99e-04 | 1392.2ms/step | 14.4 samples/s +Step 1870/44000 | Loss: 3.7179 | LR: 4.99e-04 | 1387.4ms/step | 14.4 samples/s +Step 1880/44000 | Loss: 3.7167 | LR: 4.99e-04 | 1380.3ms/step | 14.5 samples/s +Step 1890/44000 | Loss: 3.7099 | LR: 4.99e-04 | 1381.5ms/step | 14.5 samples/s +Step 1900/44000 | Loss: 3.7197 | LR: 4.99e-04 | 1383.8ms/step | 14.5 samples/s +Step 1910/44000 | Loss: 3.6896 | LR: 4.99e-04 | 1384.7ms/step | 14.4 samples/s +Step 1920/44000 | Loss: 3.7083 | LR: 4.99e-04 | 1386.8ms/step | 14.4 samples/s +Step 1930/44000 | Loss: 3.7013 | LR: 4.99e-04 | 1386.9ms/step | 14.4 samples/s +Step 1940/44000 | Loss: 3.6969 | LR: 4.99e-04 | 1386.5ms/step | 14.4 samples/s +Step 1950/44000 | Loss: 3.6947 | LR: 4.99e-04 | 1389.6ms/step | 14.4 samples/s +Step 1960/44000 | Loss: 3.6834 | LR: 4.99e-04 | 1386.3ms/step | 14.4 samples/s +Step 1970/44000 | Loss: 3.6871 | LR: 4.99e-04 | 1388.2ms/step | 14.4 samples/s +Step 1980/44000 | Loss: 3.6656 | LR: 4.99e-04 | 1387.9ms/step | 14.4 samples/s +Step 1990/44000 | Loss: 3.6476 | LR: 4.99e-04 | 1387.0ms/step | 14.4 samples/s +Step 2000/44000 | Loss: 3.6577 | LR: 4.99e-04 | 1386.9ms/step | 14.4 samples/s +Validation at step 2000: Loss = 3.6756, Perplexity = 39.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-2000 +Step 2010/44000 | Loss: 3.6347 | LR: 4.99e-04 | 3878.2ms/step | 5.2 samples/s +Step 2020/44000 | Loss: 3.6960 | LR: 4.99e-04 | 1386.8ms/step | 14.4 samples/s +Step 2030/44000 | Loss: 3.6717 | LR: 4.99e-04 | 1386.0ms/step | 14.4 samples/s +Step 2040/44000 | Loss: 3.6570 | LR: 4.99e-04 | 1384.6ms/step | 14.4 samples/s +Step 2050/44000 | Loss: 3.6443 | LR: 4.99e-04 | 1386.3ms/step | 14.4 samples/s +Step 2060/44000 | Loss: 3.6162 | LR: 4.99e-04 | 1386.1ms/step | 14.4 samples/s +Step 2070/44000 | Loss: 3.6375 | LR: 4.99e-04 | 1387.7ms/step | 14.4 samples/s +Step 2080/44000 | Loss: 3.6080 | LR: 4.99e-04 | 1388.2ms/step | 14.4 samples/s +Step 2090/44000 | Loss: 3.6475 | LR: 4.99e-04 | 1390.1ms/step | 14.4 samples/s +Step 2100/44000 | Loss: 3.6393 | LR: 4.99e-04 | 1389.7ms/step | 14.4 samples/s +Step 2110/44000 | Loss: 3.6547 | LR: 4.99e-04 | 1389.9ms/step | 14.4 samples/s +Step 2120/44000 | Loss: 3.6426 | LR: 4.99e-04 | 1388.0ms/step | 14.4 samples/s +Step 2130/44000 | Loss: 3.6516 | LR: 4.99e-04 | 1391.0ms/step | 14.4 samples/s +Step 2140/44000 | Loss: 3.6007 | LR: 4.99e-04 | 1390.2ms/step | 14.4 samples/s +Step 2150/44000 | Loss: 3.6146 | LR: 4.99e-04 | 1389.9ms/step | 14.4 samples/s +Step 2160/44000 | Loss: 3.6217 | LR: 4.99e-04 | 1390.4ms/step | 14.4 samples/s +Step 2170/44000 | Loss: 3.6251 | LR: 4.99e-04 | 1388.5ms/step | 14.4 samples/s +Step 2180/44000 | Loss: 3.6105 | LR: 4.99e-04 | 1388.6ms/step | 14.4 samples/s +Step 2190/44000 | Loss: 3.6203 | LR: 4.99e-04 | 1391.3ms/step | 14.4 samples/s +Step 2200/44000 | Loss: 3.5920 | LR: 4.99e-04 | 1388.4ms/step | 14.4 samples/s +Step 2210/44000 | Loss: 3.5931 | LR: 4.99e-04 | 1385.9ms/step | 14.4 samples/s +Step 2220/44000 | Loss: 3.5748 | LR: 4.99e-04 | 1387.8ms/step | 14.4 samples/s +Step 2230/44000 | Loss: 3.6180 | LR: 4.99e-04 | 1389.3ms/step | 14.4 samples/s +Step 2240/44000 | Loss: 3.5992 | LR: 4.99e-04 | 1387.5ms/step | 14.4 samples/s +Step 2250/44000 | Loss: 3.6017 | LR: 4.99e-04 | 1388.4ms/step | 14.4 samples/s +Validation at step 2250: Loss = 3.6131, Perplexity = 37.08 +Step 2260/44000 | Loss: 3.5979 | LR: 4.98e-04 | 3591.3ms/step | 5.6 samples/s +Step 2270/44000 | Loss: 3.5871 | LR: 4.98e-04 | 1391.5ms/step | 14.4 samples/s +Step 2280/44000 | Loss: 3.6106 | LR: 4.98e-04 | 1388.9ms/step | 14.4 samples/s +Step 2290/44000 | Loss: 3.6016 | LR: 4.98e-04 | 1393.8ms/step | 14.3 samples/s +Step 2300/44000 | Loss: 3.5804 | LR: 4.98e-04 | 1393.9ms/step | 14.3 samples/s +Step 2310/44000 | Loss: 3.5954 | LR: 4.98e-04 | 1388.8ms/step | 14.4 samples/s +Step 2320/44000 | Loss: 3.5836 | LR: 4.98e-04 | 1388.4ms/step | 14.4 samples/s +Step 2330/44000 | Loss: 3.5858 | LR: 4.98e-04 | 1389.0ms/step | 14.4 samples/s +Step 2340/44000 | Loss: 3.5626 | LR: 4.98e-04 | 1389.2ms/step | 14.4 samples/s +Step 2350/44000 | Loss: 3.5736 | LR: 4.98e-04 | 1388.5ms/step | 14.4 samples/s +Step 2360/44000 | Loss: 3.5571 | LR: 4.98e-04 | 1387.6ms/step | 14.4 samples/s +Step 2370/44000 | Loss: 3.5964 | LR: 4.98e-04 | 1388.6ms/step | 14.4 samples/s +Step 2380/44000 | Loss: 3.5773 | LR: 4.98e-04 | 1389.6ms/step | 14.4 samples/s +Step 2390/44000 | Loss: 3.5426 | LR: 4.98e-04 | 1387.8ms/step | 14.4 samples/s +Step 2400/44000 | Loss: 3.5415 | LR: 4.98e-04 | 1390.7ms/step | 14.4 samples/s +Step 2410/44000 | Loss: 3.5919 | LR: 4.98e-04 | 1391.4ms/step | 14.4 samples/s +Step 2420/44000 | Loss: 3.5997 | LR: 4.98e-04 | 1390.0ms/step | 14.4 samples/s +Step 2430/44000 | Loss: 3.5662 | LR: 4.98e-04 | 1388.5ms/step | 14.4 samples/s +Step 2440/44000 | Loss: 3.5420 | LR: 4.98e-04 | 1387.7ms/step | 14.4 samples/s +Step 2450/44000 | Loss: 3.5709 | LR: 4.98e-04 | 1388.4ms/step | 14.4 samples/s +Step 2460/44000 | Loss: 3.5640 | LR: 4.98e-04 | 1387.9ms/step | 14.4 samples/s +Step 2470/44000 | Loss: 3.5440 | LR: 4.98e-04 | 1387.8ms/step | 14.4 samples/s +Step 2480/44000 | Loss: 3.5751 | LR: 4.98e-04 | 1389.4ms/step | 14.4 samples/s +Step 2490/44000 | Loss: 3.5323 | LR: 4.98e-04 | 1390.6ms/step | 14.4 samples/s +Step 2500/44000 | Loss: 3.5455 | LR: 4.98e-04 | 1386.3ms/step | 14.4 samples/s +Validation at step 2500: Loss = 3.5627, Perplexity = 35.26 +Step 2510/44000 | Loss: 3.5301 | LR: 4.98e-04 | 3588.7ms/step | 5.6 samples/s +Step 2520/44000 | Loss: 3.5472 | LR: 4.98e-04 | 1391.0ms/step | 14.4 samples/s +Step 2530/44000 | Loss: 3.5474 | LR: 4.98e-04 | 1389.4ms/step | 14.4 samples/s +Step 2540/44000 | Loss: 3.5613 | LR: 4.98e-04 | 1389.0ms/step | 14.4 samples/s +Step 2550/44000 | Loss: 3.5493 | LR: 4.98e-04 | 1388.2ms/step | 14.4 samples/s +Step 2560/44000 | Loss: 3.5601 | LR: 4.98e-04 | 1387.5ms/step | 14.4 samples/s +Step 2570/44000 | Loss: 3.5313 | LR: 4.98e-04 | 1384.4ms/step | 14.4 samples/s +Step 2580/44000 | Loss: 3.5708 | LR: 4.98e-04 | 1386.0ms/step | 14.4 samples/s +Step 2590/44000 | Loss: 3.5251 | LR: 4.98e-04 | 1387.6ms/step | 14.4 samples/s +Step 2600/44000 | Loss: 3.5342 | LR: 4.98e-04 | 1388.1ms/step | 14.4 samples/s +Step 2610/44000 | Loss: 3.5274 | LR: 4.98e-04 | 1386.0ms/step | 14.4 samples/s +Step 2620/44000 | Loss: 3.5313 | LR: 4.98e-04 | 1388.9ms/step | 14.4 samples/s +Step 2630/44000 | Loss: 3.5223 | LR: 4.98e-04 | 1388.5ms/step | 14.4 samples/s +Step 2640/44000 | Loss: 3.5352 | LR: 4.98e-04 | 1388.7ms/step | 14.4 samples/s +Step 2650/44000 | Loss: 3.5526 | LR: 4.98e-04 | 1389.3ms/step | 14.4 samples/s +Step 2660/44000 | Loss: 3.5220 | LR: 4.98e-04 | 1387.6ms/step | 14.4 samples/s +Step 2670/44000 | Loss: 3.5104 | LR: 4.98e-04 | 1388.5ms/step | 14.4 samples/s +Step 2680/44000 | Loss: 3.5324 | LR: 4.98e-04 | 1387.1ms/step | 14.4 samples/s +Step 2690/44000 | Loss: 3.5239 | LR: 4.98e-04 | 1386.9ms/step | 14.4 samples/s +Step 2700/44000 | Loss: 3.5560 | LR: 4.97e-04 | 1388.5ms/step | 14.4 samples/s +Step 2710/44000 | Loss: 3.5202 | LR: 4.97e-04 | 1390.7ms/step | 14.4 samples/s +Step 2720/44000 | Loss: 3.5213 | LR: 4.97e-04 | 1387.6ms/step | 14.4 samples/s +Step 2730/44000 | Loss: 3.5173 | LR: 4.97e-04 | 1387.7ms/step | 14.4 samples/s +Step 2740/44000 | Loss: 3.5175 | LR: 4.97e-04 | 1390.6ms/step | 14.4 samples/s +Step 2750/44000 | Loss: 3.5068 | LR: 4.97e-04 | 1388.6ms/step | 14.4 samples/s +Validation at step 2750: Loss = 3.5222, Perplexity = 33.86 +Step 2760/44000 | Loss: 3.4792 | LR: 4.97e-04 | 3593.1ms/step | 5.6 samples/s +Step 2770/44000 | Loss: 3.5035 | LR: 4.97e-04 | 1388.2ms/step | 14.4 samples/s +Step 2780/44000 | Loss: 3.4823 | LR: 4.97e-04 | 1388.5ms/step | 14.4 samples/s +Step 2790/44000 | Loss: 3.5059 | LR: 4.97e-04 | 1388.5ms/step | 14.4 samples/s +Step 2800/44000 | Loss: 3.4978 | LR: 4.97e-04 | 1391.1ms/step | 14.4 samples/s +Step 2810/44000 | Loss: 3.4825 | LR: 4.97e-04 | 1392.8ms/step | 14.4 samples/s +Step 2820/44000 | Loss: 3.4929 | LR: 4.97e-04 | 1385.1ms/step | 14.4 samples/s +Step 2830/44000 | Loss: 3.4707 | LR: 4.97e-04 | 1387.9ms/step | 14.4 samples/s +Step 2840/44000 | Loss: 3.4802 | LR: 4.97e-04 | 1388.1ms/step | 14.4 samples/s +Step 2850/44000 | Loss: 3.4886 | LR: 4.97e-04 | 1387.7ms/step | 14.4 samples/s +Step 2860/44000 | Loss: 3.4824 | LR: 4.97e-04 | 1387.2ms/step | 14.4 samples/s +Step 2870/44000 | Loss: 3.4929 | LR: 4.97e-04 | 1387.2ms/step | 14.4 samples/s +Step 2880/44000 | Loss: 3.4538 | LR: 4.97e-04 | 1386.1ms/step | 14.4 samples/s +Step 2890/44000 | Loss: 3.4996 | LR: 4.97e-04 | 1390.0ms/step | 14.4 samples/s +Step 2900/44000 | Loss: 3.4992 | LR: 4.97e-04 | 1389.4ms/step | 14.4 samples/s +Step 2910/44000 | Loss: 3.4701 | LR: 4.97e-04 | 1389.5ms/step | 14.4 samples/s +Step 2920/44000 | Loss: 3.4617 | LR: 4.97e-04 | 1385.5ms/step | 14.4 samples/s +Step 2930/44000 | Loss: 3.4550 | LR: 4.97e-04 | 1380.1ms/step | 14.5 samples/s +Step 2940/44000 | Loss: 3.4759 | LR: 4.97e-04 | 1383.6ms/step | 14.5 samples/s +Step 2950/44000 | Loss: 3.4766 | LR: 4.97e-04 | 1384.8ms/step | 14.4 samples/s +Step 2960/44000 | Loss: 3.4895 | LR: 4.97e-04 | 1385.5ms/step | 14.4 samples/s +Step 2970/44000 | Loss: 3.4735 | LR: 4.97e-04 | 1386.3ms/step | 14.4 samples/s +Step 2980/44000 | Loss: 3.4767 | LR: 4.97e-04 | 1386.9ms/step | 14.4 samples/s +Step 2990/44000 | Loss: 3.4642 | LR: 4.97e-04 | 1387.4ms/step | 14.4 samples/s +Step 3000/44000 | Loss: 3.4938 | LR: 4.97e-04 | 1386.8ms/step | 14.4 samples/s +Validation at step 3000: Loss = 3.4828, Perplexity = 32.55 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-3000 +Step 3010/44000 | Loss: 3.4727 | LR: 4.97e-04 | 3881.6ms/step | 5.2 samples/s +Step 3020/44000 | Loss: 3.4560 | LR: 4.97e-04 | 1389.9ms/step | 14.4 samples/s +Step 3030/44000 | Loss: 3.4736 | LR: 4.97e-04 | 1390.4ms/step | 14.4 samples/s +Step 3040/44000 | Loss: 3.4350 | LR: 4.97e-04 | 1388.1ms/step | 14.4 samples/s +Step 3050/44000 | Loss: 3.4623 | LR: 4.97e-04 | 1390.0ms/step | 14.4 samples/s +Step 3060/44000 | Loss: 3.4324 | LR: 4.96e-04 | 1389.6ms/step | 14.4 samples/s +Step 3070/44000 | Loss: 3.4811 | LR: 4.96e-04 | 1390.5ms/step | 14.4 samples/s +Step 3080/44000 | Loss: 3.4997 | LR: 4.96e-04 | 1389.9ms/step | 14.4 samples/s +Step 3090/44000 | Loss: 3.4879 | LR: 4.96e-04 | 1388.0ms/step | 14.4 samples/s +Step 3100/44000 | Loss: 3.4470 | LR: 4.96e-04 | 1387.9ms/step | 14.4 samples/s +Step 3110/44000 | Loss: 3.4506 | LR: 4.96e-04 | 1387.7ms/step | 14.4 samples/s +Step 3120/44000 | Loss: 3.4685 | LR: 4.96e-04 | 1388.4ms/step | 14.4 samples/s +Step 3130/44000 | Loss: 3.4389 | LR: 4.96e-04 | 1388.4ms/step | 14.4 samples/s +Step 3140/44000 | Loss: 3.4574 | LR: 4.96e-04 | 1390.6ms/step | 14.4 samples/s +Step 3150/44000 | Loss: 3.4495 | LR: 4.96e-04 | 1388.8ms/step | 14.4 samples/s +Step 3160/44000 | Loss: 3.4741 | LR: 4.96e-04 | 1387.9ms/step | 14.4 samples/s +Step 3170/44000 | Loss: 3.4144 | LR: 4.96e-04 | 1387.8ms/step | 14.4 samples/s +Step 3180/44000 | Loss: 3.4577 | LR: 4.96e-04 | 1387.6ms/step | 14.4 samples/s +Step 3190/44000 | Loss: 3.4443 | LR: 4.96e-04 | 1387.5ms/step | 14.4 samples/s +Step 3200/44000 | Loss: 3.4699 | LR: 4.96e-04 | 1387.5ms/step | 14.4 samples/s +Step 3210/44000 | Loss: 3.4419 | LR: 4.96e-04 | 1387.9ms/step | 14.4 samples/s +Step 3220/44000 | Loss: 3.4450 | LR: 4.96e-04 | 1388.7ms/step | 14.4 samples/s +Step 3230/44000 | Loss: 3.4424 | LR: 4.96e-04 | 1389.3ms/step | 14.4 samples/s +Step 3240/44000 | Loss: 3.4385 | LR: 4.96e-04 | 1389.9ms/step | 14.4 samples/s +Step 3250/44000 | Loss: 3.4416 | LR: 4.96e-04 | 1387.9ms/step | 14.4 samples/s +Validation at step 3250: Loss = 3.4496, Perplexity = 31.49 +Step 3260/44000 | Loss: 3.4611 | LR: 4.96e-04 | 3586.8ms/step | 5.6 samples/s +Step 3270/44000 | Loss: 3.4261 | LR: 4.96e-04 | 1389.0ms/step | 14.4 samples/s +Step 3280/44000 | Loss: 3.4148 | LR: 4.96e-04 | 1388.9ms/step | 14.4 samples/s +Step 3290/44000 | Loss: 3.4352 | LR: 4.96e-04 | 1391.3ms/step | 14.4 samples/s +Step 3300/44000 | Loss: 3.4614 | LR: 4.96e-04 | 1389.8ms/step | 14.4 samples/s +Step 3310/44000 | Loss: 3.4374 | LR: 4.96e-04 | 1386.8ms/step | 14.4 samples/s +Step 3320/44000 | Loss: 3.4586 | LR: 4.96e-04 | 1387.9ms/step | 14.4 samples/s +Step 3330/44000 | Loss: 3.4619 | LR: 4.96e-04 | 1389.8ms/step | 14.4 samples/s +Step 3340/44000 | Loss: 3.4192 | LR: 4.96e-04 | 1389.6ms/step | 14.4 samples/s +Step 3350/44000 | Loss: 3.4336 | LR: 4.96e-04 | 1389.1ms/step | 14.4 samples/s +Step 3360/44000 | Loss: 3.4167 | LR: 4.96e-04 | 1387.4ms/step | 14.4 samples/s +Step 3370/44000 | Loss: 3.3874 | LR: 4.95e-04 | 1388.6ms/step | 14.4 samples/s +Step 3380/44000 | Loss: 3.4362 | LR: 4.95e-04 | 1387.8ms/step | 14.4 samples/s +Step 3390/44000 | Loss: 3.4150 | LR: 4.95e-04 | 1391.5ms/step | 14.4 samples/s +Step 3400/44000 | Loss: 3.4486 | LR: 4.95e-04 | 1389.3ms/step | 14.4 samples/s +Step 3410/44000 | Loss: 3.4229 | LR: 4.95e-04 | 1387.8ms/step | 14.4 samples/s +Step 3420/44000 | Loss: 3.4332 | LR: 4.95e-04 | 1386.4ms/step | 14.4 samples/s +Step 3430/44000 | Loss: 3.4231 | LR: 4.95e-04 | 1388.8ms/step | 14.4 samples/s +Step 3440/44000 | Loss: 3.4163 | LR: 4.95e-04 | 1388.3ms/step | 14.4 samples/s +Step 3450/44000 | Loss: 3.3995 | LR: 4.95e-04 | 1389.6ms/step | 14.4 samples/s +Step 3460/44000 | Loss: 3.4378 | LR: 4.95e-04 | 1387.6ms/step | 14.4 samples/s +Step 3470/44000 | Loss: 3.4115 | LR: 4.95e-04 | 1389.3ms/step | 14.4 samples/s +Step 3480/44000 | Loss: 3.4744 | LR: 4.95e-04 | 1387.3ms/step | 14.4 samples/s +Step 3490/44000 | Loss: 3.3887 | LR: 4.95e-04 | 1387.9ms/step | 14.4 samples/s +Step 3500/44000 | Loss: 3.3950 | LR: 4.95e-04 | 1388.4ms/step | 14.4 samples/s +Validation at step 3500: Loss = 3.4228, Perplexity = 30.66 +Step 3510/44000 | Loss: 3.4265 | LR: 4.95e-04 | 3583.2ms/step | 5.6 samples/s +Step 3520/44000 | Loss: 3.4139 | LR: 4.95e-04 | 1390.0ms/step | 14.4 samples/s +Step 3530/44000 | Loss: 3.4127 | LR: 4.95e-04 | 1390.3ms/step | 14.4 samples/s +Step 3540/44000 | Loss: 3.4198 | LR: 4.95e-04 | 1389.0ms/step | 14.4 samples/s +Step 3550/44000 | Loss: 3.3960 | LR: 4.95e-04 | 1389.2ms/step | 14.4 samples/s +Step 3560/44000 | Loss: 3.3830 | LR: 4.95e-04 | 1389.8ms/step | 14.4 samples/s +Step 3570/44000 | Loss: 3.4069 | LR: 4.95e-04 | 1390.0ms/step | 14.4 samples/s +Step 3580/44000 | Loss: 3.3876 | LR: 4.95e-04 | 1390.9ms/step | 14.4 samples/s +Step 3590/44000 | Loss: 3.4012 | LR: 4.95e-04 | 1389.1ms/step | 14.4 samples/s +Step 3600/44000 | Loss: 3.3886 | LR: 4.95e-04 | 1388.2ms/step | 14.4 samples/s +Step 3610/44000 | Loss: 3.3993 | LR: 4.95e-04 | 1387.9ms/step | 14.4 samples/s +Step 3620/44000 | Loss: 3.4004 | LR: 4.95e-04 | 1388.7ms/step | 14.4 samples/s +Step 3630/44000 | Loss: 3.3899 | LR: 4.95e-04 | 1387.1ms/step | 14.4 samples/s +Step 3640/44000 | Loss: 3.4025 | LR: 4.95e-04 | 1386.4ms/step | 14.4 samples/s +Step 3650/44000 | Loss: 3.3914 | LR: 4.94e-04 | 1389.7ms/step | 14.4 samples/s +Step 3660/44000 | Loss: 3.3890 | LR: 4.94e-04 | 1389.1ms/step | 14.4 samples/s +Step 3670/44000 | Loss: 3.3829 | LR: 4.94e-04 | 1381.4ms/step | 14.5 samples/s +Step 3680/44000 | Loss: 3.3949 | LR: 4.94e-04 | 1381.5ms/step | 14.5 samples/s +Step 3690/44000 | Loss: 3.3890 | LR: 4.94e-04 | 1382.8ms/step | 14.5 samples/s +Step 3700/44000 | Loss: 3.4039 | LR: 4.94e-04 | 1382.5ms/step | 14.5 samples/s +Step 3710/44000 | Loss: 3.3975 | LR: 4.94e-04 | 1382.9ms/step | 14.5 samples/s +Step 3720/44000 | Loss: 3.3509 | LR: 4.94e-04 | 1383.4ms/step | 14.5 samples/s +Step 3730/44000 | Loss: 3.4008 | LR: 4.94e-04 | 1386.1ms/step | 14.4 samples/s +Step 3740/44000 | Loss: 3.3959 | LR: 4.94e-04 | 1385.1ms/step | 14.4 samples/s +Step 3750/44000 | Loss: 3.4030 | LR: 4.94e-04 | 1387.3ms/step | 14.4 samples/s +Validation at step 3750: Loss = 3.3962, Perplexity = 29.85 +Step 3760/44000 | Loss: 3.3816 | LR: 4.94e-04 | 3585.0ms/step | 5.6 samples/s +Step 3770/44000 | Loss: 3.3755 | LR: 4.94e-04 | 1386.5ms/step | 14.4 samples/s +Step 3780/44000 | Loss: 3.3714 | LR: 4.94e-04 | 1387.0ms/step | 14.4 samples/s +Step 3790/44000 | Loss: 3.3955 | LR: 4.94e-04 | 1386.7ms/step | 14.4 samples/s +Step 3800/44000 | Loss: 3.3763 | LR: 4.94e-04 | 1386.7ms/step | 14.4 samples/s +Step 3810/44000 | Loss: 3.3815 | LR: 4.94e-04 | 1385.7ms/step | 14.4 samples/s +Step 3820/44000 | Loss: 3.3872 | LR: 4.94e-04 | 1387.7ms/step | 14.4 samples/s +Step 3830/44000 | Loss: 3.3793 | LR: 4.94e-04 | 1386.6ms/step | 14.4 samples/s +Step 3840/44000 | Loss: 3.3653 | LR: 4.94e-04 | 1388.3ms/step | 14.4 samples/s +Step 3850/44000 | Loss: 3.3735 | LR: 4.94e-04 | 1387.5ms/step | 14.4 samples/s +Step 3860/44000 | Loss: 3.3740 | LR: 4.94e-04 | 1383.8ms/step | 14.5 samples/s +Step 3870/44000 | Loss: 3.3783 | LR: 4.94e-04 | 1385.1ms/step | 14.4 samples/s +Step 3880/44000 | Loss: 3.3477 | LR: 4.94e-04 | 1387.9ms/step | 14.4 samples/s +Step 3890/44000 | Loss: 3.4035 | LR: 4.94e-04 | 1388.1ms/step | 14.4 samples/s +Step 3900/44000 | Loss: 3.3626 | LR: 4.93e-04 | 1387.2ms/step | 14.4 samples/s +Step 3910/44000 | Loss: 3.3538 | LR: 4.93e-04 | 1387.3ms/step | 14.4 samples/s +Step 3920/44000 | Loss: 3.3574 | LR: 4.93e-04 | 1387.2ms/step | 14.4 samples/s +Step 3930/44000 | Loss: 3.3670 | LR: 4.93e-04 | 1387.8ms/step | 14.4 samples/s +Step 3940/44000 | Loss: 3.3666 | LR: 4.93e-04 | 1390.0ms/step | 14.4 samples/s +Step 3950/44000 | Loss: 3.3938 | LR: 4.93e-04 | 1390.1ms/step | 14.4 samples/s +Step 3960/44000 | Loss: 3.3851 | LR: 4.93e-04 | 1388.1ms/step | 14.4 samples/s +Step 3970/44000 | Loss: 3.3374 | LR: 4.93e-04 | 1387.7ms/step | 14.4 samples/s +Step 3980/44000 | Loss: 3.3497 | LR: 4.93e-04 | 1387.1ms/step | 14.4 samples/s +Step 3990/44000 | Loss: 3.3785 | LR: 4.93e-04 | 1389.7ms/step | 14.4 samples/s +Step 4000/44000 | Loss: 3.3596 | LR: 4.93e-04 | 1389.5ms/step | 14.4 samples/s +Validation at step 4000: Loss = 3.3751, Perplexity = 29.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-4000 +Step 4010/44000 | Loss: 3.4129 | LR: 4.93e-04 | 3871.7ms/step | 5.2 samples/s +Step 4020/44000 | Loss: 3.3705 | LR: 4.93e-04 | 1387.5ms/step | 14.4 samples/s +Step 4030/44000 | Loss: 3.3457 | LR: 4.93e-04 | 1385.3ms/step | 14.4 samples/s +Step 4040/44000 | Loss: 3.3628 | LR: 4.93e-04 | 1387.2ms/step | 14.4 samples/s +Step 4050/44000 | Loss: 3.3983 | LR: 4.93e-04 | 1388.8ms/step | 14.4 samples/s +Step 4060/44000 | Loss: 3.3456 | LR: 4.93e-04 | 1389.5ms/step | 14.4 samples/s +Step 4070/44000 | Loss: 3.3655 | LR: 4.93e-04 | 1388.8ms/step | 14.4 samples/s +Step 4080/44000 | Loss: 3.3799 | LR: 4.93e-04 | 1388.1ms/step | 14.4 samples/s +Step 4090/44000 | Loss: 3.3362 | LR: 4.93e-04 | 1387.2ms/step | 14.4 samples/s +Step 4100/44000 | Loss: 3.3655 | LR: 4.93e-04 | 1389.0ms/step | 14.4 samples/s +Step 4110/44000 | Loss: 3.3348 | LR: 4.93e-04 | 1388.1ms/step | 14.4 samples/s +Step 4120/44000 | Loss: 3.3646 | LR: 4.93e-04 | 1389.9ms/step | 14.4 samples/s +Step 4130/44000 | Loss: 3.3639 | LR: 4.93e-04 | 1388.8ms/step | 14.4 samples/s +Step 4140/44000 | Loss: 3.3439 | LR: 4.92e-04 | 1390.2ms/step | 14.4 samples/s +Step 4150/44000 | Loss: 3.3913 | LR: 4.92e-04 | 1391.9ms/step | 14.4 samples/s +Step 4160/44000 | Loss: 3.3700 | LR: 4.92e-04 | 1389.1ms/step | 14.4 samples/s +Step 4170/44000 | Loss: 3.3535 | LR: 4.92e-04 | 1388.8ms/step | 14.4 samples/s +Step 4180/44000 | Loss: 3.3343 | LR: 4.92e-04 | 1391.3ms/step | 14.4 samples/s +Step 4190/44000 | Loss: 3.3463 | LR: 4.92e-04 | 1390.0ms/step | 14.4 samples/s +Step 4200/44000 | Loss: 3.3416 | LR: 4.92e-04 | 1388.3ms/step | 14.4 samples/s +Step 4210/44000 | Loss: 3.3731 | LR: 4.92e-04 | 1390.7ms/step | 14.4 samples/s +Step 4220/44000 | Loss: 3.3357 | LR: 4.92e-04 | 1391.8ms/step | 14.4 samples/s +Step 4230/44000 | Loss: 3.3603 | LR: 4.92e-04 | 1390.1ms/step | 14.4 samples/s +Step 4240/44000 | Loss: 3.4199 | LR: 4.92e-04 | 1389.1ms/step | 14.4 samples/s +Step 4250/44000 | Loss: 3.3947 | LR: 4.92e-04 | 1389.2ms/step | 14.4 samples/s +Validation at step 4250: Loss = 3.3610, Perplexity = 28.82 +Step 4260/44000 | Loss: 3.3639 | LR: 4.92e-04 | 3603.5ms/step | 5.6 samples/s +Step 4270/44000 | Loss: 3.2979 | LR: 4.92e-04 | 1388.6ms/step | 14.4 samples/s +Step 4280/44000 | Loss: 3.3207 | LR: 4.92e-04 | 1389.9ms/step | 14.4 samples/s +Step 4290/44000 | Loss: 3.3530 | LR: 4.92e-04 | 1391.9ms/step | 14.4 samples/s +Step 4300/44000 | Loss: 3.3446 | LR: 4.92e-04 | 1390.0ms/step | 14.4 samples/s +Step 4310/44000 | Loss: 3.3469 | LR: 4.92e-04 | 1388.7ms/step | 14.4 samples/s +Step 4320/44000 | Loss: 3.3509 | LR: 4.92e-04 | 1388.7ms/step | 14.4 samples/s +Step 4330/44000 | Loss: 3.3385 | LR: 4.92e-04 | 1389.6ms/step | 14.4 samples/s +Step 4340/44000 | Loss: 3.3532 | LR: 4.92e-04 | 1391.1ms/step | 14.4 samples/s +Step 4350/44000 | Loss: 3.3601 | LR: 4.92e-04 | 1391.5ms/step | 14.4 samples/s +Step 4360/44000 | Loss: 3.3091 | LR: 4.91e-04 | 1390.1ms/step | 14.4 samples/s +Step 4370/44000 | Loss: 3.3282 | LR: 4.91e-04 | 1389.5ms/step | 14.4 samples/s +Step 4380/44000 | Loss: 3.3753 | LR: 4.91e-04 | 1388.9ms/step | 14.4 samples/s +Step 4390/44000 | Loss: 3.3580 | LR: 4.91e-04 | 1389.3ms/step | 14.4 samples/s +Step 4400/44000 | Loss: 3.3558 | LR: 4.91e-04 | 1390.4ms/step | 14.4 samples/s +Step 4410/44000 | Loss: 3.3292 | LR: 4.91e-04 | 1388.9ms/step | 14.4 samples/s +Step 4420/44000 | Loss: 3.3372 | LR: 4.91e-04 | 1390.6ms/step | 14.4 samples/s +Step 4430/44000 | Loss: 3.3389 | LR: 4.91e-04 | 1390.5ms/step | 14.4 samples/s +Step 4440/44000 | Loss: 3.3497 | LR: 4.91e-04 | 1391.2ms/step | 14.4 samples/s +Step 4450/44000 | Loss: 3.3405 | LR: 4.91e-04 | 1392.8ms/step | 14.4 samples/s +Step 4460/44000 | Loss: 3.3385 | LR: 4.91e-04 | 1391.9ms/step | 14.4 samples/s +Step 4470/44000 | Loss: 3.3021 | LR: 4.91e-04 | 1388.7ms/step | 14.4 samples/s +Step 4480/44000 | Loss: 3.3382 | LR: 4.91e-04 | 1389.5ms/step | 14.4 samples/s +Step 4490/44000 | Loss: 3.3418 | LR: 4.91e-04 | 1388.0ms/step | 14.4 samples/s +Step 4500/44000 | Loss: 3.3346 | LR: 4.91e-04 | 1389.8ms/step | 14.4 samples/s +Validation at step 4500: Loss = 3.3377, Perplexity = 28.15 +Step 4510/44000 | Loss: 3.3248 | LR: 4.91e-04 | 3583.3ms/step | 5.6 samples/s +Step 4520/44000 | Loss: 3.3523 | LR: 4.91e-04 | 1391.3ms/step | 14.4 samples/s +Step 4530/44000 | Loss: 3.3474 | LR: 4.91e-04 | 1389.3ms/step | 14.4 samples/s +Step 4540/44000 | Loss: 3.3073 | LR: 4.91e-04 | 1390.4ms/step | 14.4 samples/s +Step 4550/44000 | Loss: 3.3010 | LR: 4.91e-04 | 1389.9ms/step | 14.4 samples/s +Step 4560/44000 | Loss: 3.3091 | LR: 4.90e-04 | 1390.3ms/step | 14.4 samples/s +Step 4570/44000 | Loss: 3.3286 | LR: 4.90e-04 | 1388.5ms/step | 14.4 samples/s +Step 4580/44000 | Loss: 3.2862 | LR: 4.90e-04 | 1390.7ms/step | 14.4 samples/s +Step 4590/44000 | Loss: 3.2876 | LR: 4.90e-04 | 1388.4ms/step | 14.4 samples/s +Step 4600/44000 | Loss: 3.3090 | LR: 4.90e-04 | 1392.0ms/step | 14.4 samples/s +Step 4610/44000 | Loss: 3.3171 | LR: 4.90e-04 | 1391.8ms/step | 14.4 samples/s +Step 4620/44000 | Loss: 3.3059 | LR: 4.90e-04 | 1392.2ms/step | 14.4 samples/s +Step 4630/44000 | Loss: 3.3177 | LR: 4.90e-04 | 1391.5ms/step | 14.4 samples/s +Step 4640/44000 | Loss: 3.3283 | LR: 4.90e-04 | 1391.1ms/step | 14.4 samples/s +Step 4650/44000 | Loss: 3.3157 | LR: 4.90e-04 | 1391.9ms/step | 14.4 samples/s +Step 4660/44000 | Loss: 3.2954 | LR: 4.90e-04 | 1389.7ms/step | 14.4 samples/s +Step 4670/44000 | Loss: 3.2888 | LR: 4.90e-04 | 1387.6ms/step | 14.4 samples/s +Step 4680/44000 | Loss: 3.3125 | LR: 4.90e-04 | 1391.8ms/step | 14.4 samples/s +Step 4690/44000 | Loss: 3.3056 | LR: 4.90e-04 | 1389.8ms/step | 14.4 samples/s +Step 4700/44000 | Loss: 3.3402 | LR: 4.90e-04 | 1391.9ms/step | 14.4 samples/s +Step 4710/44000 | Loss: 3.3149 | LR: 4.90e-04 | 1392.9ms/step | 14.4 samples/s +Step 4720/44000 | Loss: 3.2931 | LR: 4.90e-04 | 1390.0ms/step | 14.4 samples/s +Step 4730/44000 | Loss: 3.3124 | LR: 4.90e-04 | 1392.3ms/step | 14.4 samples/s +Step 4740/44000 | Loss: 3.3033 | LR: 4.90e-04 | 1389.8ms/step | 14.4 samples/s +Step 4750/44000 | Loss: 3.3142 | LR: 4.90e-04 | 1391.3ms/step | 14.4 samples/s +Validation at step 4750: Loss = 3.3186, Perplexity = 27.62 +Step 4760/44000 | Loss: 3.2923 | LR: 4.89e-04 | 3594.8ms/step | 5.6 samples/s +Step 4770/44000 | Loss: 3.3170 | LR: 4.89e-04 | 1391.0ms/step | 14.4 samples/s +Step 4780/44000 | Loss: 3.3137 | LR: 4.89e-04 | 1390.5ms/step | 14.4 samples/s +Step 4790/44000 | Loss: 3.2854 | LR: 4.89e-04 | 1388.5ms/step | 14.4 samples/s +Step 4800/44000 | Loss: 3.3099 | LR: 4.89e-04 | 1389.7ms/step | 14.4 samples/s +Step 4810/44000 | Loss: 3.3092 | LR: 4.89e-04 | 1388.1ms/step | 14.4 samples/s +Step 4820/44000 | Loss: 3.3112 | LR: 4.89e-04 | 1390.9ms/step | 14.4 samples/s +Step 4830/44000 | Loss: 3.2981 | LR: 4.89e-04 | 1390.7ms/step | 14.4 samples/s +Step 4840/44000 | Loss: 3.3127 | LR: 4.89e-04 | 1389.8ms/step | 14.4 samples/s +Step 4850/44000 | Loss: 3.3127 | LR: 4.89e-04 | 1392.1ms/step | 14.4 samples/s +Step 4860/44000 | Loss: 3.3208 | LR: 4.89e-04 | 1390.6ms/step | 14.4 samples/s +Step 4870/44000 | Loss: 3.2821 | LR: 4.89e-04 | 1391.9ms/step | 14.4 samples/s +Step 4880/44000 | Loss: 3.2838 | LR: 4.89e-04 | 1391.8ms/step | 14.4 samples/s +Step 4890/44000 | Loss: 3.2759 | LR: 4.89e-04 | 1390.6ms/step | 14.4 samples/s +Step 4900/44000 | Loss: 3.2585 | LR: 4.89e-04 | 1391.5ms/step | 14.4 samples/s +Step 4910/44000 | Loss: 3.2990 | LR: 4.89e-04 | 1392.4ms/step | 14.4 samples/s +Step 4920/44000 | Loss: 3.2955 | LR: 4.89e-04 | 1390.3ms/step | 14.4 samples/s +Step 4930/44000 | Loss: 3.2792 | LR: 4.89e-04 | 1389.8ms/step | 14.4 samples/s +Step 4940/44000 | Loss: 3.2901 | LR: 4.89e-04 | 1391.0ms/step | 14.4 samples/s +Step 4950/44000 | Loss: 3.2926 | LR: 4.88e-04 | 1391.0ms/step | 14.4 samples/s +Step 4960/44000 | Loss: 3.2813 | LR: 4.88e-04 | 1392.0ms/step | 14.4 samples/s +Step 4970/44000 | Loss: 3.2734 | LR: 4.88e-04 | 1389.7ms/step | 14.4 samples/s +Step 4980/44000 | Loss: 3.2701 | LR: 4.88e-04 | 1387.9ms/step | 14.4 samples/s +Step 4990/44000 | Loss: 3.2984 | LR: 4.88e-04 | 1389.9ms/step | 14.4 samples/s +Step 5000/44000 | Loss: 3.2938 | LR: 4.88e-04 | 1388.5ms/step | 14.4 samples/s +Validation at step 5000: Loss = 3.3038, Perplexity = 27.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-5000 +Step 5010/44000 | Loss: 3.3024 | LR: 4.88e-04 | 3870.2ms/step | 5.2 samples/s +Step 5020/44000 | Loss: 3.2820 | LR: 4.88e-04 | 1387.7ms/step | 14.4 samples/s +Step 5030/44000 | Loss: 3.2885 | LR: 4.88e-04 | 1386.0ms/step | 14.4 samples/s +Step 5040/44000 | Loss: 3.2977 | LR: 4.88e-04 | 1388.0ms/step | 14.4 samples/s +Step 5050/44000 | Loss: 3.2737 | LR: 4.88e-04 | 1388.5ms/step | 14.4 samples/s +Step 5060/44000 | Loss: 3.3024 | LR: 4.88e-04 | 1389.4ms/step | 14.4 samples/s +Step 5070/44000 | Loss: 3.3089 | LR: 4.88e-04 | 1389.8ms/step | 14.4 samples/s +Step 5080/44000 | Loss: 3.3003 | LR: 4.88e-04 | 1388.4ms/step | 14.4 samples/s +Step 5090/44000 | Loss: 3.2701 | LR: 4.88e-04 | 1389.5ms/step | 14.4 samples/s +Step 5100/44000 | Loss: 3.2976 | LR: 4.88e-04 | 1389.2ms/step | 14.4 samples/s +Step 5110/44000 | Loss: 3.3180 | LR: 4.88e-04 | 1390.7ms/step | 14.4 samples/s +Step 5120/44000 | Loss: 3.3150 | LR: 4.88e-04 | 1390.6ms/step | 14.4 samples/s +Step 5130/44000 | Loss: 3.2954 | LR: 4.87e-04 | 1390.0ms/step | 14.4 samples/s +Step 5140/44000 | Loss: 3.2718 | LR: 4.87e-04 | 1391.6ms/step | 14.4 samples/s +Step 5150/44000 | Loss: 3.3150 | LR: 4.87e-04 | 1391.3ms/step | 14.4 samples/s +Step 5160/44000 | Loss: 3.3146 | LR: 4.87e-04 | 1389.7ms/step | 14.4 samples/s +Step 5170/44000 | Loss: 3.3029 | LR: 4.87e-04 | 1389.1ms/step | 14.4 samples/s +Step 5180/44000 | Loss: 3.2612 | LR: 4.87e-04 | 1390.1ms/step | 14.4 samples/s +Step 5190/44000 | Loss: 3.2867 | LR: 4.87e-04 | 1389.4ms/step | 14.4 samples/s +Step 5200/44000 | Loss: 3.2551 | LR: 4.87e-04 | 1390.2ms/step | 14.4 samples/s +Step 5210/44000 | Loss: 3.2936 | LR: 4.87e-04 | 1388.7ms/step | 14.4 samples/s +Step 5220/44000 | Loss: 3.2849 | LR: 4.87e-04 | 1391.7ms/step | 14.4 samples/s +Step 5230/44000 | Loss: 3.2396 | LR: 4.87e-04 | 1389.1ms/step | 14.4 samples/s +Step 5240/44000 | Loss: 3.2688 | LR: 4.87e-04 | 1391.3ms/step | 14.4 samples/s +Step 5250/44000 | Loss: 3.2685 | LR: 4.87e-04 | 1389.7ms/step | 14.4 samples/s +Validation at step 5250: Loss = 3.2889, Perplexity = 26.81 +Step 5260/44000 | Loss: 3.2906 | LR: 4.87e-04 | 3592.7ms/step | 5.6 samples/s +Step 5270/44000 | Loss: 3.2583 | LR: 4.87e-04 | 1389.1ms/step | 14.4 samples/s +Step 5280/44000 | Loss: 3.2859 | LR: 4.87e-04 | 1390.5ms/step | 14.4 samples/s +Step 5290/44000 | Loss: 3.2722 | LR: 4.87e-04 | 1389.5ms/step | 14.4 samples/s +Step 5300/44000 | Loss: 3.2255 | LR: 4.86e-04 | 1390.2ms/step | 14.4 samples/s +Step 5310/44000 | Loss: 3.2686 | LR: 4.86e-04 | 1389.2ms/step | 14.4 samples/s +Step 5320/44000 | Loss: 3.2808 | LR: 4.86e-04 | 1388.8ms/step | 14.4 samples/s +Step 5330/44000 | Loss: 3.2757 | LR: 4.86e-04 | 1392.9ms/step | 14.4 samples/s +Step 5340/44000 | Loss: 3.2866 | LR: 4.86e-04 | 1390.1ms/step | 14.4 samples/s +Step 5350/44000 | Loss: 3.2518 | LR: 4.86e-04 | 1389.9ms/step | 14.4 samples/s +Step 5360/44000 | Loss: 3.2777 | LR: 4.86e-04 | 1389.2ms/step | 14.4 samples/s +Step 5370/44000 | Loss: 3.2766 | LR: 4.86e-04 | 1391.4ms/step | 14.4 samples/s +Step 5380/44000 | Loss: 3.2640 | LR: 4.86e-04 | 1389.6ms/step | 14.4 samples/s +Step 5390/44000 | Loss: 3.2862 | LR: 4.86e-04 | 1387.3ms/step | 14.4 samples/s +Step 5400/44000 | Loss: 3.2921 | LR: 4.86e-04 | 1390.1ms/step | 14.4 samples/s +Step 5410/44000 | Loss: 3.2787 | LR: 4.86e-04 | 1389.4ms/step | 14.4 samples/s +Step 5420/44000 | Loss: 3.2736 | LR: 4.86e-04 | 1389.9ms/step | 14.4 samples/s +Step 5430/44000 | Loss: 3.2580 | LR: 4.86e-04 | 1388.1ms/step | 14.4 samples/s +Step 5440/44000 | Loss: 3.2685 | LR: 4.86e-04 | 1388.2ms/step | 14.4 samples/s +Step 5450/44000 | Loss: 3.2811 | LR: 4.86e-04 | 1390.1ms/step | 14.4 samples/s +Step 5460/44000 | Loss: 3.2961 | LR: 4.86e-04 | 1389.1ms/step | 14.4 samples/s +Step 5470/44000 | Loss: 3.2767 | LR: 4.85e-04 | 1388.8ms/step | 14.4 samples/s +Step 5480/44000 | Loss: 3.2577 | LR: 4.85e-04 | 1389.2ms/step | 14.4 samples/s +Step 5490/44000 | Loss: 3.2449 | LR: 4.85e-04 | 1388.8ms/step | 14.4 samples/s +Step 5500/44000 | Loss: 3.2688 | LR: 4.85e-04 | 1390.1ms/step | 14.4 samples/s +Validation at step 5500: Loss = 3.2743, Perplexity = 26.42 +Step 5510/44000 | Loss: 3.2643 | LR: 4.85e-04 | 3586.7ms/step | 5.6 samples/s +Step 5520/44000 | Loss: 3.2689 | LR: 4.85e-04 | 1388.3ms/step | 14.4 samples/s +Step 5530/44000 | Loss: 3.2374 | LR: 4.85e-04 | 1388.4ms/step | 14.4 samples/s +Step 5540/44000 | Loss: 3.2709 | LR: 4.85e-04 | 1390.4ms/step | 14.4 samples/s +Step 5550/44000 | Loss: 3.2522 | LR: 4.85e-04 | 1388.0ms/step | 14.4 samples/s +Step 5560/44000 | Loss: 3.2625 | LR: 4.85e-04 | 1384.2ms/step | 14.4 samples/s +Step 5570/44000 | Loss: 3.2555 | LR: 4.85e-04 | 1386.0ms/step | 14.4 samples/s +Step 5580/44000 | Loss: 3.2451 | LR: 4.85e-04 | 1386.2ms/step | 14.4 samples/s +Step 5590/44000 | Loss: 3.2394 | LR: 4.85e-04 | 1388.6ms/step | 14.4 samples/s +Step 5600/44000 | Loss: 3.2361 | LR: 4.85e-04 | 1387.5ms/step | 14.4 samples/s +Step 5610/44000 | Loss: 3.2707 | LR: 4.85e-04 | 1388.6ms/step | 14.4 samples/s +Step 5620/44000 | Loss: 3.2535 | LR: 4.85e-04 | 1389.1ms/step | 14.4 samples/s +Step 5630/44000 | Loss: 3.2732 | LR: 4.84e-04 | 1389.4ms/step | 14.4 samples/s +Step 5640/44000 | Loss: 3.2452 | LR: 4.84e-04 | 1388.3ms/step | 14.4 samples/s +Step 5650/44000 | Loss: 3.2434 | LR: 4.84e-04 | 1390.3ms/step | 14.4 samples/s +Step 5660/44000 | Loss: 3.2306 | LR: 4.84e-04 | 1392.1ms/step | 14.4 samples/s +Step 5670/44000 | Loss: 3.2488 | LR: 4.84e-04 | 1389.9ms/step | 14.4 samples/s +Step 5680/44000 | Loss: 3.2307 | LR: 4.84e-04 | 1387.9ms/step | 14.4 samples/s +Step 5690/44000 | Loss: 3.2787 | LR: 4.84e-04 | 1388.3ms/step | 14.4 samples/s +Step 5700/44000 | Loss: 3.2675 | LR: 4.84e-04 | 1388.5ms/step | 14.4 samples/s +Step 5710/44000 | Loss: 3.2180 | LR: 4.84e-04 | 1389.1ms/step | 14.4 samples/s +Step 5720/44000 | Loss: 3.2180 | LR: 4.84e-04 | 1389.1ms/step | 14.4 samples/s +Step 5730/44000 | Loss: 3.2362 | LR: 4.84e-04 | 1387.1ms/step | 14.4 samples/s +Step 5740/44000 | Loss: 3.2387 | LR: 4.84e-04 | 1390.4ms/step | 14.4 samples/s +Step 5750/44000 | Loss: 3.2405 | LR: 4.84e-04 | 1387.5ms/step | 14.4 samples/s +Validation at step 5750: Loss = 3.2615, Perplexity = 26.09 +Step 5760/44000 | Loss: 3.2647 | LR: 4.84e-04 | 3589.0ms/step | 5.6 samples/s +Step 5770/44000 | Loss: 3.2707 | LR: 4.84e-04 | 1389.2ms/step | 14.4 samples/s +Step 5780/44000 | Loss: 3.2480 | LR: 4.83e-04 | 1387.9ms/step | 14.4 samples/s +Step 5790/44000 | Loss: 3.2317 | LR: 4.83e-04 | 1389.0ms/step | 14.4 samples/s +Step 5800/44000 | Loss: 3.2567 | LR: 4.83e-04 | 1389.1ms/step | 14.4 samples/s +Step 5810/44000 | Loss: 3.2706 | LR: 4.83e-04 | 1393.6ms/step | 14.4 samples/s +Step 5820/44000 | Loss: 3.2542 | LR: 4.83e-04 | 1388.9ms/step | 14.4 samples/s +Step 5830/44000 | Loss: 3.2500 | LR: 4.83e-04 | 1388.0ms/step | 14.4 samples/s +Step 5840/44000 | Loss: 3.2443 | LR: 4.83e-04 | 1388.5ms/step | 14.4 samples/s +Step 5850/44000 | Loss: 3.2237 | LR: 4.83e-04 | 1388.0ms/step | 14.4 samples/s +Step 5860/44000 | Loss: 3.2534 | LR: 4.83e-04 | 1390.4ms/step | 14.4 samples/s +Step 5870/44000 | Loss: 3.2608 | LR: 4.83e-04 | 1390.7ms/step | 14.4 samples/s +Step 5880/44000 | Loss: 3.2385 | LR: 4.83e-04 | 1390.0ms/step | 14.4 samples/s +Step 5890/44000 | Loss: 3.2557 | LR: 4.83e-04 | 1391.6ms/step | 14.4 samples/s +Step 5900/44000 | Loss: 3.2433 | LR: 4.83e-04 | 1391.2ms/step | 14.4 samples/s +Step 5910/44000 | Loss: 3.2500 | LR: 4.83e-04 | 1389.5ms/step | 14.4 samples/s +Step 5920/44000 | Loss: 3.2603 | LR: 4.83e-04 | 1391.9ms/step | 14.4 samples/s +Step 5930/44000 | Loss: 3.2111 | LR: 4.83e-04 | 1389.7ms/step | 14.4 samples/s +Step 5940/44000 | Loss: 3.2609 | LR: 4.82e-04 | 1389.0ms/step | 14.4 samples/s +Step 5950/44000 | Loss: 3.2213 | LR: 4.82e-04 | 1389.4ms/step | 14.4 samples/s +Step 5960/44000 | Loss: 3.2140 | LR: 4.82e-04 | 1388.5ms/step | 14.4 samples/s +Step 5970/44000 | Loss: 3.2196 | LR: 4.82e-04 | 1389.1ms/step | 14.4 samples/s +Step 5980/44000 | Loss: 3.2151 | LR: 4.82e-04 | 1390.9ms/step | 14.4 samples/s +Step 5990/44000 | Loss: 3.2157 | LR: 4.82e-04 | 1390.4ms/step | 14.4 samples/s +Step 6000/44000 | Loss: 3.2412 | LR: 4.82e-04 | 1389.8ms/step | 14.4 samples/s +Validation at step 6000: Loss = 3.2498, Perplexity = 25.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-6000 +Step 6010/44000 | Loss: 3.2626 | LR: 4.82e-04 | 3897.2ms/step | 5.1 samples/s +Step 6020/44000 | Loss: 3.2245 | LR: 4.82e-04 | 1390.2ms/step | 14.4 samples/s +Step 6030/44000 | Loss: 3.2300 | LR: 4.82e-04 | 1389.7ms/step | 14.4 samples/s +Step 6040/44000 | Loss: 3.2333 | LR: 4.82e-04 | 1389.6ms/step | 14.4 samples/s +Step 6050/44000 | Loss: 3.2376 | LR: 4.82e-04 | 1389.8ms/step | 14.4 samples/s +Step 6060/44000 | Loss: 3.2589 | LR: 4.82e-04 | 1387.7ms/step | 14.4 samples/s +Step 6070/44000 | Loss: 3.2378 | LR: 4.82e-04 | 1390.8ms/step | 14.4 samples/s +Step 6080/44000 | Loss: 3.2318 | LR: 4.81e-04 | 1391.2ms/step | 14.4 samples/s +Step 6090/44000 | Loss: 3.2602 | LR: 4.81e-04 | 1386.6ms/step | 14.4 samples/s +Step 6100/44000 | Loss: 3.2146 | LR: 4.81e-04 | 1388.6ms/step | 14.4 samples/s +Step 6110/44000 | Loss: 3.2306 | LR: 4.81e-04 | 1387.9ms/step | 14.4 samples/s +Step 6120/44000 | Loss: 3.2104 | LR: 4.81e-04 | 1390.0ms/step | 14.4 samples/s +Step 6130/44000 | Loss: 3.2261 | LR: 4.81e-04 | 1388.2ms/step | 14.4 samples/s +Step 6140/44000 | Loss: 3.2146 | LR: 4.81e-04 | 1390.8ms/step | 14.4 samples/s +Step 6150/44000 | Loss: 3.2204 | LR: 4.81e-04 | 1391.1ms/step | 14.4 samples/s +Step 6160/44000 | Loss: 3.2216 | LR: 4.81e-04 | 1391.6ms/step | 14.4 samples/s +Step 6170/44000 | Loss: 3.2362 | LR: 4.81e-04 | 1389.0ms/step | 14.4 samples/s +Step 6180/44000 | Loss: 3.2334 | LR: 4.81e-04 | 1389.4ms/step | 14.4 samples/s +Step 6190/44000 | Loss: 3.2200 | LR: 4.81e-04 | 1389.9ms/step | 14.4 samples/s +Step 6200/44000 | Loss: 3.2332 | LR: 4.81e-04 | 1389.6ms/step | 14.4 samples/s +Step 6210/44000 | Loss: 3.2288 | LR: 4.81e-04 | 1390.3ms/step | 14.4 samples/s +Step 6220/44000 | Loss: 3.2596 | LR: 4.81e-04 | 1391.0ms/step | 14.4 samples/s +Step 6230/44000 | Loss: 3.2260 | LR: 4.80e-04 | 1388.6ms/step | 14.4 samples/s +Step 6240/44000 | Loss: 3.2452 | LR: 4.80e-04 | 1390.3ms/step | 14.4 samples/s +Step 6250/44000 | Loss: 3.2401 | LR: 4.80e-04 | 1390.2ms/step | 14.4 samples/s +Validation at step 6250: Loss = 3.2423, Perplexity = 25.59 +Step 6260/44000 | Loss: 3.2337 | LR: 4.80e-04 | 3588.9ms/step | 5.6 samples/s +Step 6270/44000 | Loss: 3.2600 | LR: 4.80e-04 | 1392.0ms/step | 14.4 samples/s +Step 6280/44000 | Loss: 3.2104 | LR: 4.80e-04 | 1391.1ms/step | 14.4 samples/s +Step 6290/44000 | Loss: 3.2436 | LR: 4.80e-04 | 1389.3ms/step | 14.4 samples/s +Step 6300/44000 | Loss: 3.2164 | LR: 4.80e-04 | 1390.8ms/step | 14.4 samples/s +Step 6310/44000 | Loss: 3.2232 | LR: 4.80e-04 | 1392.0ms/step | 14.4 samples/s +Step 6320/44000 | Loss: 3.2151 | LR: 4.80e-04 | 1391.3ms/step | 14.4 samples/s +Step 6330/44000 | Loss: 3.2233 | LR: 4.80e-04 | 1392.3ms/step | 14.4 samples/s +Step 6340/44000 | Loss: 3.2352 | LR: 4.80e-04 | 1389.7ms/step | 14.4 samples/s +Step 6350/44000 | Loss: 3.1939 | LR: 4.80e-04 | 1390.2ms/step | 14.4 samples/s +Step 6360/44000 | Loss: 3.2493 | LR: 4.80e-04 | 1391.1ms/step | 14.4 samples/s +Step 6370/44000 | Loss: 3.2180 | LR: 4.79e-04 | 1389.4ms/step | 14.4 samples/s +Step 6380/44000 | Loss: 3.2056 | LR: 4.79e-04 | 1392.1ms/step | 14.4 samples/s +Step 6390/44000 | Loss: 3.1971 | LR: 4.79e-04 | 1388.8ms/step | 14.4 samples/s +Step 6400/44000 | Loss: 3.2308 | LR: 4.79e-04 | 1388.8ms/step | 14.4 samples/s +Step 6410/44000 | Loss: 3.1991 | LR: 4.79e-04 | 1388.5ms/step | 14.4 samples/s +Step 6420/44000 | Loss: 3.2313 | LR: 4.79e-04 | 1388.8ms/step | 14.4 samples/s +Step 6430/44000 | Loss: 3.1938 | LR: 4.79e-04 | 1386.6ms/step | 14.4 samples/s +Step 6440/44000 | Loss: 3.2107 | LR: 4.79e-04 | 1386.1ms/step | 14.4 samples/s +Step 6450/44000 | Loss: 3.2079 | LR: 4.79e-04 | 1388.0ms/step | 14.4 samples/s +Step 6460/44000 | Loss: 3.2179 | LR: 4.79e-04 | 1388.1ms/step | 14.4 samples/s +Step 6470/44000 | Loss: 3.1977 | LR: 4.79e-04 | 1385.3ms/step | 14.4 samples/s +Step 6480/44000 | Loss: 3.2093 | LR: 4.79e-04 | 1386.9ms/step | 14.4 samples/s +Step 6490/44000 | Loss: 3.2039 | LR: 4.79e-04 | 1378.9ms/step | 14.5 samples/s +Step 6500/44000 | Loss: 3.2013 | LR: 4.79e-04 | 1380.9ms/step | 14.5 samples/s +Validation at step 6500: Loss = 3.2293, Perplexity = 25.26 +Step 6510/44000 | Loss: 3.2269 | LR: 4.78e-04 | 3574.1ms/step | 5.6 samples/s +Step 6520/44000 | Loss: 3.2339 | LR: 4.78e-04 | 1385.4ms/step | 14.4 samples/s +Step 6530/44000 | Loss: 3.2285 | LR: 4.78e-04 | 1387.3ms/step | 14.4 samples/s +Step 6540/44000 | Loss: 3.2254 | LR: 4.78e-04 | 1387.1ms/step | 14.4 samples/s +Step 6550/44000 | Loss: 3.2266 | LR: 4.78e-04 | 1389.2ms/step | 14.4 samples/s +Step 6560/44000 | Loss: 3.2066 | LR: 4.78e-04 | 1385.7ms/step | 14.4 samples/s +Step 6570/44000 | Loss: 3.2077 | LR: 4.78e-04 | 1389.7ms/step | 14.4 samples/s +Step 6580/44000 | Loss: 3.2480 | LR: 4.78e-04 | 1389.2ms/step | 14.4 samples/s +Step 6590/44000 | Loss: 3.2179 | LR: 4.78e-04 | 1390.0ms/step | 14.4 samples/s +Step 6600/44000 | Loss: 3.2061 | LR: 4.78e-04 | 1387.7ms/step | 14.4 samples/s +Step 6610/44000 | Loss: 3.1712 | LR: 4.78e-04 | 1386.4ms/step | 14.4 samples/s +Step 6620/44000 | Loss: 3.2031 | LR: 4.78e-04 | 1386.2ms/step | 14.4 samples/s +Step 6630/44000 | Loss: 3.2065 | LR: 4.78e-04 | 1387.8ms/step | 14.4 samples/s +Step 6640/44000 | Loss: 3.1952 | LR: 4.77e-04 | 1389.1ms/step | 14.4 samples/s +Step 6650/44000 | Loss: 3.2220 | LR: 4.77e-04 | 1390.8ms/step | 14.4 samples/s +Step 6660/44000 | Loss: 3.2476 | LR: 4.77e-04 | 1387.8ms/step | 14.4 samples/s +Step 6670/44000 | Loss: 3.1941 | LR: 4.77e-04 | 1390.2ms/step | 14.4 samples/s +Step 6680/44000 | Loss: 3.2167 | LR: 4.77e-04 | 1386.4ms/step | 14.4 samples/s +Step 6690/44000 | Loss: 3.2185 | LR: 4.77e-04 | 1378.9ms/step | 14.5 samples/s +Step 6700/44000 | Loss: 3.2199 | LR: 4.77e-04 | 1376.7ms/step | 14.5 samples/s +Step 6710/44000 | Loss: 3.2178 | LR: 4.77e-04 | 1382.3ms/step | 14.5 samples/s +Step 6720/44000 | Loss: 3.1835 | LR: 4.77e-04 | 1381.9ms/step | 14.5 samples/s +Step 6730/44000 | Loss: 3.2159 | LR: 4.77e-04 | 1382.6ms/step | 14.5 samples/s +Step 6740/44000 | Loss: 3.1920 | LR: 4.77e-04 | 1385.9ms/step | 14.4 samples/s +Step 6750/44000 | Loss: 3.1982 | LR: 4.77e-04 | 1388.2ms/step | 14.4 samples/s +Validation at step 6750: Loss = 3.2189, Perplexity = 25.00 +Step 6760/44000 | Loss: 3.2257 | LR: 4.77e-04 | 3580.6ms/step | 5.6 samples/s +Step 6770/44000 | Loss: 3.2211 | LR: 4.76e-04 | 1386.9ms/step | 14.4 samples/s +Step 6780/44000 | Loss: 3.2187 | LR: 4.76e-04 | 1389.4ms/step | 14.4 samples/s +Step 6790/44000 | Loss: 3.2123 | LR: 4.76e-04 | 1387.0ms/step | 14.4 samples/s +Step 6800/44000 | Loss: 3.1630 | LR: 4.76e-04 | 1389.5ms/step | 14.4 samples/s +Step 6810/44000 | Loss: 3.2267 | LR: 4.76e-04 | 1388.0ms/step | 14.4 samples/s +Step 6820/44000 | Loss: 3.1996 | LR: 4.76e-04 | 1387.9ms/step | 14.4 samples/s +Step 6830/44000 | Loss: 3.2074 | LR: 4.76e-04 | 1391.4ms/step | 14.4 samples/s +Step 6840/44000 | Loss: 3.2103 | LR: 4.76e-04 | 1388.4ms/step | 14.4 samples/s +Step 6850/44000 | Loss: 3.2132 | LR: 4.76e-04 | 1391.1ms/step | 14.4 samples/s +Step 6860/44000 | Loss: 3.2113 | LR: 4.76e-04 | 1387.6ms/step | 14.4 samples/s +Step 6870/44000 | Loss: 3.1878 | LR: 4.76e-04 | 1389.4ms/step | 14.4 samples/s +Step 6880/44000 | Loss: 3.1598 | LR: 4.76e-04 | 1388.4ms/step | 14.4 samples/s +Step 6890/44000 | Loss: 3.2171 | LR: 4.76e-04 | 1388.9ms/step | 14.4 samples/s +Step 6900/44000 | Loss: 3.2295 | LR: 4.75e-04 | 1390.8ms/step | 14.4 samples/s +Step 6910/44000 | Loss: 3.1920 | LR: 4.75e-04 | 1387.5ms/step | 14.4 samples/s +Step 6920/44000 | Loss: 3.2111 | LR: 4.75e-04 | 1386.5ms/step | 14.4 samples/s +Step 6930/44000 | Loss: 3.1902 | LR: 4.75e-04 | 1386.4ms/step | 14.4 samples/s +Step 6940/44000 | Loss: 3.2166 | LR: 4.75e-04 | 1387.2ms/step | 14.4 samples/s +Step 6950/44000 | Loss: 3.2163 | LR: 4.75e-04 | 1387.2ms/step | 14.4 samples/s +Step 6960/44000 | Loss: 3.1891 | LR: 4.75e-04 | 1387.6ms/step | 14.4 samples/s +Step 6970/44000 | Loss: 3.2119 | LR: 4.75e-04 | 1389.5ms/step | 14.4 samples/s +Step 6980/44000 | Loss: 3.1807 | LR: 4.75e-04 | 1390.3ms/step | 14.4 samples/s +Step 6990/44000 | Loss: 3.1865 | LR: 4.75e-04 | 1389.3ms/step | 14.4 samples/s +Step 7000/44000 | Loss: 3.1954 | LR: 4.75e-04 | 1389.8ms/step | 14.4 samples/s +Validation at step 7000: Loss = 3.2101, Perplexity = 24.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-7000 +Step 7010/44000 | Loss: 3.1862 | LR: 4.75e-04 | 3876.4ms/step | 5.2 samples/s +Step 7020/44000 | Loss: 3.1857 | LR: 4.75e-04 | 1387.8ms/step | 14.4 samples/s +Step 7030/44000 | Loss: 3.2083 | LR: 4.74e-04 | 1388.0ms/step | 14.4 samples/s +Step 7040/44000 | Loss: 3.2003 | LR: 4.74e-04 | 1388.4ms/step | 14.4 samples/s +Step 7050/44000 | Loss: 3.2264 | LR: 4.74e-04 | 1387.8ms/step | 14.4 samples/s +Step 7060/44000 | Loss: 3.2065 | LR: 4.74e-04 | 1390.1ms/step | 14.4 samples/s +Step 7070/44000 | Loss: 3.1942 | LR: 4.74e-04 | 1390.3ms/step | 14.4 samples/s +Step 7080/44000 | Loss: 3.1864 | LR: 4.74e-04 | 1389.3ms/step | 14.4 samples/s +Step 7090/44000 | Loss: 3.1645 | LR: 4.74e-04 | 1390.5ms/step | 14.4 samples/s +Step 7100/44000 | Loss: 3.1924 | LR: 4.74e-04 | 1392.2ms/step | 14.4 samples/s +Step 7110/44000 | Loss: 3.1655 | LR: 4.74e-04 | 1386.5ms/step | 14.4 samples/s +Step 7120/44000 | Loss: 3.1875 | LR: 4.74e-04 | 1383.4ms/step | 14.5 samples/s +Step 7130/44000 | Loss: 3.1985 | LR: 4.74e-04 | 1384.1ms/step | 14.4 samples/s +Step 7140/44000 | Loss: 3.2199 | LR: 4.74e-04 | 1387.5ms/step | 14.4 samples/s +Step 7150/44000 | Loss: 3.2006 | LR: 4.73e-04 | 1386.4ms/step | 14.4 samples/s +Step 7160/44000 | Loss: 3.2090 | LR: 4.73e-04 | 1386.0ms/step | 14.4 samples/s +Step 7170/44000 | Loss: 3.2124 | LR: 4.73e-04 | 1388.6ms/step | 14.4 samples/s +Step 7180/44000 | Loss: 3.1949 | LR: 4.73e-04 | 1387.5ms/step | 14.4 samples/s +Step 7190/44000 | Loss: 3.1546 | LR: 4.73e-04 | 1389.9ms/step | 14.4 samples/s +Step 7200/44000 | Loss: 3.2072 | LR: 4.73e-04 | 1387.5ms/step | 14.4 samples/s +Step 7210/44000 | Loss: 3.2288 | LR: 4.73e-04 | 1388.3ms/step | 14.4 samples/s +Step 7220/44000 | Loss: 3.2110 | LR: 4.73e-04 | 1386.9ms/step | 14.4 samples/s +Step 7230/44000 | Loss: 3.2086 | LR: 4.73e-04 | 1387.5ms/step | 14.4 samples/s +Step 7240/44000 | Loss: 3.1987 | LR: 4.73e-04 | 1387.8ms/step | 14.4 samples/s +Step 7250/44000 | Loss: 3.2030 | LR: 4.73e-04 | 1389.7ms/step | 14.4 samples/s +Validation at step 7250: Loss = 3.2003, Perplexity = 24.54 +Step 7260/44000 | Loss: 3.1822 | LR: 4.73e-04 | 3589.6ms/step | 5.6 samples/s +Step 7270/44000 | Loss: 3.1923 | LR: 4.72e-04 | 1388.5ms/step | 14.4 samples/s +Step 7280/44000 | Loss: 3.1739 | LR: 4.72e-04 | 1387.1ms/step | 14.4 samples/s +Step 7290/44000 | Loss: 3.2149 | LR: 4.72e-04 | 1385.3ms/step | 14.4 samples/s +Step 7300/44000 | Loss: 3.1959 | LR: 4.72e-04 | 1381.1ms/step | 14.5 samples/s +Step 7310/44000 | Loss: 3.2134 | LR: 4.72e-04 | 1381.6ms/step | 14.5 samples/s +Step 7320/44000 | Loss: 3.2141 | LR: 4.72e-04 | 1384.0ms/step | 14.5 samples/s +Step 7330/44000 | Loss: 3.2057 | LR: 4.72e-04 | 1384.9ms/step | 14.4 samples/s +Step 7340/44000 | Loss: 3.2148 | LR: 4.72e-04 | 1386.3ms/step | 14.4 samples/s +Step 7350/44000 | Loss: 3.1652 | LR: 4.72e-04 | 1386.3ms/step | 14.4 samples/s +Step 7360/44000 | Loss: 3.1719 | LR: 4.72e-04 | 1389.1ms/step | 14.4 samples/s +Step 7370/44000 | Loss: 3.1615 | LR: 4.72e-04 | 1386.7ms/step | 14.4 samples/s +Step 7380/44000 | Loss: 3.1901 | LR: 4.72e-04 | 1386.6ms/step | 14.4 samples/s +Step 7390/44000 | Loss: 3.1968 | LR: 4.71e-04 | 1387.3ms/step | 14.4 samples/s +Step 7400/44000 | Loss: 3.1904 | LR: 4.71e-04 | 1386.7ms/step | 14.4 samples/s +Step 7410/44000 | Loss: 3.2122 | LR: 4.71e-04 | 1387.2ms/step | 14.4 samples/s +Step 7420/44000 | Loss: 3.2139 | LR: 4.71e-04 | 1389.9ms/step | 14.4 samples/s +Step 7430/44000 | Loss: 3.1924 | LR: 4.71e-04 | 1388.4ms/step | 14.4 samples/s +Step 7440/44000 | Loss: 3.1964 | LR: 4.71e-04 | 1387.9ms/step | 14.4 samples/s +Step 7450/44000 | Loss: 3.2105 | LR: 4.71e-04 | 1387.8ms/step | 14.4 samples/s +Step 7460/44000 | Loss: 3.1536 | LR: 4.71e-04 | 1388.0ms/step | 14.4 samples/s +Step 7470/44000 | Loss: 3.1983 | LR: 4.71e-04 | 1388.2ms/step | 14.4 samples/s +Step 7480/44000 | Loss: 3.1504 | LR: 4.71e-04 | 1389.3ms/step | 14.4 samples/s +Step 7490/44000 | Loss: 3.1922 | LR: 4.71e-04 | 1390.5ms/step | 14.4 samples/s +Step 7500/44000 | Loss: 3.1559 | LR: 4.71e-04 | 1388.3ms/step | 14.4 samples/s +Validation at step 7500: Loss = 3.1924, Perplexity = 24.35 +Step 7510/44000 | Loss: 3.1687 | LR: 4.70e-04 | 3587.0ms/step | 5.6 samples/s +Step 7520/44000 | Loss: 3.1831 | LR: 4.70e-04 | 1388.8ms/step | 14.4 samples/s +Step 7530/44000 | Loss: 3.2110 | LR: 4.70e-04 | 1388.6ms/step | 14.4 samples/s +Step 7540/44000 | Loss: 3.1686 | LR: 4.70e-04 | 1389.7ms/step | 14.4 samples/s +Step 7550/44000 | Loss: 3.1729 | LR: 4.70e-04 | 1388.0ms/step | 14.4 samples/s +Step 7560/44000 | Loss: 3.1774 | LR: 4.70e-04 | 1388.0ms/step | 14.4 samples/s +Step 7570/44000 | Loss: 3.2058 | LR: 4.70e-04 | 1388.7ms/step | 14.4 samples/s +Step 7580/44000 | Loss: 3.1644 | LR: 4.70e-04 | 1389.0ms/step | 14.4 samples/s +Step 7590/44000 | Loss: 3.2065 | LR: 4.70e-04 | 1388.3ms/step | 14.4 samples/s +Step 7600/44000 | Loss: 3.1887 | LR: 4.70e-04 | 1389.9ms/step | 14.4 samples/s +Step 7610/44000 | Loss: 3.2054 | LR: 4.70e-04 | 1389.8ms/step | 14.4 samples/s +Step 7620/44000 | Loss: 3.1578 | LR: 4.70e-04 | 1386.4ms/step | 14.4 samples/s +Step 7630/44000 | Loss: 3.2135 | LR: 4.69e-04 | 1382.1ms/step | 14.5 samples/s +Step 7640/44000 | Loss: 3.1702 | LR: 4.69e-04 | 1383.8ms/step | 14.5 samples/s +Step 7650/44000 | Loss: 3.1819 | LR: 4.69e-04 | 1386.9ms/step | 14.4 samples/s +Step 7660/44000 | Loss: 3.1918 | LR: 4.69e-04 | 1386.9ms/step | 14.4 samples/s +Step 7670/44000 | Loss: 3.1724 | LR: 4.69e-04 | 1387.8ms/step | 14.4 samples/s +Step 7680/44000 | Loss: 3.1936 | LR: 4.69e-04 | 1388.5ms/step | 14.4 samples/s +Step 7690/44000 | Loss: 3.1563 | LR: 4.69e-04 | 1387.3ms/step | 14.4 samples/s +Step 7700/44000 | Loss: 3.1785 | LR: 4.69e-04 | 1388.6ms/step | 14.4 samples/s +Step 7710/44000 | Loss: 3.2024 | LR: 4.69e-04 | 1390.0ms/step | 14.4 samples/s +Step 7720/44000 | Loss: 3.2034 | LR: 4.69e-04 | 1388.0ms/step | 14.4 samples/s +Step 7730/44000 | Loss: 3.1983 | LR: 4.69e-04 | 1387.7ms/step | 14.4 samples/s +Step 7740/44000 | Loss: 3.1660 | LR: 4.68e-04 | 1386.4ms/step | 14.4 samples/s +Step 7750/44000 | Loss: 3.1444 | LR: 4.68e-04 | 1387.6ms/step | 14.4 samples/s +Validation at step 7750: Loss = 3.1842, Perplexity = 24.15 +Step 7760/44000 | Loss: 3.1558 | LR: 4.68e-04 | 3589.9ms/step | 5.6 samples/s +Step 7770/44000 | Loss: 3.1592 | LR: 4.68e-04 | 1389.6ms/step | 14.4 samples/s +Step 7780/44000 | Loss: 3.1552 | LR: 4.68e-04 | 1390.6ms/step | 14.4 samples/s +Step 7790/44000 | Loss: 3.1787 | LR: 4.68e-04 | 1389.9ms/step | 14.4 samples/s +Step 7800/44000 | Loss: 3.1684 | LR: 4.68e-04 | 1389.5ms/step | 14.4 samples/s +Step 7810/44000 | Loss: 3.1561 | LR: 4.68e-04 | 1389.1ms/step | 14.4 samples/s +Step 7820/44000 | Loss: 3.1719 | LR: 4.68e-04 | 1388.5ms/step | 14.4 samples/s +Step 7830/44000 | Loss: 3.1871 | LR: 4.68e-04 | 1387.5ms/step | 14.4 samples/s +Step 7840/44000 | Loss: 3.1650 | LR: 4.68e-04 | 1387.6ms/step | 14.4 samples/s +Step 7850/44000 | Loss: 3.1458 | LR: 4.67e-04 | 1387.8ms/step | 14.4 samples/s +Step 7860/44000 | Loss: 3.1658 | LR: 4.67e-04 | 1389.3ms/step | 14.4 samples/s +Step 7870/44000 | Loss: 3.1692 | LR: 4.67e-04 | 1387.8ms/step | 14.4 samples/s +Step 7880/44000 | Loss: 3.1283 | LR: 4.67e-04 | 1387.9ms/step | 14.4 samples/s +Step 7890/44000 | Loss: 3.1478 | LR: 4.67e-04 | 1385.9ms/step | 14.4 samples/s +Step 7900/44000 | Loss: 3.1977 | LR: 4.67e-04 | 1387.0ms/step | 14.4 samples/s +Step 7910/44000 | Loss: 3.1433 | LR: 4.67e-04 | 1386.6ms/step | 14.4 samples/s +Step 7920/44000 | Loss: 3.1811 | LR: 4.67e-04 | 1388.1ms/step | 14.4 samples/s +Step 7930/44000 | Loss: 3.1580 | LR: 4.67e-04 | 1388.9ms/step | 14.4 samples/s +Step 7940/44000 | Loss: 3.1732 | LR: 4.67e-04 | 1388.3ms/step | 14.4 samples/s +Step 7950/44000 | Loss: 3.1693 | LR: 4.67e-04 | 1387.1ms/step | 14.4 samples/s +Step 7960/44000 | Loss: 3.1635 | LR: 4.66e-04 | 1385.3ms/step | 14.4 samples/s +Step 7970/44000 | Loss: 3.1570 | LR: 4.66e-04 | 1384.8ms/step | 14.4 samples/s +Step 7980/44000 | Loss: 3.1847 | LR: 4.66e-04 | 1385.5ms/step | 14.4 samples/s +Step 7990/44000 | Loss: 3.1806 | LR: 4.66e-04 | 1386.1ms/step | 14.4 samples/s +Step 8000/44000 | Loss: 3.1511 | LR: 4.66e-04 | 1387.4ms/step | 14.4 samples/s +Validation at step 8000: Loss = 3.1760, Perplexity = 23.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-8000 +Step 8010/44000 | Loss: 3.1221 | LR: 4.66e-04 | 3866.5ms/step | 5.2 samples/s +Step 8020/44000 | Loss: 3.1437 | LR: 4.66e-04 | 1387.3ms/step | 14.4 samples/s +Step 8030/44000 | Loss: 3.1446 | LR: 4.66e-04 | 1386.2ms/step | 14.4 samples/s +Step 8040/44000 | Loss: 3.1691 | LR: 4.66e-04 | 1390.0ms/step | 14.4 samples/s +Step 8050/44000 | Loss: 3.1530 | LR: 4.66e-04 | 1388.4ms/step | 14.4 samples/s +Step 8060/44000 | Loss: 3.1863 | LR: 4.66e-04 | 1388.8ms/step | 14.4 samples/s +Step 8070/44000 | Loss: 3.1513 | LR: 4.65e-04 | 1388.4ms/step | 14.4 samples/s +Step 8080/44000 | Loss: 3.1420 | LR: 4.65e-04 | 1390.6ms/step | 14.4 samples/s +Step 8090/44000 | Loss: 3.1428 | LR: 4.65e-04 | 1388.2ms/step | 14.4 samples/s +Step 8100/44000 | Loss: 3.1660 | LR: 4.65e-04 | 1388.2ms/step | 14.4 samples/s +Step 8110/44000 | Loss: 3.1604 | LR: 4.65e-04 | 1387.8ms/step | 14.4 samples/s +Step 8120/44000 | Loss: 3.1513 | LR: 4.65e-04 | 1388.0ms/step | 14.4 samples/s +Step 8130/44000 | Loss: 3.1540 | LR: 4.65e-04 | 1389.5ms/step | 14.4 samples/s +Step 8140/44000 | Loss: 3.1893 | LR: 4.65e-04 | 1390.5ms/step | 14.4 samples/s +Step 8150/44000 | Loss: 3.1523 | LR: 4.65e-04 | 1388.4ms/step | 14.4 samples/s +Step 8160/44000 | Loss: 3.1432 | LR: 4.65e-04 | 1388.6ms/step | 14.4 samples/s +Step 8170/44000 | Loss: 3.1667 | LR: 4.65e-04 | 1390.9ms/step | 14.4 samples/s +Step 8180/44000 | Loss: 3.1569 | LR: 4.64e-04 | 1389.9ms/step | 14.4 samples/s +Step 8190/44000 | Loss: 3.1505 | LR: 4.64e-04 | 1386.6ms/step | 14.4 samples/s +Step 8200/44000 | Loss: 3.1756 | LR: 4.64e-04 | 1388.1ms/step | 14.4 samples/s +Step 8210/44000 | Loss: 3.1763 | LR: 4.64e-04 | 1387.1ms/step | 14.4 samples/s +Step 8220/44000 | Loss: 3.1824 | LR: 4.64e-04 | 1381.7ms/step | 14.5 samples/s +Step 8230/44000 | Loss: 3.1429 | LR: 4.64e-04 | 1382.9ms/step | 14.5 samples/s +Step 8240/44000 | Loss: 3.1544 | LR: 4.64e-04 | 1385.8ms/step | 14.4 samples/s +Step 8250/44000 | Loss: 3.1533 | LR: 4.64e-04 | 1387.1ms/step | 14.4 samples/s +Validation at step 8250: Loss = 3.1681, Perplexity = 23.76 +Step 8260/44000 | Loss: 3.1376 | LR: 4.64e-04 | 3586.7ms/step | 5.6 samples/s +Step 8270/44000 | Loss: 3.1801 | LR: 4.64e-04 | 1390.1ms/step | 14.4 samples/s +Step 8280/44000 | Loss: 3.1593 | LR: 4.64e-04 | 1390.1ms/step | 14.4 samples/s +Step 8290/44000 | Loss: 3.1618 | LR: 4.63e-04 | 1388.3ms/step | 14.4 samples/s +Step 8300/44000 | Loss: 3.1184 | LR: 4.63e-04 | 1389.9ms/step | 14.4 samples/s +Step 8310/44000 | Loss: 3.1688 | LR: 4.63e-04 | 1389.7ms/step | 14.4 samples/s +Step 8320/44000 | Loss: 3.1438 | LR: 4.63e-04 | 1385.4ms/step | 14.4 samples/s +Step 8330/44000 | Loss: 3.1499 | LR: 4.63e-04 | 1384.3ms/step | 14.4 samples/s +Step 8340/44000 | Loss: 3.2031 | LR: 4.63e-04 | 1388.3ms/step | 14.4 samples/s +Step 8350/44000 | Loss: 3.1487 | LR: 4.63e-04 | 1386.4ms/step | 14.4 samples/s +Step 8360/44000 | Loss: 3.1700 | LR: 4.63e-04 | 1385.9ms/step | 14.4 samples/s +Step 8370/44000 | Loss: 3.1566 | LR: 4.63e-04 | 1386.0ms/step | 14.4 samples/s +Step 8380/44000 | Loss: 3.1975 | LR: 4.63e-04 | 1388.1ms/step | 14.4 samples/s +Step 8390/44000 | Loss: 3.1289 | LR: 4.62e-04 | 1388.1ms/step | 14.4 samples/s +Step 8400/44000 | Loss: 3.1446 | LR: 4.62e-04 | 1388.6ms/step | 14.4 samples/s +Step 8410/44000 | Loss: 3.1406 | LR: 4.62e-04 | 1391.0ms/step | 14.4 samples/s +Step 8420/44000 | Loss: 3.1823 | LR: 4.62e-04 | 1389.0ms/step | 14.4 samples/s +Step 8430/44000 | Loss: 3.1449 | LR: 4.62e-04 | 1388.4ms/step | 14.4 samples/s +Step 8440/44000 | Loss: 3.1709 | LR: 4.62e-04 | 1388.7ms/step | 14.4 samples/s +Step 8450/44000 | Loss: 3.1229 | LR: 4.62e-04 | 1389.3ms/step | 14.4 samples/s +Step 8460/44000 | Loss: 3.1721 | LR: 4.62e-04 | 1385.5ms/step | 14.4 samples/s +Step 8470/44000 | Loss: 3.1195 | LR: 4.62e-04 | 1386.2ms/step | 14.4 samples/s +Step 8480/44000 | Loss: 3.1458 | LR: 4.62e-04 | 1385.1ms/step | 14.4 samples/s +Step 8490/44000 | Loss: 3.1665 | LR: 4.62e-04 | 1385.0ms/step | 14.4 samples/s +Step 8500/44000 | Loss: 3.1740 | LR: 4.61e-04 | 1387.0ms/step | 14.4 samples/s +Validation at step 8500: Loss = 3.1612, Perplexity = 23.60 +Step 8510/44000 | Loss: 3.1715 | LR: 4.61e-04 | 3588.7ms/step | 5.6 samples/s +Step 8520/44000 | Loss: 3.1422 | LR: 4.61e-04 | 1391.4ms/step | 14.4 samples/s +Step 8530/44000 | Loss: 3.1699 | LR: 4.61e-04 | 1392.1ms/step | 14.4 samples/s +Step 8540/44000 | Loss: 3.1295 | LR: 4.61e-04 | 1389.2ms/step | 14.4 samples/s +Step 8550/44000 | Loss: 3.1491 | LR: 4.61e-04 | 1391.0ms/step | 14.4 samples/s +Step 8560/44000 | Loss: 3.1554 | LR: 4.61e-04 | 1388.5ms/step | 14.4 samples/s +Step 8570/44000 | Loss: 3.1221 | LR: 4.61e-04 | 1387.8ms/step | 14.4 samples/s +Step 8580/44000 | Loss: 3.1525 | LR: 4.61e-04 | 1388.4ms/step | 14.4 samples/s +Step 8590/44000 | Loss: 3.1516 | LR: 4.61e-04 | 1386.6ms/step | 14.4 samples/s +Step 8600/44000 | Loss: 3.1421 | LR: 4.60e-04 | 1389.7ms/step | 14.4 samples/s +Step 8610/44000 | Loss: 3.1212 | LR: 4.60e-04 | 1385.2ms/step | 14.4 samples/s +Step 8620/44000 | Loss: 3.1502 | LR: 4.60e-04 | 1386.4ms/step | 14.4 samples/s +Step 8630/44000 | Loss: 3.1397 | LR: 4.60e-04 | 1387.1ms/step | 14.4 samples/s +Step 8640/44000 | Loss: 3.1468 | LR: 4.60e-04 | 1389.0ms/step | 14.4 samples/s +Step 8650/44000 | Loss: 3.1517 | LR: 4.60e-04 | 1389.2ms/step | 14.4 samples/s +Step 8660/44000 | Loss: 3.1132 | LR: 4.60e-04 | 1390.7ms/step | 14.4 samples/s +Step 8670/44000 | Loss: 3.1544 | LR: 4.60e-04 | 1389.1ms/step | 14.4 samples/s +Step 8680/44000 | Loss: 3.1519 | LR: 4.60e-04 | 1392.0ms/step | 14.4 samples/s +Step 8690/44000 | Loss: 3.1337 | LR: 4.60e-04 | 1390.1ms/step | 14.4 samples/s +Step 8700/44000 | Loss: 3.1111 | LR: 4.59e-04 | 1390.6ms/step | 14.4 samples/s +Step 8710/44000 | Loss: 3.1423 | LR: 4.59e-04 | 1391.0ms/step | 14.4 samples/s +Step 8720/44000 | Loss: 3.1461 | LR: 4.59e-04 | 1389.9ms/step | 14.4 samples/s +Step 8730/44000 | Loss: 3.1504 | LR: 4.59e-04 | 1386.5ms/step | 14.4 samples/s +Step 8740/44000 | Loss: 3.1433 | LR: 4.59e-04 | 1387.9ms/step | 14.4 samples/s +Step 8750/44000 | Loss: 3.1149 | LR: 4.59e-04 | 1388.2ms/step | 14.4 samples/s +Validation at step 8750: Loss = 3.1549, Perplexity = 23.45 +Step 8760/44000 | Loss: 3.1282 | LR: 4.59e-04 | 3581.6ms/step | 5.6 samples/s +Step 8770/44000 | Loss: 3.1702 | LR: 4.59e-04 | 1388.5ms/step | 14.4 samples/s +Step 8780/44000 | Loss: 3.1339 | LR: 4.59e-04 | 1389.6ms/step | 14.4 samples/s +Step 8790/44000 | Loss: 3.1222 | LR: 4.59e-04 | 1388.6ms/step | 14.4 samples/s +Step 8800/44000 | Loss: 3.1159 | LR: 4.58e-04 | 1386.3ms/step | 14.4 samples/s +Step 8810/44000 | Loss: 3.1383 | LR: 4.58e-04 | 1385.3ms/step | 14.4 samples/s +Step 8820/44000 | Loss: 3.1218 | LR: 4.58e-04 | 1387.1ms/step | 14.4 samples/s +Step 8830/44000 | Loss: 3.1278 | LR: 4.58e-04 | 1388.6ms/step | 14.4 samples/s +Step 8840/44000 | Loss: 3.1367 | LR: 4.58e-04 | 1388.4ms/step | 14.4 samples/s +Step 8850/44000 | Loss: 3.1388 | LR: 4.58e-04 | 1390.0ms/step | 14.4 samples/s +Step 8860/44000 | Loss: 3.1568 | LR: 4.58e-04 | 1386.9ms/step | 14.4 samples/s +Step 8870/44000 | Loss: 3.1393 | LR: 4.58e-04 | 1388.6ms/step | 14.4 samples/s +Step 8880/44000 | Loss: 3.1187 | LR: 4.58e-04 | 1386.6ms/step | 14.4 samples/s +Step 8890/44000 | Loss: 3.1648 | LR: 4.58e-04 | 1387.6ms/step | 14.4 samples/s +Step 8900/44000 | Loss: 3.1257 | LR: 4.57e-04 | 1390.1ms/step | 14.4 samples/s +Step 8910/44000 | Loss: 3.1607 | LR: 4.57e-04 | 1391.2ms/step | 14.4 samples/s +Step 8920/44000 | Loss: 3.1414 | LR: 4.57e-04 | 1387.1ms/step | 14.4 samples/s +Step 8930/44000 | Loss: 3.1385 | LR: 4.57e-04 | 1389.9ms/step | 14.4 samples/s +Step 8940/44000 | Loss: 3.1309 | LR: 4.57e-04 | 1387.8ms/step | 14.4 samples/s +Step 8950/44000 | Loss: 3.1343 | LR: 4.57e-04 | 1387.6ms/step | 14.4 samples/s +Step 8960/44000 | Loss: 3.1186 | LR: 4.57e-04 | 1387.4ms/step | 14.4 samples/s +Step 8970/44000 | Loss: 3.1414 | LR: 4.57e-04 | 1389.4ms/step | 14.4 samples/s +Step 8980/44000 | Loss: 3.1420 | LR: 4.57e-04 | 1388.1ms/step | 14.4 samples/s +Step 8990/44000 | Loss: 3.1275 | LR: 4.57e-04 | 1387.9ms/step | 14.4 samples/s +Step 9000/44000 | Loss: 3.1275 | LR: 4.56e-04 | 1388.6ms/step | 14.4 samples/s +Validation at step 9000: Loss = 3.1481, Perplexity = 23.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-9000 +Step 9010/44000 | Loss: 3.1279 | LR: 4.56e-04 | 3881.1ms/step | 5.2 samples/s +Step 9020/44000 | Loss: 3.1449 | LR: 4.56e-04 | 1387.3ms/step | 14.4 samples/s +Step 9030/44000 | Loss: 3.1195 | LR: 4.56e-04 | 1388.8ms/step | 14.4 samples/s +Step 9040/44000 | Loss: 3.1228 | LR: 4.56e-04 | 1389.9ms/step | 14.4 samples/s +Step 9050/44000 | Loss: 3.1246 | LR: 4.56e-04 | 1389.6ms/step | 14.4 samples/s +Step 9060/44000 | Loss: 3.1627 | LR: 4.56e-04 | 1387.4ms/step | 14.4 samples/s +Step 9070/44000 | Loss: 3.1565 | LR: 4.56e-04 | 1389.2ms/step | 14.4 samples/s +Step 9080/44000 | Loss: 3.1781 | LR: 4.56e-04 | 1388.0ms/step | 14.4 samples/s +Step 9090/44000 | Loss: 3.1065 | LR: 4.56e-04 | 1387.8ms/step | 14.4 samples/s +Step 9100/44000 | Loss: 3.1119 | LR: 4.55e-04 | 1388.2ms/step | 14.4 samples/s +Step 9110/44000 | Loss: 3.1097 | LR: 4.55e-04 | 1386.7ms/step | 14.4 samples/s +Step 9120/44000 | Loss: 3.1120 | LR: 4.55e-04 | 1379.0ms/step | 14.5 samples/s +Step 9130/44000 | Loss: 3.1573 | LR: 4.55e-04 | 1380.7ms/step | 14.5 samples/s +Step 9140/44000 | Loss: 3.1071 | LR: 4.55e-04 | 1381.4ms/step | 14.5 samples/s +Step 9150/44000 | Loss: 3.1350 | LR: 4.55e-04 | 1383.7ms/step | 14.5 samples/s +Step 9160/44000 | Loss: 3.1100 | LR: 4.55e-04 | 1386.4ms/step | 14.4 samples/s +Step 9170/44000 | Loss: 3.0901 | LR: 4.55e-04 | 1387.7ms/step | 14.4 samples/s +Step 9180/44000 | Loss: 3.1398 | LR: 4.55e-04 | 1388.5ms/step | 14.4 samples/s +Step 9190/44000 | Loss: 3.1144 | LR: 4.54e-04 | 1385.8ms/step | 14.4 samples/s +Step 9200/44000 | Loss: 3.1423 | LR: 4.54e-04 | 1389.9ms/step | 14.4 samples/s +Step 9210/44000 | Loss: 3.1371 | LR: 4.54e-04 | 1387.7ms/step | 14.4 samples/s +Step 9220/44000 | Loss: 3.1274 | LR: 4.54e-04 | 1388.4ms/step | 14.4 samples/s +Step 9230/44000 | Loss: 3.1285 | LR: 4.54e-04 | 1389.9ms/step | 14.4 samples/s +Step 9240/44000 | Loss: 3.1782 | LR: 4.54e-04 | 1390.5ms/step | 14.4 samples/s +Step 9250/44000 | Loss: 3.1310 | LR: 4.54e-04 | 1387.4ms/step | 14.4 samples/s +Validation at step 9250: Loss = 3.1437, Perplexity = 23.19 +Step 9260/44000 | Loss: 3.1379 | LR: 4.54e-04 | 3581.7ms/step | 5.6 samples/s +Step 9270/44000 | Loss: 3.1355 | LR: 4.54e-04 | 1385.2ms/step | 14.4 samples/s +Step 9280/44000 | Loss: 3.1341 | LR: 4.54e-04 | 1385.5ms/step | 14.4 samples/s +Step 9290/44000 | Loss: 3.1448 | LR: 4.53e-04 | 1384.4ms/step | 14.4 samples/s +Step 9300/44000 | Loss: 3.1047 | LR: 4.53e-04 | 1387.1ms/step | 14.4 samples/s +Step 9310/44000 | Loss: 3.1270 | LR: 4.53e-04 | 1388.2ms/step | 14.4 samples/s +Step 9320/44000 | Loss: 3.1088 | LR: 4.53e-04 | 1387.7ms/step | 14.4 samples/s +Step 9330/44000 | Loss: 3.1295 | LR: 4.53e-04 | 1388.2ms/step | 14.4 samples/s +Step 9340/44000 | Loss: 3.1105 | LR: 4.53e-04 | 1389.9ms/step | 14.4 samples/s +Step 9350/44000 | Loss: 3.1048 | LR: 4.53e-04 | 1388.2ms/step | 14.4 samples/s +Step 9360/44000 | Loss: 3.1291 | LR: 4.53e-04 | 1389.7ms/step | 14.4 samples/s +Step 9370/44000 | Loss: 3.1308 | LR: 4.53e-04 | 1388.3ms/step | 14.4 samples/s +Step 9380/44000 | Loss: 3.1297 | LR: 4.52e-04 | 1389.4ms/step | 14.4 samples/s +Step 9390/44000 | Loss: 3.1401 | LR: 4.52e-04 | 1387.6ms/step | 14.4 samples/s +Step 9400/44000 | Loss: 3.1168 | LR: 4.52e-04 | 1386.3ms/step | 14.4 samples/s +Step 9410/44000 | Loss: 3.1224 | LR: 4.52e-04 | 1383.9ms/step | 14.5 samples/s +Step 9420/44000 | Loss: 3.1345 | LR: 4.52e-04 | 1385.0ms/step | 14.4 samples/s +Step 9430/44000 | Loss: 3.1198 | LR: 4.52e-04 | 1388.6ms/step | 14.4 samples/s +Step 9440/44000 | Loss: 3.0978 | LR: 4.52e-04 | 1390.2ms/step | 14.4 samples/s +Step 9450/44000 | Loss: 3.1197 | LR: 4.52e-04 | 1385.2ms/step | 14.4 samples/s +Step 9460/44000 | Loss: 3.1318 | LR: 4.52e-04 | 1385.2ms/step | 14.4 samples/s +Step 9470/44000 | Loss: 3.1144 | LR: 4.52e-04 | 1386.1ms/step | 14.4 samples/s +Step 9480/44000 | Loss: 3.1105 | LR: 4.51e-04 | 1386.0ms/step | 14.4 samples/s +Step 9490/44000 | Loss: 3.1254 | LR: 4.51e-04 | 1384.5ms/step | 14.4 samples/s +Step 9500/44000 | Loss: 3.1290 | LR: 4.51e-04 | 1387.2ms/step | 14.4 samples/s +Validation at step 9500: Loss = 3.1368, Perplexity = 23.03 +Step 9510/44000 | Loss: 3.1588 | LR: 4.51e-04 | 3582.2ms/step | 5.6 samples/s +Step 9520/44000 | Loss: 3.1106 | LR: 4.51e-04 | 1384.8ms/step | 14.4 samples/s +Step 9530/44000 | Loss: 3.1500 | LR: 4.51e-04 | 1387.1ms/step | 14.4 samples/s +Step 9540/44000 | Loss: 3.1491 | LR: 4.51e-04 | 1386.7ms/step | 14.4 samples/s +Step 9550/44000 | Loss: 3.1346 | LR: 4.51e-04 | 1391.6ms/step | 14.4 samples/s +Step 9560/44000 | Loss: 3.1153 | LR: 4.51e-04 | 1388.2ms/step | 14.4 samples/s +Step 9570/44000 | Loss: 3.1124 | LR: 4.50e-04 | 1390.6ms/step | 14.4 samples/s +Step 9580/44000 | Loss: 3.1412 | LR: 4.50e-04 | 1389.4ms/step | 14.4 samples/s +Step 9590/44000 | Loss: 3.1063 | LR: 4.50e-04 | 1387.2ms/step | 14.4 samples/s +Step 9600/44000 | Loss: 3.1094 | LR: 4.50e-04 | 1386.9ms/step | 14.4 samples/s +Step 9610/44000 | Loss: 3.1102 | LR: 4.50e-04 | 1386.0ms/step | 14.4 samples/s +Step 9620/44000 | Loss: 3.1498 | LR: 4.50e-04 | 1388.3ms/step | 14.4 samples/s +Step 9630/44000 | Loss: 3.1259 | LR: 4.50e-04 | 1389.6ms/step | 14.4 samples/s +Step 9640/44000 | Loss: 3.1226 | LR: 4.50e-04 | 1388.3ms/step | 14.4 samples/s +Step 9650/44000 | Loss: 3.1252 | LR: 4.50e-04 | 1386.2ms/step | 14.4 samples/s +Step 9660/44000 | Loss: 3.0988 | LR: 4.49e-04 | 1388.0ms/step | 14.4 samples/s +Step 9670/44000 | Loss: 3.1216 | LR: 4.49e-04 | 1387.6ms/step | 14.4 samples/s +Step 9680/44000 | Loss: 3.1264 | LR: 4.49e-04 | 1388.8ms/step | 14.4 samples/s +Step 9690/44000 | Loss: 3.1256 | LR: 4.49e-04 | 1387.3ms/step | 14.4 samples/s +Step 9700/44000 | Loss: 3.1286 | LR: 4.49e-04 | 1387.2ms/step | 14.4 samples/s +Step 9710/44000 | Loss: 3.1270 | LR: 4.49e-04 | 1386.8ms/step | 14.4 samples/s +Step 9720/44000 | Loss: 3.1151 | LR: 4.49e-04 | 1387.9ms/step | 14.4 samples/s +Step 9730/44000 | Loss: 3.1279 | LR: 4.49e-04 | 1388.5ms/step | 14.4 samples/s +Step 9740/44000 | Loss: 3.1160 | LR: 4.49e-04 | 1387.6ms/step | 14.4 samples/s +Step 9750/44000 | Loss: 3.1118 | LR: 4.48e-04 | 1386.1ms/step | 14.4 samples/s +Validation at step 9750: Loss = 3.1300, Perplexity = 22.87 +Step 9760/44000 | Loss: 3.0882 | LR: 4.48e-04 | 3586.0ms/step | 5.6 samples/s +Step 9770/44000 | Loss: 3.0912 | LR: 4.48e-04 | 1388.6ms/step | 14.4 samples/s +Step 9780/44000 | Loss: 3.1163 | LR: 4.48e-04 | 1389.2ms/step | 14.4 samples/s +Step 9790/44000 | Loss: 3.1329 | LR: 4.48e-04 | 1391.0ms/step | 14.4 samples/s +Step 9800/44000 | Loss: 3.1165 | LR: 4.48e-04 | 1390.7ms/step | 14.4 samples/s +Step 9810/44000 | Loss: 3.1344 | LR: 4.48e-04 | 1387.8ms/step | 14.4 samples/s +Step 9820/44000 | Loss: 3.1348 | LR: 4.48e-04 | 1388.5ms/step | 14.4 samples/s +Step 9830/44000 | Loss: 3.1618 | LR: 4.48e-04 | 1388.0ms/step | 14.4 samples/s +Step 9840/44000 | Loss: 3.1074 | LR: 4.47e-04 | 1388.2ms/step | 14.4 samples/s +Step 9850/44000 | Loss: 3.1269 | LR: 4.47e-04 | 1389.8ms/step | 14.4 samples/s +Step 9860/44000 | Loss: 3.0867 | LR: 4.47e-04 | 1388.5ms/step | 14.4 samples/s +Step 9870/44000 | Loss: 3.1123 | LR: 4.47e-04 | 1390.4ms/step | 14.4 samples/s +Step 9880/44000 | Loss: 3.1159 | LR: 4.47e-04 | 1387.9ms/step | 14.4 samples/s +Step 9890/44000 | Loss: 3.0970 | LR: 4.47e-04 | 1388.4ms/step | 14.4 samples/s +Step 9900/44000 | Loss: 3.1213 | LR: 4.47e-04 | 1388.2ms/step | 14.4 samples/s +Step 9910/44000 | Loss: 3.1375 | LR: 4.47e-04 | 1388.5ms/step | 14.4 samples/s +Step 9920/44000 | Loss: 3.1007 | LR: 4.47e-04 | 1386.5ms/step | 14.4 samples/s +Step 9930/44000 | Loss: 3.0955 | LR: 4.46e-04 | 1387.7ms/step | 14.4 samples/s +Step 9940/44000 | Loss: 3.1175 | LR: 4.46e-04 | 1390.0ms/step | 14.4 samples/s +Step 9950/44000 | Loss: 3.1007 | LR: 4.46e-04 | 1388.1ms/step | 14.4 samples/s +Step 9960/44000 | Loss: 3.1147 | LR: 4.46e-04 | 1387.4ms/step | 14.4 samples/s +Step 9970/44000 | Loss: 3.1570 | LR: 4.46e-04 | 1390.2ms/step | 14.4 samples/s +Step 9980/44000 | Loss: 3.1070 | LR: 4.46e-04 | 1387.1ms/step | 14.4 samples/s +Step 9990/44000 | Loss: 3.1167 | LR: 4.46e-04 | 1384.6ms/step | 14.4 samples/s +Step 10000/44000 | Loss: 3.1116 | LR: 4.46e-04 | 1386.3ms/step | 14.4 samples/s +Validation at step 10000: Loss = 3.1236, Perplexity = 22.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-10000 +Step 10010/44000 | Loss: 3.1063 | LR: 4.46e-04 | 3882.0ms/step | 5.2 samples/s +Step 10020/44000 | Loss: 3.1371 | LR: 4.45e-04 | 1386.1ms/step | 14.4 samples/s +Step 10030/44000 | Loss: 3.1163 | LR: 4.45e-04 | 1381.7ms/step | 14.5 samples/s +Step 10040/44000 | Loss: 3.0851 | LR: 4.45e-04 | 1384.5ms/step | 14.4 samples/s +Step 10050/44000 | Loss: 3.0960 | LR: 4.45e-04 | 1387.5ms/step | 14.4 samples/s +Step 10060/44000 | Loss: 3.0944 | LR: 4.45e-04 | 1387.5ms/step | 14.4 samples/s +Step 10070/44000 | Loss: 3.0974 | LR: 4.45e-04 | 1387.3ms/step | 14.4 samples/s +Step 10080/44000 | Loss: 3.1073 | LR: 4.45e-04 | 1387.8ms/step | 14.4 samples/s +Step 10090/44000 | Loss: 3.1042 | LR: 4.45e-04 | 1387.4ms/step | 14.4 samples/s +Step 10100/44000 | Loss: 3.1226 | LR: 4.45e-04 | 1387.9ms/step | 14.4 samples/s +Step 10110/44000 | Loss: 3.1312 | LR: 4.44e-04 | 1387.6ms/step | 14.4 samples/s +Step 10120/44000 | Loss: 3.1534 | LR: 4.44e-04 | 1387.1ms/step | 14.4 samples/s +Step 10130/44000 | Loss: 3.1000 | LR: 4.44e-04 | 1387.7ms/step | 14.4 samples/s +Step 10140/44000 | Loss: 3.1581 | LR: 4.44e-04 | 1386.5ms/step | 14.4 samples/s +Step 10150/44000 | Loss: 3.1161 | LR: 4.44e-04 | 1388.5ms/step | 14.4 samples/s +Step 10160/44000 | Loss: 3.1172 | LR: 4.44e-04 | 1388.8ms/step | 14.4 samples/s +Step 10170/44000 | Loss: 3.0871 | LR: 4.44e-04 | 1385.8ms/step | 14.4 samples/s +Step 10180/44000 | Loss: 3.1218 | LR: 4.44e-04 | 1386.4ms/step | 14.4 samples/s +Step 10190/44000 | Loss: 3.0937 | LR: 4.43e-04 | 1386.0ms/step | 14.4 samples/s +Step 10200/44000 | Loss: 3.0959 | LR: 4.43e-04 | 1388.6ms/step | 14.4 samples/s +Step 10210/44000 | Loss: 3.1137 | LR: 4.43e-04 | 1386.5ms/step | 14.4 samples/s +Step 10220/44000 | Loss: 3.0975 | LR: 4.43e-04 | 1387.9ms/step | 14.4 samples/s +Step 10230/44000 | Loss: 3.1205 | LR: 4.43e-04 | 1388.0ms/step | 14.4 samples/s +Step 10240/44000 | Loss: 3.0981 | LR: 4.43e-04 | 1391.1ms/step | 14.4 samples/s +Step 10250/44000 | Loss: 3.1200 | LR: 4.43e-04 | 1388.3ms/step | 14.4 samples/s +Validation at step 10250: Loss = 3.1188, Perplexity = 22.62 +Step 10260/44000 | Loss: 3.0783 | LR: 4.43e-04 | 3587.2ms/step | 5.6 samples/s +Step 10270/44000 | Loss: 3.0667 | LR: 4.43e-04 | 1387.7ms/step | 14.4 samples/s +Step 10280/44000 | Loss: 3.0673 | LR: 4.42e-04 | 1387.0ms/step | 14.4 samples/s +Step 10290/44000 | Loss: 3.0626 | LR: 4.42e-04 | 1386.8ms/step | 14.4 samples/s +Step 10300/44000 | Loss: 3.0944 | LR: 4.42e-04 | 1388.9ms/step | 14.4 samples/s +Step 10310/44000 | Loss: 3.1197 | LR: 4.42e-04 | 1388.4ms/step | 14.4 samples/s +Step 10320/44000 | Loss: 3.0857 | LR: 4.42e-04 | 1386.3ms/step | 14.4 samples/s +Step 10330/44000 | Loss: 3.1279 | LR: 4.42e-04 | 1384.2ms/step | 14.4 samples/s +Step 10340/44000 | Loss: 3.1013 | LR: 4.42e-04 | 1386.3ms/step | 14.4 samples/s +Step 10350/44000 | Loss: 3.1151 | LR: 4.42e-04 | 1386.3ms/step | 14.4 samples/s +Step 10360/44000 | Loss: 3.0862 | LR: 4.42e-04 | 1386.8ms/step | 14.4 samples/s +Step 10370/44000 | Loss: 3.1124 | LR: 4.41e-04 | 1388.0ms/step | 14.4 samples/s +Step 10380/44000 | Loss: 3.1252 | LR: 4.41e-04 | 1391.8ms/step | 14.4 samples/s +Step 10390/44000 | Loss: 3.0951 | LR: 4.41e-04 | 1390.3ms/step | 14.4 samples/s +Step 10400/44000 | Loss: 3.1273 | LR: 4.41e-04 | 1389.7ms/step | 14.4 samples/s +Step 10410/44000 | Loss: 3.1331 | LR: 4.41e-04 | 1388.2ms/step | 14.4 samples/s +Step 10420/44000 | Loss: 3.1267 | LR: 4.41e-04 | 1389.9ms/step | 14.4 samples/s +Step 10430/44000 | Loss: 3.1165 | LR: 4.41e-04 | 1388.5ms/step | 14.4 samples/s +Step 10440/44000 | Loss: 3.0873 | LR: 4.41e-04 | 1388.0ms/step | 14.4 samples/s +Step 10450/44000 | Loss: 3.1002 | LR: 4.40e-04 | 1386.2ms/step | 14.4 samples/s +Step 10460/44000 | Loss: 3.0936 | LR: 4.40e-04 | 1386.6ms/step | 14.4 samples/s +Step 10470/44000 | Loss: 3.1249 | LR: 4.40e-04 | 1387.5ms/step | 14.4 samples/s +Step 10480/44000 | Loss: 3.0930 | LR: 4.40e-04 | 1385.7ms/step | 14.4 samples/s +Step 10490/44000 | Loss: 3.0769 | LR: 4.40e-04 | 1385.6ms/step | 14.4 samples/s +Step 10500/44000 | Loss: 3.1308 | LR: 4.40e-04 | 1386.6ms/step | 14.4 samples/s +Validation at step 10500: Loss = 3.1160, Perplexity = 22.56 +Step 10510/44000 | Loss: 3.1395 | LR: 4.40e-04 | 3580.5ms/step | 5.6 samples/s +Step 10520/44000 | Loss: 3.0763 | LR: 4.40e-04 | 1386.9ms/step | 14.4 samples/s +Step 10530/44000 | Loss: 3.0945 | LR: 4.40e-04 | 1388.0ms/step | 14.4 samples/s +Step 10540/44000 | Loss: 3.1024 | LR: 4.39e-04 | 1386.2ms/step | 14.4 samples/s +Step 10550/44000 | Loss: 3.0952 | LR: 4.39e-04 | 1385.9ms/step | 14.4 samples/s +Step 10560/44000 | Loss: 3.0820 | LR: 4.39e-04 | 1389.4ms/step | 14.4 samples/s +Step 10570/44000 | Loss: 3.0971 | LR: 4.39e-04 | 1389.0ms/step | 14.4 samples/s +Step 10580/44000 | Loss: 3.1129 | LR: 4.39e-04 | 1385.9ms/step | 14.4 samples/s +Step 10590/44000 | Loss: 3.0646 | LR: 4.39e-04 | 1386.8ms/step | 14.4 samples/s +Step 10600/44000 | Loss: 3.0861 | LR: 4.39e-04 | 1389.2ms/step | 14.4 samples/s +Step 10610/44000 | Loss: 3.0744 | LR: 4.39e-04 | 1387.9ms/step | 14.4 samples/s +Step 10620/44000 | Loss: 3.1212 | LR: 4.38e-04 | 1389.0ms/step | 14.4 samples/s +Step 10630/44000 | Loss: 3.1112 | LR: 4.38e-04 | 1389.1ms/step | 14.4 samples/s +Step 10640/44000 | Loss: 3.0896 | LR: 4.38e-04 | 1390.1ms/step | 14.4 samples/s +Step 10650/44000 | Loss: 3.1029 | LR: 4.38e-04 | 1386.0ms/step | 14.4 samples/s +Step 10660/44000 | Loss: 3.0916 | LR: 4.38e-04 | 1387.4ms/step | 14.4 samples/s +Step 10670/44000 | Loss: 3.0950 | LR: 4.38e-04 | 1387.9ms/step | 14.4 samples/s +Step 10680/44000 | Loss: 3.1116 | LR: 4.38e-04 | 1386.6ms/step | 14.4 samples/s +Step 10690/44000 | Loss: 3.0644 | LR: 4.38e-04 | 1388.5ms/step | 14.4 samples/s +Step 10700/44000 | Loss: 3.0370 | LR: 4.37e-04 | 1387.6ms/step | 14.4 samples/s +Step 10710/44000 | Loss: 3.0716 | LR: 4.37e-04 | 1388.4ms/step | 14.4 samples/s +Step 10720/44000 | Loss: 3.0604 | LR: 4.37e-04 | 1391.4ms/step | 14.4 samples/s +Step 10730/44000 | Loss: 3.0700 | LR: 4.37e-04 | 1388.1ms/step | 14.4 samples/s +Step 10740/44000 | Loss: 3.0818 | LR: 4.37e-04 | 1388.1ms/step | 14.4 samples/s +Step 10750/44000 | Loss: 3.0738 | LR: 4.37e-04 | 1390.3ms/step | 14.4 samples/s +Validation at step 10750: Loss = 3.1084, Perplexity = 22.38 +Step 10760/44000 | Loss: 3.1107 | LR: 4.37e-04 | 3586.6ms/step | 5.6 samples/s +Step 10770/44000 | Loss: 3.1120 | LR: 4.37e-04 | 1389.1ms/step | 14.4 samples/s +Step 10780/44000 | Loss: 3.0876 | LR: 4.37e-04 | 1390.4ms/step | 14.4 samples/s +Step 10790/44000 | Loss: 3.0962 | LR: 4.36e-04 | 1389.3ms/step | 14.4 samples/s +Step 10800/44000 | Loss: 3.0802 | LR: 4.36e-04 | 1388.3ms/step | 14.4 samples/s +Step 10810/44000 | Loss: 3.0976 | LR: 4.36e-04 | 1388.0ms/step | 14.4 samples/s +Step 10820/44000 | Loss: 3.1060 | LR: 4.36e-04 | 1388.6ms/step | 14.4 samples/s +Step 10830/44000 | Loss: 3.0907 | LR: 4.36e-04 | 1387.6ms/step | 14.4 samples/s +Step 10840/44000 | Loss: 3.1141 | LR: 4.36e-04 | 1389.8ms/step | 14.4 samples/s +Step 10850/44000 | Loss: 3.1076 | LR: 4.36e-04 | 1389.8ms/step | 14.4 samples/s +Step 10860/44000 | Loss: 3.1258 | LR: 4.36e-04 | 1390.2ms/step | 14.4 samples/s +Step 10870/44000 | Loss: 3.0904 | LR: 4.35e-04 | 1390.2ms/step | 14.4 samples/s +Step 10880/44000 | Loss: 3.0860 | LR: 4.35e-04 | 1389.1ms/step | 14.4 samples/s +Step 10890/44000 | Loss: 3.0940 | LR: 4.35e-04 | 1388.7ms/step | 14.4 samples/s +Step 10900/44000 | Loss: 3.0956 | LR: 4.35e-04 | 1387.3ms/step | 14.4 samples/s +Step 10910/44000 | Loss: 3.0723 | LR: 4.35e-04 | 1386.9ms/step | 14.4 samples/s +Step 10920/44000 | Loss: 3.0591 | LR: 4.35e-04 | 1386.7ms/step | 14.4 samples/s +Step 10930/44000 | Loss: 3.1273 | LR: 4.35e-04 | 1388.6ms/step | 14.4 samples/s +Step 10940/44000 | Loss: 3.0921 | LR: 4.35e-04 | 1380.3ms/step | 14.5 samples/s +Step 10950/44000 | Loss: 3.0743 | LR: 4.34e-04 | 1380.1ms/step | 14.5 samples/s +Step 10960/44000 | Loss: 3.1026 | LR: 4.34e-04 | 1382.0ms/step | 14.5 samples/s +Step 10970/44000 | Loss: 3.0883 | LR: 4.34e-04 | 1383.6ms/step | 14.5 samples/s +Step 10980/44000 | Loss: 3.0793 | LR: 4.34e-04 | 1385.6ms/step | 14.4 samples/s +Step 10990/44000 | Loss: 3.0933 | LR: 4.34e-04 | 1386.1ms/step | 14.4 samples/s +Step 11000/44000 | Loss: 3.0848 | LR: 4.34e-04 | 1385.7ms/step | 14.4 samples/s +Validation at step 11000: Loss = 3.1036, Perplexity = 22.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-11000 +Step 11010/44000 | Loss: 3.1001 | LR: 4.34e-04 | 3868.7ms/step | 5.2 samples/s +Step 11020/44000 | Loss: 3.1115 | LR: 4.34e-04 | 1386.5ms/step | 14.4 samples/s +Step 11030/44000 | Loss: 3.0567 | LR: 4.33e-04 | 1390.0ms/step | 14.4 samples/s +Step 11040/44000 | Loss: 3.0980 | LR: 4.33e-04 | 1390.2ms/step | 14.4 samples/s +Step 11050/44000 | Loss: 3.0799 | LR: 4.33e-04 | 1389.7ms/step | 14.4 samples/s +Step 11060/44000 | Loss: 3.0952 | LR: 4.33e-04 | 1390.0ms/step | 14.4 samples/s +Step 11070/44000 | Loss: 3.0883 | LR: 4.33e-04 | 1389.9ms/step | 14.4 samples/s +Step 11080/44000 | Loss: 3.0517 | LR: 4.33e-04 | 1390.3ms/step | 14.4 samples/s +Step 11090/44000 | Loss: 3.1114 | LR: 4.33e-04 | 1388.9ms/step | 14.4 samples/s +Step 11100/44000 | Loss: 3.0878 | LR: 4.33e-04 | 1387.8ms/step | 14.4 samples/s +Step 11110/44000 | Loss: 3.0512 | LR: 4.32e-04 | 1387.5ms/step | 14.4 samples/s +Step 11120/44000 | Loss: 3.0791 | LR: 4.32e-04 | 1387.3ms/step | 14.4 samples/s +Step 11130/44000 | Loss: 3.0755 | LR: 4.32e-04 | 1388.4ms/step | 14.4 samples/s +Step 11140/44000 | Loss: 3.0783 | LR: 4.32e-04 | 1389.6ms/step | 14.4 samples/s +Step 11150/44000 | Loss: 3.0999 | LR: 4.32e-04 | 1387.1ms/step | 14.4 samples/s +Step 11160/44000 | Loss: 3.0827 | LR: 4.32e-04 | 1390.0ms/step | 14.4 samples/s +Step 11170/44000 | Loss: 3.0787 | LR: 4.32e-04 | 1386.7ms/step | 14.4 samples/s +Step 11180/44000 | Loss: 3.0888 | LR: 4.32e-04 | 1386.6ms/step | 14.4 samples/s +Step 11190/44000 | Loss: 3.0845 | LR: 4.31e-04 | 1389.7ms/step | 14.4 samples/s +Step 11200/44000 | Loss: 3.0852 | LR: 4.31e-04 | 1388.2ms/step | 14.4 samples/s +Step 11210/44000 | Loss: 3.0887 | LR: 4.31e-04 | 1389.7ms/step | 14.4 samples/s +Step 11220/44000 | Loss: 3.0638 | LR: 4.31e-04 | 1390.2ms/step | 14.4 samples/s +Step 11230/44000 | Loss: 3.0896 | LR: 4.31e-04 | 1389.7ms/step | 14.4 samples/s +Step 11240/44000 | Loss: 3.0756 | LR: 4.31e-04 | 1388.3ms/step | 14.4 samples/s +Step 11250/44000 | Loss: 3.0838 | LR: 4.31e-04 | 1389.9ms/step | 14.4 samples/s +Validation at step 11250: Loss = 3.0981, Perplexity = 22.15 +Step 11260/44000 | Loss: 3.0586 | LR: 4.31e-04 | 3590.0ms/step | 5.6 samples/s +Step 11270/44000 | Loss: 3.1006 | LR: 4.30e-04 | 1390.8ms/step | 14.4 samples/s +Step 11280/44000 | Loss: 3.0760 | LR: 4.30e-04 | 1393.3ms/step | 14.4 samples/s +Step 11290/44000 | Loss: 3.0857 | LR: 4.30e-04 | 1389.6ms/step | 14.4 samples/s +Step 11300/44000 | Loss: 3.1029 | LR: 4.30e-04 | 1387.8ms/step | 14.4 samples/s +Step 11310/44000 | Loss: 3.0698 | LR: 4.30e-04 | 1388.1ms/step | 14.4 samples/s +Step 11320/44000 | Loss: 3.0841 | LR: 4.30e-04 | 1389.0ms/step | 14.4 samples/s +Step 11330/44000 | Loss: 3.0848 | LR: 4.30e-04 | 1388.2ms/step | 14.4 samples/s +Step 11340/44000 | Loss: 3.0555 | LR: 4.30e-04 | 1386.0ms/step | 14.4 samples/s +Step 11350/44000 | Loss: 3.0826 | LR: 4.29e-04 | 1388.4ms/step | 14.4 samples/s +Step 11360/44000 | Loss: 3.0651 | LR: 4.29e-04 | 1391.2ms/step | 14.4 samples/s +Step 11370/44000 | Loss: 3.0956 | LR: 4.29e-04 | 1392.3ms/step | 14.4 samples/s +Step 11380/44000 | Loss: 3.0969 | LR: 4.29e-04 | 1393.4ms/step | 14.4 samples/s +Step 11390/44000 | Loss: 3.1121 | LR: 4.29e-04 | 1388.1ms/step | 14.4 samples/s +Step 11400/44000 | Loss: 3.0754 | LR: 4.29e-04 | 1385.9ms/step | 14.4 samples/s +Step 11410/44000 | Loss: 3.0439 | LR: 4.29e-04 | 1382.7ms/step | 14.5 samples/s +Step 11420/44000 | Loss: 3.0947 | LR: 4.29e-04 | 1387.1ms/step | 14.4 samples/s +Step 11430/44000 | Loss: 3.0765 | LR: 4.28e-04 | 1386.7ms/step | 14.4 samples/s +Step 11440/44000 | Loss: 3.0996 | LR: 4.28e-04 | 1387.9ms/step | 14.4 samples/s +Step 11450/44000 | Loss: 3.0755 | LR: 4.28e-04 | 1387.7ms/step | 14.4 samples/s +Step 11460/44000 | Loss: 3.1087 | LR: 4.28e-04 | 1388.1ms/step | 14.4 samples/s +Step 11470/44000 | Loss: 3.0858 | LR: 4.28e-04 | 1387.5ms/step | 14.4 samples/s +Step 11480/44000 | Loss: 3.0739 | LR: 4.28e-04 | 1386.6ms/step | 14.4 samples/s +Step 11490/44000 | Loss: 3.0762 | LR: 4.28e-04 | 1387.8ms/step | 14.4 samples/s +Step 11500/44000 | Loss: 3.0774 | LR: 4.28e-04 | 1390.2ms/step | 14.4 samples/s +Validation at step 11500: Loss = 3.0933, Perplexity = 22.05 +Step 11510/44000 | Loss: 3.0932 | LR: 4.27e-04 | 3590.0ms/step | 5.6 samples/s +Step 11520/44000 | Loss: 3.0511 | LR: 4.27e-04 | 1390.0ms/step | 14.4 samples/s +Step 11530/44000 | Loss: 3.0809 | LR: 4.27e-04 | 1393.6ms/step | 14.4 samples/s +Step 11540/44000 | Loss: 3.0819 | LR: 4.27e-04 | 1387.4ms/step | 14.4 samples/s +Step 11550/44000 | Loss: 3.0646 | LR: 4.27e-04 | 1388.7ms/step | 14.4 samples/s +Step 11560/44000 | Loss: 3.0804 | LR: 4.27e-04 | 1389.5ms/step | 14.4 samples/s +Step 11570/44000 | Loss: 3.0807 | LR: 4.27e-04 | 1388.8ms/step | 14.4 samples/s +Step 11580/44000 | Loss: 3.0808 | LR: 4.27e-04 | 1391.3ms/step | 14.4 samples/s +Step 11590/44000 | Loss: 3.0891 | LR: 4.26e-04 | 1387.7ms/step | 14.4 samples/s +Step 11600/44000 | Loss: 3.0745 | LR: 4.26e-04 | 1390.2ms/step | 14.4 samples/s +Step 11610/44000 | Loss: 3.0822 | LR: 4.26e-04 | 1390.8ms/step | 14.4 samples/s +Step 11620/44000 | Loss: 3.0766 | LR: 4.26e-04 | 1388.3ms/step | 14.4 samples/s +Step 11630/44000 | Loss: 3.0554 | LR: 4.26e-04 | 1387.4ms/step | 14.4 samples/s +Step 11640/44000 | Loss: 3.0814 | LR: 4.26e-04 | 1390.5ms/step | 14.4 samples/s +Step 11650/44000 | Loss: 3.0817 | LR: 4.26e-04 | 1390.8ms/step | 14.4 samples/s +Step 11660/44000 | Loss: 3.0511 | LR: 4.26e-04 | 1391.6ms/step | 14.4 samples/s +Step 11670/44000 | Loss: 3.0747 | LR: 4.25e-04 | 1388.3ms/step | 14.4 samples/s +Step 11680/44000 | Loss: 3.0506 | LR: 4.25e-04 | 1389.1ms/step | 14.4 samples/s +Step 11690/44000 | Loss: 3.1070 | LR: 4.25e-04 | 1389.1ms/step | 14.4 samples/s +Step 11700/44000 | Loss: 3.0455 | LR: 4.25e-04 | 1388.7ms/step | 14.4 samples/s +Step 11710/44000 | Loss: 3.0977 | LR: 4.25e-04 | 1390.3ms/step | 14.4 samples/s +Step 11720/44000 | Loss: 3.0878 | LR: 4.25e-04 | 1388.7ms/step | 14.4 samples/s +Step 11730/44000 | Loss: 3.0859 | LR: 4.25e-04 | 1389.0ms/step | 14.4 samples/s +Step 11740/44000 | Loss: 3.1100 | LR: 4.24e-04 | 1389.8ms/step | 14.4 samples/s +Step 11750/44000 | Loss: 3.0723 | LR: 4.24e-04 | 1388.8ms/step | 14.4 samples/s +Validation at step 11750: Loss = 3.0898, Perplexity = 21.97 +Step 11760/44000 | Loss: 3.1080 | LR: 4.24e-04 | 3589.5ms/step | 5.6 samples/s +Step 11770/44000 | Loss: 3.1007 | LR: 4.24e-04 | 1390.6ms/step | 14.4 samples/s +Step 11780/44000 | Loss: 3.1070 | LR: 4.24e-04 | 1389.0ms/step | 14.4 samples/s +Step 11790/44000 | Loss: 3.0696 | LR: 4.24e-04 | 1388.5ms/step | 14.4 samples/s +Step 11800/44000 | Loss: 3.0641 | LR: 4.24e-04 | 1390.8ms/step | 14.4 samples/s +Step 11810/44000 | Loss: 3.0550 | LR: 4.24e-04 | 1387.2ms/step | 14.4 samples/s +Step 11820/44000 | Loss: 3.0828 | LR: 4.23e-04 | 1389.7ms/step | 14.4 samples/s +Step 11830/44000 | Loss: 3.1081 | LR: 4.23e-04 | 1389.8ms/step | 14.4 samples/s +Step 11840/44000 | Loss: 3.0485 | LR: 4.23e-04 | 1380.5ms/step | 14.5 samples/s +Step 11850/44000 | Loss: 3.0891 | LR: 4.23e-04 | 1384.2ms/step | 14.4 samples/s +Step 11860/44000 | Loss: 3.0634 | LR: 4.23e-04 | 1385.0ms/step | 14.4 samples/s +Step 11870/44000 | Loss: 3.0893 | LR: 4.23e-04 | 1386.8ms/step | 14.4 samples/s +Step 11880/44000 | Loss: 3.0731 | LR: 4.23e-04 | 1386.2ms/step | 14.4 samples/s +Step 11890/44000 | Loss: 3.0474 | LR: 4.23e-04 | 1387.7ms/step | 14.4 samples/s +Step 11900/44000 | Loss: 3.1046 | LR: 4.22e-04 | 1390.0ms/step | 14.4 samples/s +Step 11910/44000 | Loss: 3.0755 | LR: 4.22e-04 | 1387.8ms/step | 14.4 samples/s +Step 11920/44000 | Loss: 3.0746 | LR: 4.22e-04 | 1388.2ms/step | 14.4 samples/s +Step 11930/44000 | Loss: 3.0780 | LR: 4.22e-04 | 1388.6ms/step | 14.4 samples/s +Step 11940/44000 | Loss: 3.0649 | LR: 4.22e-04 | 1389.0ms/step | 14.4 samples/s +Step 11950/44000 | Loss: 3.0416 | LR: 4.22e-04 | 1386.7ms/step | 14.4 samples/s +Step 11960/44000 | Loss: 3.0616 | LR: 4.22e-04 | 1387.7ms/step | 14.4 samples/s +Step 11970/44000 | Loss: 3.0910 | LR: 4.21e-04 | 1388.0ms/step | 14.4 samples/s +Step 11980/44000 | Loss: 3.0655 | LR: 4.21e-04 | 1389.1ms/step | 14.4 samples/s +Step 11990/44000 | Loss: 3.0611 | LR: 4.21e-04 | 1388.2ms/step | 14.4 samples/s +Step 12000/44000 | Loss: 3.0788 | LR: 4.21e-04 | 1387.7ms/step | 14.4 samples/s +Validation at step 12000: Loss = 3.0846, Perplexity = 21.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-12000 +Step 12010/44000 | Loss: 3.0641 | LR: 4.21e-04 | 3877.5ms/step | 5.2 samples/s +Step 12020/44000 | Loss: 3.0457 | LR: 4.21e-04 | 1382.7ms/step | 14.5 samples/s +Step 12030/44000 | Loss: 3.0832 | LR: 4.21e-04 | 1382.4ms/step | 14.5 samples/s +Step 12040/44000 | Loss: 3.0844 | LR: 4.21e-04 | 1385.9ms/step | 14.4 samples/s +Step 12050/44000 | Loss: 3.0679 | LR: 4.20e-04 | 1384.9ms/step | 14.4 samples/s +Step 12060/44000 | Loss: 3.0617 | LR: 4.20e-04 | 1386.6ms/step | 14.4 samples/s +Step 12070/44000 | Loss: 3.1019 | LR: 4.20e-04 | 1388.1ms/step | 14.4 samples/s +Step 12080/44000 | Loss: 3.0757 | LR: 4.20e-04 | 1389.2ms/step | 14.4 samples/s +Step 12090/44000 | Loss: 3.0906 | LR: 4.20e-04 | 1392.1ms/step | 14.4 samples/s +Step 12100/44000 | Loss: 3.0823 | LR: 4.20e-04 | 1391.2ms/step | 14.4 samples/s +Step 12110/44000 | Loss: 3.0786 | LR: 4.20e-04 | 1390.2ms/step | 14.4 samples/s +Step 12120/44000 | Loss: 3.0797 | LR: 4.19e-04 | 1386.8ms/step | 14.4 samples/s +Step 12130/44000 | Loss: 3.0955 | LR: 4.19e-04 | 1389.6ms/step | 14.4 samples/s +Step 12140/44000 | Loss: 3.0430 | LR: 4.19e-04 | 1391.0ms/step | 14.4 samples/s +Step 12150/44000 | Loss: 3.0593 | LR: 4.19e-04 | 1390.6ms/step | 14.4 samples/s +Step 12160/44000 | Loss: 3.0906 | LR: 4.19e-04 | 1391.1ms/step | 14.4 samples/s +Step 12170/44000 | Loss: 3.0779 | LR: 4.19e-04 | 1391.2ms/step | 14.4 samples/s +Step 12180/44000 | Loss: 3.0567 | LR: 4.19e-04 | 1395.8ms/step | 14.3 samples/s +Step 12190/44000 | Loss: 3.0696 | LR: 4.19e-04 | 1388.3ms/step | 14.4 samples/s +Step 12200/44000 | Loss: 3.0357 | LR: 4.18e-04 | 1390.2ms/step | 14.4 samples/s +Step 12210/44000 | Loss: 3.0665 | LR: 4.18e-04 | 1391.6ms/step | 14.4 samples/s +Step 12220/44000 | Loss: 3.0420 | LR: 4.18e-04 | 1388.4ms/step | 14.4 samples/s +Step 12230/44000 | Loss: 3.0395 | LR: 4.18e-04 | 1390.3ms/step | 14.4 samples/s +Step 12240/44000 | Loss: 3.0933 | LR: 4.18e-04 | 1391.5ms/step | 14.4 samples/s +Step 12250/44000 | Loss: 3.0778 | LR: 4.18e-04 | 1392.0ms/step | 14.4 samples/s +Validation at step 12250: Loss = 3.0806, Perplexity = 21.77 +Step 12260/44000 | Loss: 3.0556 | LR: 4.18e-04 | 3596.4ms/step | 5.6 samples/s +Step 12270/44000 | Loss: 3.0811 | LR: 4.17e-04 | 1389.9ms/step | 14.4 samples/s +Step 12280/44000 | Loss: 3.0588 | LR: 4.17e-04 | 1395.4ms/step | 14.3 samples/s +Step 12290/44000 | Loss: 3.0759 | LR: 4.17e-04 | 1392.5ms/step | 14.4 samples/s +Step 12300/44000 | Loss: 3.0773 | LR: 4.17e-04 | 1390.7ms/step | 14.4 samples/s +Step 12310/44000 | Loss: 3.0817 | LR: 4.17e-04 | 1391.3ms/step | 14.4 samples/s +Step 12320/44000 | Loss: 3.0609 | LR: 4.17e-04 | 1388.7ms/step | 14.4 samples/s +Step 12330/44000 | Loss: 3.0912 | LR: 4.17e-04 | 1392.9ms/step | 14.4 samples/s +Step 12340/44000 | Loss: 3.0712 | LR: 4.17e-04 | 1390.2ms/step | 14.4 samples/s +Step 12350/44000 | Loss: 3.0868 | LR: 4.16e-04 | 1391.2ms/step | 14.4 samples/s +Step 12360/44000 | Loss: 3.0692 | LR: 4.16e-04 | 1390.2ms/step | 14.4 samples/s +Step 12370/44000 | Loss: 3.0661 | LR: 4.16e-04 | 1388.6ms/step | 14.4 samples/s +Step 12380/44000 | Loss: 3.0374 | LR: 4.16e-04 | 1389.9ms/step | 14.4 samples/s +Step 12390/44000 | Loss: 3.0714 | LR: 4.16e-04 | 1388.3ms/step | 14.4 samples/s +Step 12400/44000 | Loss: 3.0362 | LR: 4.16e-04 | 1390.6ms/step | 14.4 samples/s +Step 12410/44000 | Loss: 3.0601 | LR: 4.16e-04 | 1388.9ms/step | 14.4 samples/s +Step 12420/44000 | Loss: 3.0721 | LR: 4.15e-04 | 1388.4ms/step | 14.4 samples/s +Step 12430/44000 | Loss: 3.0546 | LR: 4.15e-04 | 1389.1ms/step | 14.4 samples/s +Step 12440/44000 | Loss: 3.0669 | LR: 4.15e-04 | 1390.9ms/step | 14.4 samples/s +Step 12450/44000 | Loss: 3.0636 | LR: 4.15e-04 | 1391.7ms/step | 14.4 samples/s +Step 12460/44000 | Loss: 3.0717 | LR: 4.15e-04 | 1392.4ms/step | 14.4 samples/s +Step 12470/44000 | Loss: 3.0623 | LR: 4.15e-04 | 1388.2ms/step | 14.4 samples/s +Step 12480/44000 | Loss: 3.0544 | LR: 4.15e-04 | 1382.2ms/step | 14.5 samples/s +Step 12490/44000 | Loss: 3.0366 | LR: 4.14e-04 | 1383.9ms/step | 14.5 samples/s +Step 12500/44000 | Loss: 3.0303 | LR: 4.14e-04 | 1384.6ms/step | 14.4 samples/s +Validation at step 12500: Loss = 3.0763, Perplexity = 21.68 +Step 12510/44000 | Loss: 3.0866 | LR: 4.14e-04 | 3582.8ms/step | 5.6 samples/s +Step 12520/44000 | Loss: 3.0691 | LR: 4.14e-04 | 1388.1ms/step | 14.4 samples/s +Step 12530/44000 | Loss: 3.0625 | LR: 4.14e-04 | 1391.7ms/step | 14.4 samples/s +Step 12540/44000 | Loss: 3.0636 | LR: 4.14e-04 | 1389.3ms/step | 14.4 samples/s +Step 12550/44000 | Loss: 3.0790 | LR: 4.14e-04 | 1390.9ms/step | 14.4 samples/s +Step 12560/44000 | Loss: 3.0434 | LR: 4.14e-04 | 1388.4ms/step | 14.4 samples/s +Step 12570/44000 | Loss: 3.0551 | LR: 4.13e-04 | 1389.5ms/step | 14.4 samples/s +Step 12580/44000 | Loss: 3.0988 | LR: 4.13e-04 | 1390.3ms/step | 14.4 samples/s +Step 12590/44000 | Loss: 3.0575 | LR: 4.13e-04 | 1391.7ms/step | 14.4 samples/s +Step 12600/44000 | Loss: 3.0618 | LR: 4.13e-04 | 1388.9ms/step | 14.4 samples/s +Step 12610/44000 | Loss: 3.0763 | LR: 4.13e-04 | 1385.4ms/step | 14.4 samples/s +Step 12620/44000 | Loss: 3.1088 | LR: 4.13e-04 | 1383.8ms/step | 14.5 samples/s +Step 12630/44000 | Loss: 3.0701 | LR: 4.13e-04 | 1385.7ms/step | 14.4 samples/s +Step 12640/44000 | Loss: 3.0608 | LR: 4.12e-04 | 1386.3ms/step | 14.4 samples/s +Step 12650/44000 | Loss: 3.0704 | LR: 4.12e-04 | 1388.5ms/step | 14.4 samples/s +Step 12660/44000 | Loss: 3.0591 | LR: 4.12e-04 | 1393.0ms/step | 14.4 samples/s +Step 12670/44000 | Loss: 3.0746 | LR: 4.12e-04 | 1391.6ms/step | 14.4 samples/s +Step 12680/44000 | Loss: 3.0745 | LR: 4.12e-04 | 1390.8ms/step | 14.4 samples/s +Step 12690/44000 | Loss: 3.0652 | LR: 4.12e-04 | 1392.1ms/step | 14.4 samples/s +Step 12700/44000 | Loss: 3.0599 | LR: 4.12e-04 | 1391.1ms/step | 14.4 samples/s +Step 12710/44000 | Loss: 3.0266 | LR: 4.11e-04 | 1392.0ms/step | 14.4 samples/s +Step 12720/44000 | Loss: 3.0238 | LR: 4.11e-04 | 1389.1ms/step | 14.4 samples/s +Step 12730/44000 | Loss: 3.0510 | LR: 4.11e-04 | 1390.9ms/step | 14.4 samples/s +Step 12740/44000 | Loss: 3.0683 | LR: 4.11e-04 | 1390.9ms/step | 14.4 samples/s +Step 12750/44000 | Loss: 3.0311 | LR: 4.11e-04 | 1392.5ms/step | 14.4 samples/s +Validation at step 12750: Loss = 3.0711, Perplexity = 21.56 +Step 12760/44000 | Loss: 3.0568 | LR: 4.11e-04 | 3591.1ms/step | 5.6 samples/s +Step 12770/44000 | Loss: 3.0673 | LR: 4.11e-04 | 1388.7ms/step | 14.4 samples/s +Step 12780/44000 | Loss: 3.0468 | LR: 4.10e-04 | 1389.8ms/step | 14.4 samples/s +Step 12790/44000 | Loss: 3.0519 | LR: 4.10e-04 | 1389.8ms/step | 14.4 samples/s +Step 12800/44000 | Loss: 3.0719 | LR: 4.10e-04 | 1388.4ms/step | 14.4 samples/s +Step 12810/44000 | Loss: 3.0718 | LR: 4.10e-04 | 1390.0ms/step | 14.4 samples/s +Step 12820/44000 | Loss: 3.0481 | LR: 4.10e-04 | 1390.6ms/step | 14.4 samples/s +Step 12830/44000 | Loss: 3.0604 | LR: 4.10e-04 | 1387.7ms/step | 14.4 samples/s +Step 12840/44000 | Loss: 3.0392 | LR: 4.10e-04 | 1388.2ms/step | 14.4 samples/s +Step 12850/44000 | Loss: 3.0582 | LR: 4.09e-04 | 1388.6ms/step | 14.4 samples/s +Step 12860/44000 | Loss: 3.0356 | LR: 4.09e-04 | 1388.6ms/step | 14.4 samples/s +Step 12870/44000 | Loss: 3.0639 | LR: 4.09e-04 | 1387.8ms/step | 14.4 samples/s +Step 12880/44000 | Loss: 3.0258 | LR: 4.09e-04 | 1389.4ms/step | 14.4 samples/s +Step 12890/44000 | Loss: 3.0789 | LR: 4.09e-04 | 1391.1ms/step | 14.4 samples/s +Step 12900/44000 | Loss: 3.0410 | LR: 4.09e-04 | 1390.3ms/step | 14.4 samples/s +Step 12910/44000 | Loss: 3.0571 | LR: 4.09e-04 | 1389.2ms/step | 14.4 samples/s +Step 12920/44000 | Loss: 3.0587 | LR: 4.09e-04 | 1388.7ms/step | 14.4 samples/s +Step 12930/44000 | Loss: 3.0588 | LR: 4.08e-04 | 1391.5ms/step | 14.4 samples/s +Step 12940/44000 | Loss: 3.0750 | LR: 4.08e-04 | 1388.3ms/step | 14.4 samples/s +Step 12950/44000 | Loss: 3.0579 | LR: 4.08e-04 | 1391.5ms/step | 14.4 samples/s +Step 12960/44000 | Loss: 3.0649 | LR: 4.08e-04 | 1387.1ms/step | 14.4 samples/s +Step 12970/44000 | Loss: 3.0728 | LR: 4.08e-04 | 1390.4ms/step | 14.4 samples/s +Step 12980/44000 | Loss: 3.0683 | LR: 4.08e-04 | 1389.0ms/step | 14.4 samples/s +Step 12990/44000 | Loss: 3.0614 | LR: 4.08e-04 | 1388.9ms/step | 14.4 samples/s +Step 13000/44000 | Loss: 3.0476 | LR: 4.07e-04 | 1389.0ms/step | 14.4 samples/s +Validation at step 13000: Loss = 3.0677, Perplexity = 21.49 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-13000 +Step 13010/44000 | Loss: 3.0549 | LR: 4.07e-04 | 3868.2ms/step | 5.2 samples/s +Step 13020/44000 | Loss: 3.0562 | LR: 4.07e-04 | 1388.0ms/step | 14.4 samples/s +Step 13030/44000 | Loss: 3.0616 | LR: 4.07e-04 | 1390.4ms/step | 14.4 samples/s +Step 13040/44000 | Loss: 3.0353 | LR: 4.07e-04 | 1389.1ms/step | 14.4 samples/s +Step 13050/44000 | Loss: 3.0206 | LR: 4.07e-04 | 1388.7ms/step | 14.4 samples/s +Step 13060/44000 | Loss: 3.0666 | LR: 4.07e-04 | 1388.6ms/step | 14.4 samples/s +Step 13070/44000 | Loss: 3.0611 | LR: 4.06e-04 | 1387.1ms/step | 14.4 samples/s +Step 13080/44000 | Loss: 3.0770 | LR: 4.06e-04 | 1391.7ms/step | 14.4 samples/s +Step 13090/44000 | Loss: 3.0438 | LR: 4.06e-04 | 1392.3ms/step | 14.4 samples/s +Step 13100/44000 | Loss: 3.0493 | LR: 4.06e-04 | 1390.3ms/step | 14.4 samples/s +Step 13110/44000 | Loss: 3.0605 | LR: 4.06e-04 | 1390.5ms/step | 14.4 samples/s +Step 13120/44000 | Loss: 3.0703 | LR: 4.06e-04 | 1389.0ms/step | 14.4 samples/s +Step 13130/44000 | Loss: 3.0696 | LR: 4.06e-04 | 1389.9ms/step | 14.4 samples/s +Step 13140/44000 | Loss: 3.0708 | LR: 4.05e-04 | 1387.7ms/step | 14.4 samples/s +Step 13150/44000 | Loss: 3.0521 | LR: 4.05e-04 | 1389.1ms/step | 14.4 samples/s +Step 13160/44000 | Loss: 3.0457 | LR: 4.05e-04 | 1391.4ms/step | 14.4 samples/s +Step 13170/44000 | Loss: 3.0498 | LR: 4.05e-04 | 1388.3ms/step | 14.4 samples/s +Step 13180/44000 | Loss: 3.0724 | LR: 4.05e-04 | 1389.7ms/step | 14.4 samples/s +Step 13190/44000 | Loss: 3.0482 | LR: 4.05e-04 | 1390.1ms/step | 14.4 samples/s +Step 13200/44000 | Loss: 3.0802 | LR: 4.05e-04 | 1388.2ms/step | 14.4 samples/s +Step 13210/44000 | Loss: 3.0339 | LR: 4.04e-04 | 1388.7ms/step | 14.4 samples/s +Step 13220/44000 | Loss: 3.0378 | LR: 4.04e-04 | 1388.9ms/step | 14.4 samples/s +Step 13230/44000 | Loss: 3.0684 | LR: 4.04e-04 | 1388.6ms/step | 14.4 samples/s +Step 13240/44000 | Loss: 3.0480 | LR: 4.04e-04 | 1388.7ms/step | 14.4 samples/s +Step 13250/44000 | Loss: 3.0656 | LR: 4.04e-04 | 1389.3ms/step | 14.4 samples/s +Validation at step 13250: Loss = 3.0640, Perplexity = 21.41 +Step 13260/44000 | Loss: 3.0547 | LR: 4.04e-04 | 3590.6ms/step | 5.6 samples/s +Step 13270/44000 | Loss: 3.0453 | LR: 4.04e-04 | 1387.4ms/step | 14.4 samples/s +Step 13280/44000 | Loss: 3.0731 | LR: 4.03e-04 | 1389.3ms/step | 14.4 samples/s +Step 13290/44000 | Loss: 3.0582 | LR: 4.03e-04 | 1388.2ms/step | 14.4 samples/s +Step 13300/44000 | Loss: 3.0189 | LR: 4.03e-04 | 1391.9ms/step | 14.4 samples/s +Step 13310/44000 | Loss: 3.0505 | LR: 4.03e-04 | 1387.4ms/step | 14.4 samples/s +Step 13320/44000 | Loss: 3.0335 | LR: 4.03e-04 | 1389.0ms/step | 14.4 samples/s +Step 13330/44000 | Loss: 3.0402 | LR: 4.03e-04 | 1389.6ms/step | 14.4 samples/s +Step 13340/44000 | Loss: 3.0498 | LR: 4.03e-04 | 1388.3ms/step | 14.4 samples/s +Step 13350/44000 | Loss: 3.0362 | LR: 4.02e-04 | 1390.1ms/step | 14.4 samples/s +Step 13360/44000 | Loss: 3.0499 | LR: 4.02e-04 | 1389.0ms/step | 14.4 samples/s +Step 13370/44000 | Loss: 3.0805 | LR: 4.02e-04 | 1389.7ms/step | 14.4 samples/s +Step 13380/44000 | Loss: 3.0724 | LR: 4.02e-04 | 1387.8ms/step | 14.4 samples/s +Step 13390/44000 | Loss: 3.0410 | LR: 4.02e-04 | 1389.3ms/step | 14.4 samples/s +Step 13400/44000 | Loss: 3.0418 | LR: 4.02e-04 | 1390.1ms/step | 14.4 samples/s +Step 13410/44000 | Loss: 3.0359 | LR: 4.02e-04 | 1390.9ms/step | 14.4 samples/s +Step 13420/44000 | Loss: 3.0046 | LR: 4.01e-04 | 1390.3ms/step | 14.4 samples/s +Step 13430/44000 | Loss: 3.0613 | LR: 4.01e-04 | 1390.9ms/step | 14.4 samples/s +Step 13440/44000 | Loss: 3.0513 | LR: 4.01e-04 | 1389.9ms/step | 14.4 samples/s +Step 13450/44000 | Loss: 3.0614 | LR: 4.01e-04 | 1389.8ms/step | 14.4 samples/s +Step 13460/44000 | Loss: 3.0391 | LR: 4.01e-04 | 1387.3ms/step | 14.4 samples/s +Step 13470/44000 | Loss: 3.0523 | LR: 4.01e-04 | 1388.9ms/step | 14.4 samples/s +Step 13480/44000 | Loss: 3.0273 | LR: 4.01e-04 | 1387.8ms/step | 14.4 samples/s +Step 13490/44000 | Loss: 3.0527 | LR: 4.00e-04 | 1388.2ms/step | 14.4 samples/s +Step 13500/44000 | Loss: 3.0545 | LR: 4.00e-04 | 1390.6ms/step | 14.4 samples/s +Validation at step 13500: Loss = 3.0594, Perplexity = 21.31 +Step 13510/44000 | Loss: 3.0411 | LR: 4.00e-04 | 3592.5ms/step | 5.6 samples/s +Step 13520/44000 | Loss: 3.0387 | LR: 4.00e-04 | 1390.6ms/step | 14.4 samples/s +Step 13530/44000 | Loss: 3.0358 | LR: 4.00e-04 | 1388.3ms/step | 14.4 samples/s +Step 13540/44000 | Loss: 3.0254 | LR: 4.00e-04 | 1388.3ms/step | 14.4 samples/s +Step 13550/44000 | Loss: 3.0448 | LR: 4.00e-04 | 1387.7ms/step | 14.4 samples/s +Step 13560/44000 | Loss: 3.0364 | LR: 3.99e-04 | 1388.0ms/step | 14.4 samples/s +Step 13570/44000 | Loss: 3.0726 | LR: 3.99e-04 | 1392.2ms/step | 14.4 samples/s +Step 13580/44000 | Loss: 3.0514 | LR: 3.99e-04 | 1388.1ms/step | 14.4 samples/s +Step 13590/44000 | Loss: 3.0279 | LR: 3.99e-04 | 1392.0ms/step | 14.4 samples/s +Step 13600/44000 | Loss: 3.0415 | LR: 3.99e-04 | 1392.5ms/step | 14.4 samples/s +Step 13610/44000 | Loss: 3.0315 | LR: 3.99e-04 | 1391.5ms/step | 14.4 samples/s +Step 13620/44000 | Loss: 3.0636 | LR: 3.98e-04 | 1389.8ms/step | 14.4 samples/s +Step 13630/44000 | Loss: 3.0543 | LR: 3.98e-04 | 1391.6ms/step | 14.4 samples/s +Step 13640/44000 | Loss: 3.0296 | LR: 3.98e-04 | 1390.8ms/step | 14.4 samples/s +Step 13650/44000 | Loss: 3.0474 | LR: 3.98e-04 | 1390.6ms/step | 14.4 samples/s +Step 13660/44000 | Loss: 3.0345 | LR: 3.98e-04 | 1387.1ms/step | 14.4 samples/s +Step 13670/44000 | Loss: 3.0215 | LR: 3.98e-04 | 1384.2ms/step | 14.4 samples/s +Step 13680/44000 | Loss: 3.0358 | LR: 3.98e-04 | 1386.0ms/step | 14.4 samples/s +Step 13690/44000 | Loss: 3.0529 | LR: 3.97e-04 | 1388.6ms/step | 14.4 samples/s +Step 13700/44000 | Loss: 3.0387 | LR: 3.97e-04 | 1386.8ms/step | 14.4 samples/s +Step 13710/44000 | Loss: 3.0482 | LR: 3.97e-04 | 1386.3ms/step | 14.4 samples/s +Step 13720/44000 | Loss: 3.0273 | LR: 3.97e-04 | 1385.9ms/step | 14.4 samples/s +Step 13730/44000 | Loss: 3.0470 | LR: 3.97e-04 | 1387.2ms/step | 14.4 samples/s +Step 13740/44000 | Loss: 3.0547 | LR: 3.97e-04 | 1386.3ms/step | 14.4 samples/s +Step 13750/44000 | Loss: 3.0856 | LR: 3.97e-04 | 1386.8ms/step | 14.4 samples/s +Validation at step 13750: Loss = 3.0559, Perplexity = 21.24 +Step 13760/44000 | Loss: 3.0743 | LR: 3.96e-04 | 3590.6ms/step | 5.6 samples/s +Step 13770/44000 | Loss: 3.0432 | LR: 3.96e-04 | 1389.1ms/step | 14.4 samples/s diff --git a/epfl_final/logs/l8k365m_dense_resume.log b/epfl_final/logs/l8k365m_dense_resume.log new file mode 100644 index 0000000000000000000000000000000000000000..9ececd00c1e215a2fc2a67960610172827b6dc9c --- /dev/null +++ b/epfl_final/logs/l8k365m_dense_resume.log @@ -0,0 +1,3227 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-13000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 8k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "dense" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-13000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-13000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 13000 (epoch 0, micro_step 130000) +Resuming training from step 13000 +Resuming wandb run qlqhc3gy +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run qlqhc3gy +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_170941-qlqhc3gy +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m dense 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qlqhc3gy +Starting training... +Validation at step 13000: Loss = 3.0677, Perplexity = 21.49 +wandb: WARNING Tried to log to step 13000 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 13001: dense (dense step interval) +Step 13010/44000 | Loss: 3.0262 | LR: 4.07e-04 | 34893.2ms/step | 0.6 samples/s +wandb: WARNING Tried to log to step 13010 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13020/44000 | Loss: 3.0646 | LR: 4.07e-04 | 1382.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13020 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13030/44000 | Loss: 3.0603 | LR: 4.07e-04 | 1384.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13030 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13040/44000 | Loss: 3.0704 | LR: 4.07e-04 | 1389.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13040 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13050/44000 | Loss: 3.0357 | LR: 4.07e-04 | 1386.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13050 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13060/44000 | Loss: 3.0516 | LR: 4.07e-04 | 1385.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13060 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13070/44000 | Loss: 3.0410 | LR: 4.06e-04 | 1387.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13070 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13080/44000 | Loss: 3.0268 | LR: 4.06e-04 | 1385.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13080 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13090/44000 | Loss: 3.0331 | LR: 4.06e-04 | 1388.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13090 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13100/44000 | Loss: 3.0612 | LR: 4.06e-04 | 1389.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13100 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13110/44000 | Loss: 3.0246 | LR: 4.06e-04 | 1387.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13110 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13120/44000 | Loss: 3.0297 | LR: 4.06e-04 | 1386.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13120 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13130/44000 | Loss: 3.0353 | LR: 4.06e-04 | 1391.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13130 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13140/44000 | Loss: 3.0591 | LR: 4.05e-04 | 1390.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13140 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13150/44000 | Loss: 3.0290 | LR: 4.05e-04 | 1391.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13150 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13160/44000 | Loss: 3.0404 | LR: 4.05e-04 | 1386.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13160 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13170/44000 | Loss: 3.0635 | LR: 4.05e-04 | 1385.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13170 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13180/44000 | Loss: 3.0211 | LR: 4.05e-04 | 1386.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13180 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13190/44000 | Loss: 3.0586 | LR: 4.05e-04 | 1384.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13190 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13200/44000 | Loss: 3.0530 | LR: 4.05e-04 | 1384.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13200 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13210/44000 | Loss: 3.0507 | LR: 4.04e-04 | 1389.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13210 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13220/44000 | Loss: 3.0544 | LR: 4.04e-04 | 1393.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13220 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13230/44000 | Loss: 3.0792 | LR: 4.04e-04 | 1390.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13230 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13240/44000 | Loss: 3.0246 | LR: 4.04e-04 | 1387.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13240 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13250/44000 | Loss: 3.0555 | LR: 4.04e-04 | 1385.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13250 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 13250: Loss = 3.0643, Perplexity = 21.42 +Step 13260/44000 | Loss: 3.0370 | LR: 4.04e-04 | 3605.3ms/step | 5.5 samples/s +wandb: WARNING Tried to log to step 13260 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13270/44000 | Loss: 3.0711 | LR: 4.04e-04 | 1397.5ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13270 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13280/44000 | Loss: 3.0568 | LR: 4.03e-04 | 1395.5ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13280 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13290/44000 | Loss: 3.0195 | LR: 4.03e-04 | 1393.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13290 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13300/44000 | Loss: 3.0184 | LR: 4.03e-04 | 1393.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13300 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13310/44000 | Loss: 3.0141 | LR: 4.03e-04 | 1395.1ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13310 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13320/44000 | Loss: 3.0623 | LR: 4.03e-04 | 1392.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13320 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13330/44000 | Loss: 3.0416 | LR: 4.03e-04 | 1394.6ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13330 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13340/44000 | Loss: 3.0936 | LR: 4.03e-04 | 1394.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13340 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13350/44000 | Loss: 3.0506 | LR: 4.02e-04 | 1393.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13350 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13360/44000 | Loss: 3.0145 | LR: 4.02e-04 | 1394.3ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13360 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13370/44000 | Loss: 3.0322 | LR: 4.02e-04 | 1400.1ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13370 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13380/44000 | Loss: 3.0376 | LR: 4.02e-04 | 1395.3ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13380 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13390/44000 | Loss: 3.0394 | LR: 4.02e-04 | 1398.7ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13390 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13400/44000 | Loss: 3.0538 | LR: 4.02e-04 | 1395.5ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13400 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13410/44000 | Loss: 3.0400 | LR: 4.02e-04 | 1393.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13410 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13420/44000 | Loss: 3.0743 | LR: 4.01e-04 | 1397.1ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13420 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13430/44000 | Loss: 3.0327 | LR: 4.01e-04 | 1395.9ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13430 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13440/44000 | Loss: 3.0648 | LR: 4.01e-04 | 1397.5ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13440 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13450/44000 | Loss: 3.0334 | LR: 4.01e-04 | 1394.7ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13450 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13460/44000 | Loss: 3.0546 | LR: 4.01e-04 | 1392.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13460 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13470/44000 | Loss: 3.0503 | LR: 4.01e-04 | 1389.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13470 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13480/44000 | Loss: 3.0205 | LR: 4.01e-04 | 1391.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13480 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13490/44000 | Loss: 3.0568 | LR: 4.00e-04 | 1392.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13490 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13500/44000 | Loss: 3.0424 | LR: 4.00e-04 | 1393.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13500 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 13500: Loss = 3.0598, Perplexity = 21.32 +Step 13510/44000 | Loss: 3.0214 | LR: 4.00e-04 | 3594.2ms/step | 5.6 samples/s +wandb: WARNING Tried to log to step 13510 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13520/44000 | Loss: 3.0635 | LR: 4.00e-04 | 1396.1ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13520 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13530/44000 | Loss: 3.0731 | LR: 4.00e-04 | 1398.1ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13530 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13540/44000 | Loss: 3.0631 | LR: 4.00e-04 | 1400.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13540 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13550/44000 | Loss: 3.0682 | LR: 4.00e-04 | 1397.8ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13550 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13560/44000 | Loss: 3.0575 | LR: 3.99e-04 | 1392.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13560 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13570/44000 | Loss: 3.0424 | LR: 3.99e-04 | 1393.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13570 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13580/44000 | Loss: 3.0521 | LR: 3.99e-04 | 1394.5ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13580 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13590/44000 | Loss: 3.0540 | LR: 3.99e-04 | 1398.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13590 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13600/44000 | Loss: 3.0424 | LR: 3.99e-04 | 1396.1ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13600 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13610/44000 | Loss: 3.0417 | LR: 3.99e-04 | 1390.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13610 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13620/44000 | Loss: 3.0163 | LR: 3.98e-04 | 1393.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13620 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13630/44000 | Loss: 3.0397 | LR: 3.98e-04 | 1391.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13630 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13640/44000 | Loss: 3.0567 | LR: 3.98e-04 | 1394.5ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13640 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13650/44000 | Loss: 3.0479 | LR: 3.98e-04 | 1394.4ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13650 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13660/44000 | Loss: 3.0609 | LR: 3.98e-04 | 1397.7ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13660 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13670/44000 | Loss: 3.0442 | LR: 3.98e-04 | 1400.3ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13670 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13680/44000 | Loss: 3.0541 | LR: 3.98e-04 | 1398.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13680 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13690/44000 | Loss: 3.0660 | LR: 3.97e-04 | 1395.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13690 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13700/44000 | Loss: 3.0349 | LR: 3.97e-04 | 1391.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13700 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13710/44000 | Loss: 3.0454 | LR: 3.97e-04 | 1391.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13710 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13720/44000 | Loss: 3.0447 | LR: 3.97e-04 | 1393.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 13720 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13730/44000 | Loss: 3.0367 | LR: 3.97e-04 | 1393.8ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13730 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13740/44000 | Loss: 3.0328 | LR: 3.97e-04 | 1396.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13740 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13750/44000 | Loss: 3.0575 | LR: 3.97e-04 | 1396.2ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13750 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 13750: Loss = 3.0549, Perplexity = 21.22 +Step 13760/44000 | Loss: 3.0399 | LR: 3.96e-04 | 3600.7ms/step | 5.6 samples/s +wandb: WARNING Tried to log to step 13760 that is less than the current step 13761. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13770/44000 | Loss: 3.0526 | LR: 3.96e-04 | 1399.7ms/step | 14.3 samples/s +Step 13780/44000 | Loss: 3.0685 | LR: 3.96e-04 | 1398.6ms/step | 14.3 samples/s +Step 13790/44000 | Loss: 3.0268 | LR: 3.96e-04 | 1395.7ms/step | 14.3 samples/s +Step 13800/44000 | Loss: 3.0436 | LR: 3.96e-04 | 1397.9ms/step | 14.3 samples/s +Step 13810/44000 | Loss: 3.0456 | LR: 3.96e-04 | 1400.0ms/step | 14.3 samples/s +Step 13820/44000 | Loss: 3.0571 | LR: 3.96e-04 | 1398.1ms/step | 14.3 samples/s +Step 13830/44000 | Loss: 3.0519 | LR: 3.95e-04 | 1390.9ms/step | 14.4 samples/s +Step 13840/44000 | Loss: 3.0309 | LR: 3.95e-04 | 1389.2ms/step | 14.4 samples/s +Step 13850/44000 | Loss: 3.0358 | LR: 3.95e-04 | 1389.7ms/step | 14.4 samples/s +Step 13860/44000 | Loss: 3.0720 | LR: 3.95e-04 | 1395.8ms/step | 14.3 samples/s +Step 13870/44000 | Loss: 3.0407 | LR: 3.95e-04 | 1394.1ms/step | 14.3 samples/s +Step 13880/44000 | Loss: 3.0191 | LR: 3.95e-04 | 1398.4ms/step | 14.3 samples/s +Step 13890/44000 | Loss: 3.0000 | LR: 3.95e-04 | 1392.9ms/step | 14.4 samples/s +Step 13900/44000 | Loss: 3.0516 | LR: 3.94e-04 | 1393.1ms/step | 14.4 samples/s +Step 13910/44000 | Loss: 3.0053 | LR: 3.94e-04 | 1387.9ms/step | 14.4 samples/s +Step 13920/44000 | Loss: 3.0231 | LR: 3.94e-04 | 1388.6ms/step | 14.4 samples/s +Step 13930/44000 | Loss: 3.0043 | LR: 3.94e-04 | 1392.5ms/step | 14.4 samples/s +Step 13940/44000 | Loss: 3.0517 | LR: 3.94e-04 | 1394.5ms/step | 14.3 samples/s +Step 13950/44000 | Loss: 3.0361 | LR: 3.94e-04 | 1392.1ms/step | 14.4 samples/s +Step 13960/44000 | Loss: 3.0502 | LR: 3.93e-04 | 1392.8ms/step | 14.4 samples/s +Step 13970/44000 | Loss: 3.0627 | LR: 3.93e-04 | 1386.4ms/step | 14.4 samples/s +Step 13980/44000 | Loss: 3.0565 | LR: 3.93e-04 | 1387.5ms/step | 14.4 samples/s +Step 13990/44000 | Loss: 3.0168 | LR: 3.93e-04 | 1389.5ms/step | 14.4 samples/s +Step 14000/44000 | Loss: 3.0439 | LR: 3.93e-04 | 1392.0ms/step | 14.4 samples/s +Validation at step 14000: Loss = 3.0517, Perplexity = 21.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-14000 +Step 14010/44000 | Loss: 3.0310 | LR: 3.93e-04 | 3974.0ms/step | 5.0 samples/s +Step 14020/44000 | Loss: 3.0448 | LR: 3.93e-04 | 1394.4ms/step | 14.3 samples/s +Step 14030/44000 | Loss: 3.0159 | LR: 3.92e-04 | 1397.4ms/step | 14.3 samples/s +Step 14040/44000 | Loss: 3.0654 | LR: 3.92e-04 | 1396.4ms/step | 14.3 samples/s +Step 14050/44000 | Loss: 3.0311 | LR: 3.92e-04 | 1396.0ms/step | 14.3 samples/s +Step 14060/44000 | Loss: 3.0762 | LR: 3.92e-04 | 1391.7ms/step | 14.4 samples/s +Step 14070/44000 | Loss: 3.0657 | LR: 3.92e-04 | 1392.7ms/step | 14.4 samples/s +Step 14080/44000 | Loss: 3.0440 | LR: 3.92e-04 | 1389.5ms/step | 14.4 samples/s +Step 14090/44000 | Loss: 3.0152 | LR: 3.92e-04 | 1393.9ms/step | 14.3 samples/s +Step 14100/44000 | Loss: 3.0290 | LR: 3.91e-04 | 1399.7ms/step | 14.3 samples/s +Step 14110/44000 | Loss: 3.0005 | LR: 3.91e-04 | 1400.3ms/step | 14.3 samples/s +Step 14120/44000 | Loss: 3.0218 | LR: 3.91e-04 | 1399.8ms/step | 14.3 samples/s +Step 14130/44000 | Loss: 3.0551 | LR: 3.91e-04 | 1398.3ms/step | 14.3 samples/s +Step 14140/44000 | Loss: 3.0220 | LR: 3.91e-04 | 1392.0ms/step | 14.4 samples/s +Step 14150/44000 | Loss: 3.0269 | LR: 3.91e-04 | 1395.8ms/step | 14.3 samples/s +Step 14160/44000 | Loss: 3.0240 | LR: 3.90e-04 | 1390.2ms/step | 14.4 samples/s +Step 14170/44000 | Loss: 3.0669 | LR: 3.90e-04 | 1398.2ms/step | 14.3 samples/s +Step 14180/44000 | Loss: 3.0176 | LR: 3.90e-04 | 1400.0ms/step | 14.3 samples/s +Step 14190/44000 | Loss: 3.0534 | LR: 3.90e-04 | 1400.2ms/step | 14.3 samples/s +Step 14200/44000 | Loss: 3.0615 | LR: 3.90e-04 | 1399.8ms/step | 14.3 samples/s +Step 14210/44000 | Loss: 3.0325 | LR: 3.90e-04 | 1400.0ms/step | 14.3 samples/s +Step 14220/44000 | Loss: 3.0172 | LR: 3.90e-04 | 1400.0ms/step | 14.3 samples/s +Step 14230/44000 | Loss: 3.0373 | LR: 3.89e-04 | 1399.8ms/step | 14.3 samples/s +Step 14240/44000 | Loss: 3.0099 | LR: 3.89e-04 | 1400.3ms/step | 14.3 samples/s +Step 14250/44000 | Loss: 3.0317 | LR: 3.89e-04 | 1399.9ms/step | 14.3 samples/s +Validation at step 14250: Loss = 3.0486, Perplexity = 21.09 +Step 14260/44000 | Loss: 3.0088 | LR: 3.89e-04 | 3624.1ms/step | 5.5 samples/s +Step 14270/44000 | Loss: 3.0241 | LR: 3.89e-04 | 1400.0ms/step | 14.3 samples/s +Step 14280/44000 | Loss: 3.0278 | LR: 3.89e-04 | 1398.1ms/step | 14.3 samples/s +Step 14290/44000 | Loss: 3.0152 | LR: 3.89e-04 | 1398.2ms/step | 14.3 samples/s +Step 14300/44000 | Loss: 3.0124 | LR: 3.88e-04 | 1400.0ms/step | 14.3 samples/s +Step 14310/44000 | Loss: 3.0749 | LR: 3.88e-04 | 1399.0ms/step | 14.3 samples/s +Step 14320/44000 | Loss: 3.0374 | LR: 3.88e-04 | 1396.7ms/step | 14.3 samples/s +Step 14330/44000 | Loss: 3.0545 | LR: 3.88e-04 | 1393.8ms/step | 14.3 samples/s +Step 14340/44000 | Loss: 3.0248 | LR: 3.88e-04 | 1394.3ms/step | 14.3 samples/s +Step 14350/44000 | Loss: 3.0523 | LR: 3.88e-04 | 1394.1ms/step | 14.3 samples/s +Step 14360/44000 | Loss: 3.0301 | LR: 3.87e-04 | 1391.5ms/step | 14.4 samples/s +Step 14370/44000 | Loss: 3.0260 | LR: 3.87e-04 | 1392.6ms/step | 14.4 samples/s +Step 14380/44000 | Loss: 3.0031 | LR: 3.87e-04 | 1393.8ms/step | 14.3 samples/s +Step 14390/44000 | Loss: 3.0274 | LR: 3.87e-04 | 1393.7ms/step | 14.3 samples/s +Step 14400/44000 | Loss: 2.9985 | LR: 3.87e-04 | 1388.5ms/step | 14.4 samples/s +Step 14410/44000 | Loss: 3.0203 | LR: 3.87e-04 | 1389.9ms/step | 14.4 samples/s +Step 14420/44000 | Loss: 3.0113 | LR: 3.87e-04 | 1399.8ms/step | 14.3 samples/s +Step 14430/44000 | Loss: 3.0249 | LR: 3.86e-04 | 1398.2ms/step | 14.3 samples/s +Step 14440/44000 | Loss: 3.0131 | LR: 3.86e-04 | 1400.0ms/step | 14.3 samples/s +Step 14450/44000 | Loss: 3.0352 | LR: 3.86e-04 | 1400.3ms/step | 14.3 samples/s +Step 14460/44000 | Loss: 3.0067 | LR: 3.86e-04 | 1399.8ms/step | 14.3 samples/s +Step 14470/44000 | Loss: 3.0514 | LR: 3.86e-04 | 1393.1ms/step | 14.4 samples/s +Step 14480/44000 | Loss: 3.0297 | LR: 3.86e-04 | 1390.5ms/step | 14.4 samples/s +Step 14490/44000 | Loss: 3.0302 | LR: 3.85e-04 | 1392.2ms/step | 14.4 samples/s +Step 14500/44000 | Loss: 3.0404 | LR: 3.85e-04 | 1392.0ms/step | 14.4 samples/s +Validation at step 14500: Loss = 3.0455, Perplexity = 21.02 +Step 14510/44000 | Loss: 3.0428 | LR: 3.85e-04 | 3606.1ms/step | 5.5 samples/s +Step 14520/44000 | Loss: 3.0148 | LR: 3.85e-04 | 1397.6ms/step | 14.3 samples/s +Step 14530/44000 | Loss: 3.0438 | LR: 3.85e-04 | 1391.4ms/step | 14.4 samples/s +Step 14540/44000 | Loss: 3.0356 | LR: 3.85e-04 | 1393.3ms/step | 14.4 samples/s +Step 14550/44000 | Loss: 3.0077 | LR: 3.85e-04 | 1391.8ms/step | 14.4 samples/s +Step 14560/44000 | Loss: 3.0533 | LR: 3.84e-04 | 1394.0ms/step | 14.3 samples/s +Step 14570/44000 | Loss: 3.0163 | LR: 3.84e-04 | 1397.9ms/step | 14.3 samples/s +Step 14580/44000 | Loss: 3.0216 | LR: 3.84e-04 | 1400.0ms/step | 14.3 samples/s +Step 14590/44000 | Loss: 3.0249 | LR: 3.84e-04 | 1395.9ms/step | 14.3 samples/s +Step 14600/44000 | Loss: 2.9964 | LR: 3.84e-04 | 1396.2ms/step | 14.3 samples/s +Step 14610/44000 | Loss: 3.0260 | LR: 3.84e-04 | 1397.9ms/step | 14.3 samples/s +Step 14620/44000 | Loss: 3.0621 | LR: 3.83e-04 | 1392.1ms/step | 14.4 samples/s +Step 14630/44000 | Loss: 3.0128 | LR: 3.83e-04 | 1391.8ms/step | 14.4 samples/s +Step 14640/44000 | Loss: 3.0107 | LR: 3.83e-04 | 1392.3ms/step | 14.4 samples/s +Step 14650/44000 | Loss: 3.0298 | LR: 3.83e-04 | 1400.1ms/step | 14.3 samples/s +Step 14660/44000 | Loss: 3.0279 | LR: 3.83e-04 | 1396.3ms/step | 14.3 samples/s +Step 14670/44000 | Loss: 3.0152 | LR: 3.83e-04 | 1393.4ms/step | 14.4 samples/s +Step 14680/44000 | Loss: 3.0402 | LR: 3.83e-04 | 1390.1ms/step | 14.4 samples/s +Step 14690/44000 | Loss: 3.0441 | LR: 3.82e-04 | 1393.7ms/step | 14.4 samples/s +Step 14700/44000 | Loss: 3.0151 | LR: 3.82e-04 | 1394.2ms/step | 14.3 samples/s +Step 14710/44000 | Loss: 3.0453 | LR: 3.82e-04 | 1391.2ms/step | 14.4 samples/s +Step 14720/44000 | Loss: 3.0420 | LR: 3.82e-04 | 1398.8ms/step | 14.3 samples/s +Step 14730/44000 | Loss: 3.0334 | LR: 3.82e-04 | 1396.1ms/step | 14.3 samples/s +Step 14740/44000 | Loss: 3.0369 | LR: 3.82e-04 | 1394.1ms/step | 14.3 samples/s +Step 14750/44000 | Loss: 2.9947 | LR: 3.81e-04 | 1389.1ms/step | 14.4 samples/s +Validation at step 14750: Loss = 3.0419, Perplexity = 20.94 +Step 14760/44000 | Loss: 3.0402 | LR: 3.81e-04 | 3599.0ms/step | 5.6 samples/s +Step 14770/44000 | Loss: 3.0050 | LR: 3.81e-04 | 1393.5ms/step | 14.4 samples/s +Step 14780/44000 | Loss: 3.0023 | LR: 3.81e-04 | 1391.5ms/step | 14.4 samples/s +Step 14790/44000 | Loss: 3.0205 | LR: 3.81e-04 | 1392.9ms/step | 14.4 samples/s +Step 14800/44000 | Loss: 3.0404 | LR: 3.81e-04 | 1393.9ms/step | 14.3 samples/s +Step 14810/44000 | Loss: 2.9997 | LR: 3.81e-04 | 1395.7ms/step | 14.3 samples/s +Step 14820/44000 | Loss: 3.0355 | LR: 3.80e-04 | 1394.7ms/step | 14.3 samples/s +Step 14830/44000 | Loss: 3.0167 | LR: 3.80e-04 | 1391.3ms/step | 14.4 samples/s +Step 14840/44000 | Loss: 3.0248 | LR: 3.80e-04 | 1390.2ms/step | 14.4 samples/s +Step 14850/44000 | Loss: 3.0228 | LR: 3.80e-04 | 1389.3ms/step | 14.4 samples/s +Step 14860/44000 | Loss: 2.9985 | LR: 3.80e-04 | 1392.0ms/step | 14.4 samples/s +Step 14870/44000 | Loss: 2.9972 | LR: 3.80e-04 | 1392.8ms/step | 14.4 samples/s +Step 14880/44000 | Loss: 2.9931 | LR: 3.79e-04 | 1389.5ms/step | 14.4 samples/s +Step 14890/44000 | Loss: 3.0314 | LR: 3.79e-04 | 1390.2ms/step | 14.4 samples/s +Step 14900/44000 | Loss: 3.0371 | LR: 3.79e-04 | 1388.5ms/step | 14.4 samples/s +Step 14910/44000 | Loss: 3.0016 | LR: 3.79e-04 | 1394.0ms/step | 14.3 samples/s +Step 14920/44000 | Loss: 3.0413 | LR: 3.79e-04 | 1392.4ms/step | 14.4 samples/s +Step 14930/44000 | Loss: 3.0233 | LR: 3.79e-04 | 1397.9ms/step | 14.3 samples/s +Step 14940/44000 | Loss: 3.0310 | LR: 3.79e-04 | 1400.0ms/step | 14.3 samples/s +Step 14950/44000 | Loss: 3.0025 | LR: 3.78e-04 | 1399.6ms/step | 14.3 samples/s +Step 14960/44000 | Loss: 2.9620 | LR: 3.78e-04 | 1400.5ms/step | 14.3 samples/s +Step 14970/44000 | Loss: 3.0162 | LR: 3.78e-04 | 1399.9ms/step | 14.3 samples/s +Step 14980/44000 | Loss: 2.9999 | LR: 3.78e-04 | 1400.0ms/step | 14.3 samples/s +Step 14990/44000 | Loss: 3.0326 | LR: 3.78e-04 | 1396.0ms/step | 14.3 samples/s +Step 15000/44000 | Loss: 3.0642 | LR: 3.78e-04 | 1394.2ms/step | 14.3 samples/s +Validation at step 15000: Loss = 3.0391, Perplexity = 20.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-15000 +Step 15010/44000 | Loss: 3.0406 | LR: 3.77e-04 | 3911.6ms/step | 5.1 samples/s +Step 15020/44000 | Loss: 3.0102 | LR: 3.77e-04 | 1392.4ms/step | 14.4 samples/s +Step 15030/44000 | Loss: 3.0286 | LR: 3.77e-04 | 1394.1ms/step | 14.3 samples/s +Step 15040/44000 | Loss: 3.0379 | LR: 3.77e-04 | 1394.8ms/step | 14.3 samples/s +Step 15050/44000 | Loss: 3.0129 | LR: 3.77e-04 | 1390.9ms/step | 14.4 samples/s +Step 15060/44000 | Loss: 3.0472 | LR: 3.77e-04 | 1396.0ms/step | 14.3 samples/s +Step 15070/44000 | Loss: 3.0095 | LR: 3.77e-04 | 1394.0ms/step | 14.3 samples/s +Step 15080/44000 | Loss: 3.0502 | LR: 3.76e-04 | 1391.9ms/step | 14.4 samples/s +Step 15090/44000 | Loss: 3.0245 | LR: 3.76e-04 | 1390.2ms/step | 14.4 samples/s +Step 15100/44000 | Loss: 2.9995 | LR: 3.76e-04 | 1395.8ms/step | 14.3 samples/s +Step 15110/44000 | Loss: 3.0420 | LR: 3.76e-04 | 1392.5ms/step | 14.4 samples/s +Step 15120/44000 | Loss: 3.0446 | LR: 3.76e-04 | 1394.7ms/step | 14.3 samples/s +Step 15130/44000 | Loss: 3.0184 | LR: 3.76e-04 | 1394.7ms/step | 14.3 samples/s +Step 15140/44000 | Loss: 3.0281 | LR: 3.75e-04 | 1394.4ms/step | 14.3 samples/s +Step 15150/44000 | Loss: 3.0283 | LR: 3.75e-04 | 1398.3ms/step | 14.3 samples/s +Step 15160/44000 | Loss: 3.0490 | LR: 3.75e-04 | 1399.9ms/step | 14.3 samples/s +Step 15170/44000 | Loss: 3.0202 | LR: 3.75e-04 | 1399.8ms/step | 14.3 samples/s +Step 15180/44000 | Loss: 3.0533 | LR: 3.75e-04 | 1399.9ms/step | 14.3 samples/s +Step 15190/44000 | Loss: 3.0164 | LR: 3.75e-04 | 1395.2ms/step | 14.3 samples/s +Step 15200/44000 | Loss: 3.0552 | LR: 3.74e-04 | 1390.3ms/step | 14.4 samples/s +Step 15210/44000 | Loss: 3.0209 | LR: 3.74e-04 | 1394.4ms/step | 14.3 samples/s +Step 15220/44000 | Loss: 3.0104 | LR: 3.74e-04 | 1392.2ms/step | 14.4 samples/s +Step 15230/44000 | Loss: 2.9948 | LR: 3.74e-04 | 1395.7ms/step | 14.3 samples/s +Step 15240/44000 | Loss: 2.9763 | LR: 3.74e-04 | 1398.4ms/step | 14.3 samples/s +Step 15250/44000 | Loss: 3.0337 | LR: 3.74e-04 | 1400.1ms/step | 14.3 samples/s +Validation at step 15250: Loss = 3.0345, Perplexity = 20.79 +Step 15260/44000 | Loss: 3.0068 | LR: 3.74e-04 | 3601.8ms/step | 5.6 samples/s +Step 15270/44000 | Loss: 3.0046 | LR: 3.73e-04 | 1396.1ms/step | 14.3 samples/s +Step 15280/44000 | Loss: 3.0084 | LR: 3.73e-04 | 1394.1ms/step | 14.3 samples/s +Step 15290/44000 | Loss: 3.0184 | LR: 3.73e-04 | 1393.7ms/step | 14.4 samples/s +Step 15300/44000 | Loss: 3.0238 | LR: 3.73e-04 | 1395.6ms/step | 14.3 samples/s +Step 15310/44000 | Loss: 3.0355 | LR: 3.73e-04 | 1396.8ms/step | 14.3 samples/s +Step 15320/44000 | Loss: 3.0247 | LR: 3.73e-04 | 1399.9ms/step | 14.3 samples/s +Step 15330/44000 | Loss: 3.0090 | LR: 3.72e-04 | 1399.9ms/step | 14.3 samples/s +Step 15340/44000 | Loss: 3.0149 | LR: 3.72e-04 | 1400.3ms/step | 14.3 samples/s +Step 15350/44000 | Loss: 3.0052 | LR: 3.72e-04 | 1399.8ms/step | 14.3 samples/s +Step 15360/44000 | Loss: 3.0105 | LR: 3.72e-04 | 1400.3ms/step | 14.3 samples/s +Step 15370/44000 | Loss: 3.0158 | LR: 3.72e-04 | 1397.5ms/step | 14.3 samples/s +Step 15380/44000 | Loss: 3.0170 | LR: 3.72e-04 | 1392.3ms/step | 14.4 samples/s +Step 15390/44000 | Loss: 2.9849 | LR: 3.71e-04 | 1396.0ms/step | 14.3 samples/s +Step 15400/44000 | Loss: 2.9959 | LR: 3.71e-04 | 1393.9ms/step | 14.3 samples/s +Step 15410/44000 | Loss: 2.9897 | LR: 3.71e-04 | 1393.2ms/step | 14.4 samples/s +Step 15420/44000 | Loss: 3.0200 | LR: 3.71e-04 | 1397.0ms/step | 14.3 samples/s +Step 15430/44000 | Loss: 3.0057 | LR: 3.71e-04 | 1400.2ms/step | 14.3 samples/s +Step 15440/44000 | Loss: 3.0444 | LR: 3.71e-04 | 1399.9ms/step | 14.3 samples/s +Step 15450/44000 | Loss: 3.0214 | LR: 3.71e-04 | 1400.3ms/step | 14.3 samples/s +Step 15460/44000 | Loss: 3.0203 | LR: 3.70e-04 | 1399.6ms/step | 14.3 samples/s +Step 15470/44000 | Loss: 3.0151 | LR: 3.70e-04 | 1400.1ms/step | 14.3 samples/s +Step 15480/44000 | Loss: 3.0140 | LR: 3.70e-04 | 1399.9ms/step | 14.3 samples/s +Step 15490/44000 | Loss: 3.0073 | LR: 3.70e-04 | 1400.1ms/step | 14.3 samples/s +Step 15500/44000 | Loss: 3.0480 | LR: 3.70e-04 | 1399.9ms/step | 14.3 samples/s +Validation at step 15500: Loss = 3.0314, Perplexity = 20.73 +Step 15510/44000 | Loss: 3.0158 | LR: 3.70e-04 | 3616.0ms/step | 5.5 samples/s +Step 15520/44000 | Loss: 3.0418 | LR: 3.69e-04 | 1400.0ms/step | 14.3 samples/s +Step 15530/44000 | Loss: 3.0006 | LR: 3.69e-04 | 1399.7ms/step | 14.3 samples/s +Step 15540/44000 | Loss: 2.9908 | LR: 3.69e-04 | 1397.8ms/step | 14.3 samples/s +Step 15550/44000 | Loss: 2.9937 | LR: 3.69e-04 | 1390.3ms/step | 14.4 samples/s +Step 15560/44000 | Loss: 3.0068 | LR: 3.69e-04 | 1392.3ms/step | 14.4 samples/s +Step 15570/44000 | Loss: 3.0162 | LR: 3.69e-04 | 1397.7ms/step | 14.3 samples/s +Step 15580/44000 | Loss: 3.0212 | LR: 3.68e-04 | 1400.1ms/step | 14.3 samples/s +Step 15590/44000 | Loss: 3.0318 | LR: 3.68e-04 | 1400.1ms/step | 14.3 samples/s +Step 15600/44000 | Loss: 3.0112 | LR: 3.68e-04 | 1398.0ms/step | 14.3 samples/s +Step 15610/44000 | Loss: 3.0263 | LR: 3.68e-04 | 1399.3ms/step | 14.3 samples/s +Step 15620/44000 | Loss: 3.0363 | LR: 3.68e-04 | 1398.5ms/step | 14.3 samples/s +Step 15630/44000 | Loss: 2.9988 | LR: 3.68e-04 | 1396.2ms/step | 14.3 samples/s +Step 15640/44000 | Loss: 3.0180 | LR: 3.67e-04 | 1394.0ms/step | 14.3 samples/s +Step 15650/44000 | Loss: 3.0178 | LR: 3.67e-04 | 1397.8ms/step | 14.3 samples/s +Step 15660/44000 | Loss: 2.9908 | LR: 3.67e-04 | 1398.2ms/step | 14.3 samples/s +Step 15670/44000 | Loss: 3.0066 | LR: 3.67e-04 | 1395.8ms/step | 14.3 samples/s +Step 15680/44000 | Loss: 3.0632 | LR: 3.67e-04 | 1392.4ms/step | 14.4 samples/s +Step 15690/44000 | Loss: 3.0334 | LR: 3.67e-04 | 1391.1ms/step | 14.4 samples/s +Step 15700/44000 | Loss: 3.0486 | LR: 3.67e-04 | 1395.2ms/step | 14.3 samples/s +Step 15710/44000 | Loss: 3.0575 | LR: 3.66e-04 | 1397.5ms/step | 14.3 samples/s +Step 15720/44000 | Loss: 2.9989 | LR: 3.66e-04 | 1395.6ms/step | 14.3 samples/s +Step 15730/44000 | Loss: 3.0100 | LR: 3.66e-04 | 1394.3ms/step | 14.3 samples/s +Step 15740/44000 | Loss: 3.0061 | LR: 3.66e-04 | 1397.9ms/step | 14.3 samples/s +Step 15750/44000 | Loss: 3.0017 | LR: 3.66e-04 | 1394.4ms/step | 14.3 samples/s +Validation at step 15750: Loss = 3.0281, Perplexity = 20.66 +Step 15760/44000 | Loss: 3.0023 | LR: 3.66e-04 | 3592.2ms/step | 5.6 samples/s +Step 15770/44000 | Loss: 2.9937 | LR: 3.65e-04 | 1392.5ms/step | 14.4 samples/s +Step 15780/44000 | Loss: 3.0092 | LR: 3.65e-04 | 1391.9ms/step | 14.4 samples/s +Step 15790/44000 | Loss: 3.0139 | LR: 3.65e-04 | 1396.7ms/step | 14.3 samples/s +Step 15800/44000 | Loss: 3.0083 | LR: 3.65e-04 | 1398.7ms/step | 14.3 samples/s +Step 15810/44000 | Loss: 2.9938 | LR: 3.65e-04 | 1398.0ms/step | 14.3 samples/s +Step 15820/44000 | Loss: 2.9996 | LR: 3.65e-04 | 1400.0ms/step | 14.3 samples/s +Step 15830/44000 | Loss: 3.0223 | LR: 3.64e-04 | 1397.9ms/step | 14.3 samples/s +Step 15840/44000 | Loss: 3.0100 | LR: 3.64e-04 | 1395.9ms/step | 14.3 samples/s +Step 15850/44000 | Loss: 2.9974 | LR: 3.64e-04 | 1393.7ms/step | 14.4 samples/s +Step 15860/44000 | Loss: 3.0012 | LR: 3.64e-04 | 1396.6ms/step | 14.3 samples/s +Step 15870/44000 | Loss: 3.0115 | LR: 3.64e-04 | 1395.7ms/step | 14.3 samples/s +Step 15880/44000 | Loss: 3.0172 | LR: 3.64e-04 | 1390.1ms/step | 14.4 samples/s +Step 15890/44000 | Loss: 2.9824 | LR: 3.63e-04 | 1394.1ms/step | 14.3 samples/s +Step 15900/44000 | Loss: 3.0261 | LR: 3.63e-04 | 1400.0ms/step | 14.3 samples/s +Step 15910/44000 | Loss: 3.0246 | LR: 3.63e-04 | 1394.2ms/step | 14.3 samples/s +Step 15920/44000 | Loss: 3.0205 | LR: 3.63e-04 | 1393.8ms/step | 14.3 samples/s +Step 15930/44000 | Loss: 2.9790 | LR: 3.63e-04 | 1392.0ms/step | 14.4 samples/s +Step 15940/44000 | Loss: 3.0000 | LR: 3.63e-04 | 1394.2ms/step | 14.3 samples/s +Step 15950/44000 | Loss: 2.9839 | LR: 3.62e-04 | 1399.6ms/step | 14.3 samples/s +Step 15960/44000 | Loss: 2.9909 | LR: 3.62e-04 | 1399.0ms/step | 14.3 samples/s +Step 15970/44000 | Loss: 3.0032 | LR: 3.62e-04 | 1394.7ms/step | 14.3 samples/s +Step 15980/44000 | Loss: 3.0117 | LR: 3.62e-04 | 1396.5ms/step | 14.3 samples/s +Step 15990/44000 | Loss: 3.0170 | LR: 3.62e-04 | 1392.0ms/step | 14.4 samples/s +Step 16000/44000 | Loss: 3.0214 | LR: 3.62e-04 | 1391.6ms/step | 14.4 samples/s +Validation at step 16000: Loss = 3.0254, Perplexity = 20.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-16000 +Step 16010/44000 | Loss: 2.9716 | LR: 3.62e-04 | 3894.4ms/step | 5.1 samples/s +Step 16020/44000 | Loss: 2.9927 | LR: 3.61e-04 | 1388.2ms/step | 14.4 samples/s +Step 16030/44000 | Loss: 3.0083 | LR: 3.61e-04 | 1389.9ms/step | 14.4 samples/s +Step 16040/44000 | Loss: 2.9941 | LR: 3.61e-04 | 1392.8ms/step | 14.4 samples/s +Step 16050/44000 | Loss: 3.0282 | LR: 3.61e-04 | 1396.4ms/step | 14.3 samples/s +Step 16060/44000 | Loss: 2.9982 | LR: 3.61e-04 | 1394.7ms/step | 14.3 samples/s +Step 16070/44000 | Loss: 3.0217 | LR: 3.61e-04 | 1395.2ms/step | 14.3 samples/s +Step 16080/44000 | Loss: 3.0088 | LR: 3.60e-04 | 1392.9ms/step | 14.4 samples/s +Step 16090/44000 | Loss: 2.9769 | LR: 3.60e-04 | 1391.8ms/step | 14.4 samples/s +Step 16100/44000 | Loss: 2.9986 | LR: 3.60e-04 | 1391.1ms/step | 14.4 samples/s +Step 16110/44000 | Loss: 3.0140 | LR: 3.60e-04 | 1395.2ms/step | 14.3 samples/s +Step 16120/44000 | Loss: 3.0127 | LR: 3.60e-04 | 1395.8ms/step | 14.3 samples/s +Step 16130/44000 | Loss: 2.9961 | LR: 3.60e-04 | 1390.2ms/step | 14.4 samples/s +Step 16140/44000 | Loss: 3.0038 | LR: 3.59e-04 | 1390.1ms/step | 14.4 samples/s +Step 16150/44000 | Loss: 2.9810 | LR: 3.59e-04 | 1391.2ms/step | 14.4 samples/s +Step 16160/44000 | Loss: 2.9990 | LR: 3.59e-04 | 1393.8ms/step | 14.3 samples/s +Step 16170/44000 | Loss: 3.0220 | LR: 3.59e-04 | 1396.9ms/step | 14.3 samples/s +Step 16180/44000 | Loss: 3.0280 | LR: 3.59e-04 | 1400.1ms/step | 14.3 samples/s +Step 16190/44000 | Loss: 3.0187 | LR: 3.59e-04 | 1399.7ms/step | 14.3 samples/s +Step 16200/44000 | Loss: 2.9890 | LR: 3.58e-04 | 1392.2ms/step | 14.4 samples/s +Step 16210/44000 | Loss: 2.9894 | LR: 3.58e-04 | 1388.6ms/step | 14.4 samples/s +Step 16220/44000 | Loss: 3.0346 | LR: 3.58e-04 | 1387.5ms/step | 14.4 samples/s +Step 16230/44000 | Loss: 2.9713 | LR: 3.58e-04 | 1390.1ms/step | 14.4 samples/s +Step 16240/44000 | Loss: 2.9878 | LR: 3.58e-04 | 1393.8ms/step | 14.3 samples/s +Step 16250/44000 | Loss: 3.0294 | LR: 3.58e-04 | 1395.8ms/step | 14.3 samples/s +Validation at step 16250: Loss = 3.0214, Perplexity = 20.52 +Step 16260/44000 | Loss: 2.9884 | LR: 3.57e-04 | 3608.2ms/step | 5.5 samples/s +Step 16270/44000 | Loss: 3.0472 | LR: 3.57e-04 | 1392.3ms/step | 14.4 samples/s +Step 16280/44000 | Loss: 3.0050 | LR: 3.57e-04 | 1392.5ms/step | 14.4 samples/s +Step 16290/44000 | Loss: 2.9874 | LR: 3.57e-04 | 1393.0ms/step | 14.4 samples/s +Step 16300/44000 | Loss: 3.0319 | LR: 3.57e-04 | 1396.2ms/step | 14.3 samples/s +Step 16310/44000 | Loss: 3.0181 | LR: 3.57e-04 | 1389.8ms/step | 14.4 samples/s +Step 16320/44000 | Loss: 2.9744 | LR: 3.56e-04 | 1390.3ms/step | 14.4 samples/s +Step 16330/44000 | Loss: 2.9910 | LR: 3.56e-04 | 1391.8ms/step | 14.4 samples/s +Step 16340/44000 | Loss: 3.0156 | LR: 3.56e-04 | 1390.1ms/step | 14.4 samples/s +Step 16350/44000 | Loss: 3.0145 | LR: 3.56e-04 | 1390.0ms/step | 14.4 samples/s +Step 16360/44000 | Loss: 3.0218 | LR: 3.56e-04 | 1394.2ms/step | 14.3 samples/s +Step 16370/44000 | Loss: 2.9926 | LR: 3.56e-04 | 1400.1ms/step | 14.3 samples/s +Step 16380/44000 | Loss: 3.0077 | LR: 3.55e-04 | 1395.0ms/step | 14.3 samples/s +Step 16390/44000 | Loss: 3.0400 | LR: 3.55e-04 | 1393.1ms/step | 14.4 samples/s +Step 16400/44000 | Loss: 2.9877 | LR: 3.55e-04 | 1397.8ms/step | 14.3 samples/s +Step 16410/44000 | Loss: 2.9991 | LR: 3.55e-04 | 1395.4ms/step | 14.3 samples/s +Step 16420/44000 | Loss: 2.9939 | LR: 3.55e-04 | 1392.7ms/step | 14.4 samples/s +Step 16430/44000 | Loss: 3.0149 | LR: 3.55e-04 | 1397.9ms/step | 14.3 samples/s +Step 16440/44000 | Loss: 3.0015 | LR: 3.54e-04 | 1400.1ms/step | 14.3 samples/s +Step 16450/44000 | Loss: 3.0041 | LR: 3.54e-04 | 1398.0ms/step | 14.3 samples/s +Step 16460/44000 | Loss: 3.0172 | LR: 3.54e-04 | 1393.7ms/step | 14.4 samples/s +Step 16470/44000 | Loss: 2.9818 | LR: 3.54e-04 | 1390.1ms/step | 14.4 samples/s +Step 16480/44000 | Loss: 2.9857 | LR: 3.54e-04 | 1398.4ms/step | 14.3 samples/s +Step 16490/44000 | Loss: 2.9812 | LR: 3.54e-04 | 1400.1ms/step | 14.3 samples/s +Step 16500/44000 | Loss: 2.9962 | LR: 3.53e-04 | 1395.1ms/step | 14.3 samples/s +Validation at step 16500: Loss = 3.0183, Perplexity = 20.46 +Step 16510/44000 | Loss: 3.0187 | LR: 3.53e-04 | 3602.7ms/step | 5.6 samples/s +Step 16520/44000 | Loss: 2.9956 | LR: 3.53e-04 | 1399.6ms/step | 14.3 samples/s +Step 16530/44000 | Loss: 2.9697 | LR: 3.53e-04 | 1391.8ms/step | 14.4 samples/s +Step 16540/44000 | Loss: 3.0046 | LR: 3.53e-04 | 1394.5ms/step | 14.3 samples/s +Step 16550/44000 | Loss: 2.9612 | LR: 3.53e-04 | 1398.2ms/step | 14.3 samples/s +Step 16560/44000 | Loss: 2.9865 | LR: 3.52e-04 | 1399.8ms/step | 14.3 samples/s +Step 16570/44000 | Loss: 3.0200 | LR: 3.52e-04 | 1398.0ms/step | 14.3 samples/s +Step 16580/44000 | Loss: 2.9855 | LR: 3.52e-04 | 1400.0ms/step | 14.3 samples/s +Step 16590/44000 | Loss: 2.9965 | LR: 3.52e-04 | 1400.2ms/step | 14.3 samples/s +Step 16600/44000 | Loss: 2.9828 | LR: 3.52e-04 | 1400.1ms/step | 14.3 samples/s +Step 16610/44000 | Loss: 3.0016 | LR: 3.52e-04 | 1400.0ms/step | 14.3 samples/s +Step 16620/44000 | Loss: 3.0052 | LR: 3.51e-04 | 1399.8ms/step | 14.3 samples/s +Step 16630/44000 | Loss: 2.9830 | LR: 3.51e-04 | 1400.2ms/step | 14.3 samples/s +Step 16640/44000 | Loss: 3.0154 | LR: 3.51e-04 | 1399.8ms/step | 14.3 samples/s +Step 16650/44000 | Loss: 2.9823 | LR: 3.51e-04 | 1398.0ms/step | 14.3 samples/s +Step 16660/44000 | Loss: 2.9820 | LR: 3.51e-04 | 1396.0ms/step | 14.3 samples/s +Step 16670/44000 | Loss: 3.0086 | LR: 3.51e-04 | 1396.1ms/step | 14.3 samples/s +Step 16680/44000 | Loss: 2.9618 | LR: 3.50e-04 | 1398.1ms/step | 14.3 samples/s +Step 16690/44000 | Loss: 2.9888 | LR: 3.50e-04 | 1400.0ms/step | 14.3 samples/s +Step 16700/44000 | Loss: 3.0236 | LR: 3.50e-04 | 1399.8ms/step | 14.3 samples/s +Step 16710/44000 | Loss: 2.9635 | LR: 3.50e-04 | 1400.1ms/step | 14.3 samples/s +Step 16720/44000 | Loss: 3.0103 | LR: 3.50e-04 | 1400.1ms/step | 14.3 samples/s +Step 16730/44000 | Loss: 2.9752 | LR: 3.50e-04 | 1399.8ms/step | 14.3 samples/s +Step 16740/44000 | Loss: 2.9800 | LR: 3.49e-04 | 1399.9ms/step | 14.3 samples/s +Step 16750/44000 | Loss: 2.9734 | LR: 3.49e-04 | 1392.1ms/step | 14.4 samples/s +Validation at step 16750: Loss = 3.0156, Perplexity = 20.40 +Step 16760/44000 | Loss: 3.0175 | LR: 3.49e-04 | 3606.2ms/step | 5.5 samples/s +Step 16770/44000 | Loss: 3.0220 | LR: 3.49e-04 | 1400.0ms/step | 14.3 samples/s +Step 16780/44000 | Loss: 3.0114 | LR: 3.49e-04 | 1399.7ms/step | 14.3 samples/s +Step 16790/44000 | Loss: 3.0291 | LR: 3.49e-04 | 1400.3ms/step | 14.3 samples/s +Step 16800/44000 | Loss: 3.0043 | LR: 3.48e-04 | 1399.9ms/step | 14.3 samples/s +Step 16810/44000 | Loss: 2.9812 | LR: 3.48e-04 | 1395.4ms/step | 14.3 samples/s +Step 16820/44000 | Loss: 3.0271 | LR: 3.48e-04 | 1392.6ms/step | 14.4 samples/s +Step 16830/44000 | Loss: 3.0220 | LR: 3.48e-04 | 1393.9ms/step | 14.3 samples/s +Step 16840/44000 | Loss: 2.9935 | LR: 3.48e-04 | 1392.6ms/step | 14.4 samples/s +Step 16850/44000 | Loss: 2.9810 | LR: 3.48e-04 | 1394.7ms/step | 14.3 samples/s +Step 16860/44000 | Loss: 3.0225 | LR: 3.47e-04 | 1394.9ms/step | 14.3 samples/s +Step 16870/44000 | Loss: 2.9918 | LR: 3.47e-04 | 1394.1ms/step | 14.3 samples/s +Step 16880/44000 | Loss: 3.0154 | LR: 3.47e-04 | 1393.7ms/step | 14.4 samples/s +Step 16890/44000 | Loss: 2.9837 | LR: 3.47e-04 | 1394.4ms/step | 14.3 samples/s +Step 16900/44000 | Loss: 3.0093 | LR: 3.47e-04 | 1400.1ms/step | 14.3 samples/s +Step 16910/44000 | Loss: 2.9963 | LR: 3.47e-04 | 1395.7ms/step | 14.3 samples/s +Step 16920/44000 | Loss: 3.0097 | LR: 3.46e-04 | 1393.0ms/step | 14.4 samples/s +Step 16930/44000 | Loss: 3.0300 | LR: 3.46e-04 | 1391.0ms/step | 14.4 samples/s +Step 16940/44000 | Loss: 2.9910 | LR: 3.46e-04 | 1391.9ms/step | 14.4 samples/s +Step 16950/44000 | Loss: 2.9802 | LR: 3.46e-04 | 1394.2ms/step | 14.3 samples/s +Step 16960/44000 | Loss: 2.9943 | LR: 3.46e-04 | 1393.8ms/step | 14.3 samples/s +Step 16970/44000 | Loss: 3.0193 | LR: 3.46e-04 | 1394.1ms/step | 14.3 samples/s +Step 16980/44000 | Loss: 3.0293 | LR: 3.45e-04 | 1398.2ms/step | 14.3 samples/s +Step 16990/44000 | Loss: 2.9844 | LR: 3.45e-04 | 1397.6ms/step | 14.3 samples/s +Step 17000/44000 | Loss: 2.9964 | LR: 3.45e-04 | 1392.2ms/step | 14.4 samples/s +Validation at step 17000: Loss = 3.0130, Perplexity = 20.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-17000 +Step 17010/44000 | Loss: 3.0126 | LR: 3.45e-04 | 3902.1ms/step | 5.1 samples/s +Step 17020/44000 | Loss: 2.9930 | LR: 3.45e-04 | 1399.9ms/step | 14.3 samples/s +Step 17030/44000 | Loss: 2.9971 | LR: 3.45e-04 | 1400.1ms/step | 14.3 samples/s +Step 17040/44000 | Loss: 3.0329 | LR: 3.44e-04 | 1400.0ms/step | 14.3 samples/s +Step 17050/44000 | Loss: 3.0255 | LR: 3.44e-04 | 1395.9ms/step | 14.3 samples/s +Step 17060/44000 | Loss: 2.9787 | LR: 3.44e-04 | 1395.9ms/step | 14.3 samples/s +Step 17070/44000 | Loss: 2.9632 | LR: 3.44e-04 | 1400.2ms/step | 14.3 samples/s +Step 17080/44000 | Loss: 3.0105 | LR: 3.44e-04 | 1396.1ms/step | 14.3 samples/s +Step 17090/44000 | Loss: 2.9933 | LR: 3.44e-04 | 1394.0ms/step | 14.3 samples/s +Step 17100/44000 | Loss: 2.9983 | LR: 3.43e-04 | 1393.8ms/step | 14.3 samples/s +Step 17110/44000 | Loss: 2.9912 | LR: 3.43e-04 | 1398.1ms/step | 14.3 samples/s +Step 17120/44000 | Loss: 2.9956 | LR: 3.43e-04 | 1394.3ms/step | 14.3 samples/s +Step 17130/44000 | Loss: 2.9834 | LR: 3.43e-04 | 1394.9ms/step | 14.3 samples/s +Step 17140/44000 | Loss: 3.0120 | LR: 3.43e-04 | 1394.5ms/step | 14.3 samples/s +Step 17150/44000 | Loss: 3.0145 | LR: 3.43e-04 | 1398.4ms/step | 14.3 samples/s +Step 17160/44000 | Loss: 2.9790 | LR: 3.42e-04 | 1400.0ms/step | 14.3 samples/s +Step 17170/44000 | Loss: 2.9916 | LR: 3.42e-04 | 1400.1ms/step | 14.3 samples/s +Step 17180/44000 | Loss: 2.9716 | LR: 3.42e-04 | 1399.7ms/step | 14.3 samples/s +Step 17190/44000 | Loss: 3.0165 | LR: 3.42e-04 | 1395.4ms/step | 14.3 samples/s +Step 17200/44000 | Loss: 2.9895 | LR: 3.42e-04 | 1395.9ms/step | 14.3 samples/s +Step 17210/44000 | Loss: 3.0097 | LR: 3.42e-04 | 1396.9ms/step | 14.3 samples/s +Step 17220/44000 | Loss: 2.9973 | LR: 3.41e-04 | 1398.1ms/step | 14.3 samples/s +Step 17230/44000 | Loss: 2.9903 | LR: 3.41e-04 | 1399.3ms/step | 14.3 samples/s +Step 17240/44000 | Loss: 2.9931 | LR: 3.41e-04 | 1396.7ms/step | 14.3 samples/s +Step 17250/44000 | Loss: 3.0038 | LR: 3.41e-04 | 1395.7ms/step | 14.3 samples/s +Validation at step 17250: Loss = 3.0100, Perplexity = 20.29 +Step 17260/44000 | Loss: 2.9971 | LR: 3.41e-04 | 3610.2ms/step | 5.5 samples/s +Step 17270/44000 | Loss: 3.0060 | LR: 3.41e-04 | 1400.2ms/step | 14.3 samples/s +Step 17280/44000 | Loss: 2.9944 | LR: 3.40e-04 | 1396.0ms/step | 14.3 samples/s +Step 17290/44000 | Loss: 2.9940 | LR: 3.40e-04 | 1400.0ms/step | 14.3 samples/s +Step 17300/44000 | Loss: 2.9804 | LR: 3.40e-04 | 1399.8ms/step | 14.3 samples/s +Step 17310/44000 | Loss: 2.9887 | LR: 3.40e-04 | 1396.0ms/step | 14.3 samples/s +Step 17320/44000 | Loss: 2.9920 | LR: 3.40e-04 | 1391.0ms/step | 14.4 samples/s +Step 17330/44000 | Loss: 2.9980 | LR: 3.40e-04 | 1397.3ms/step | 14.3 samples/s +Step 17340/44000 | Loss: 2.9935 | LR: 3.39e-04 | 1395.6ms/step | 14.3 samples/s +Step 17350/44000 | Loss: 2.9917 | LR: 3.39e-04 | 1398.2ms/step | 14.3 samples/s +Step 17360/44000 | Loss: 2.9885 | LR: 3.39e-04 | 1391.9ms/step | 14.4 samples/s +Step 17370/44000 | Loss: 2.9646 | LR: 3.39e-04 | 1398.3ms/step | 14.3 samples/s +Step 17380/44000 | Loss: 3.0130 | LR: 3.39e-04 | 1393.7ms/step | 14.3 samples/s +Step 17390/44000 | Loss: 2.9873 | LR: 3.39e-04 | 1394.0ms/step | 14.3 samples/s +Step 17400/44000 | Loss: 3.0157 | LR: 3.38e-04 | 1398.0ms/step | 14.3 samples/s +Step 17410/44000 | Loss: 2.9954 | LR: 3.38e-04 | 1400.0ms/step | 14.3 samples/s +Step 17420/44000 | Loss: 2.9714 | LR: 3.38e-04 | 1395.2ms/step | 14.3 samples/s +Step 17430/44000 | Loss: 2.9980 | LR: 3.38e-04 | 1389.8ms/step | 14.4 samples/s +Step 17440/44000 | Loss: 2.9920 | LR: 3.38e-04 | 1391.0ms/step | 14.4 samples/s +Step 17450/44000 | Loss: 2.9667 | LR: 3.38e-04 | 1395.2ms/step | 14.3 samples/s +Step 17460/44000 | Loss: 2.9858 | LR: 3.37e-04 | 1394.0ms/step | 14.3 samples/s +Step 17470/44000 | Loss: 2.9866 | LR: 3.37e-04 | 1396.7ms/step | 14.3 samples/s +Step 17480/44000 | Loss: 3.0101 | LR: 3.37e-04 | 1398.3ms/step | 14.3 samples/s +Step 17490/44000 | Loss: 2.9654 | LR: 3.37e-04 | 1395.9ms/step | 14.3 samples/s +Step 17500/44000 | Loss: 3.0160 | LR: 3.37e-04 | 1399.8ms/step | 14.3 samples/s +Validation at step 17500: Loss = 3.0063, Perplexity = 20.21 +Step 17510/44000 | Loss: 2.9933 | LR: 3.37e-04 | 3599.9ms/step | 5.6 samples/s +Step 17520/44000 | Loss: 2.9523 | LR: 3.36e-04 | 1395.7ms/step | 14.3 samples/s +Step 17530/44000 | Loss: 2.9900 | LR: 3.36e-04 | 1394.6ms/step | 14.3 samples/s +Step 17540/44000 | Loss: 3.0165 | LR: 3.36e-04 | 1396.4ms/step | 14.3 samples/s +Step 17550/44000 | Loss: 2.9763 | LR: 3.36e-04 | 1397.3ms/step | 14.3 samples/s +Step 17560/44000 | Loss: 2.9797 | LR: 3.36e-04 | 1396.5ms/step | 14.3 samples/s +Step 17570/44000 | Loss: 3.0130 | LR: 3.35e-04 | 1397.6ms/step | 14.3 samples/s +Step 17580/44000 | Loss: 2.9896 | LR: 3.35e-04 | 1400.2ms/step | 14.3 samples/s +Step 17590/44000 | Loss: 2.9920 | LR: 3.35e-04 | 1400.1ms/step | 14.3 samples/s +Step 17600/44000 | Loss: 3.0049 | LR: 3.35e-04 | 1398.3ms/step | 14.3 samples/s +Step 17610/44000 | Loss: 2.9976 | LR: 3.35e-04 | 1397.8ms/step | 14.3 samples/s +Step 17620/44000 | Loss: 3.0504 | LR: 3.35e-04 | 1399.9ms/step | 14.3 samples/s +Step 17630/44000 | Loss: 3.0193 | LR: 3.34e-04 | 1399.8ms/step | 14.3 samples/s +Step 17640/44000 | Loss: 3.0047 | LR: 3.34e-04 | 1400.5ms/step | 14.3 samples/s +Step 17650/44000 | Loss: 2.9721 | LR: 3.34e-04 | 1399.8ms/step | 14.3 samples/s +Step 17660/44000 | Loss: 3.0028 | LR: 3.34e-04 | 1393.8ms/step | 14.3 samples/s +Step 17670/44000 | Loss: 3.0159 | LR: 3.34e-04 | 1389.8ms/step | 14.4 samples/s +Step 17680/44000 | Loss: 2.9626 | LR: 3.34e-04 | 1390.1ms/step | 14.4 samples/s +Step 17690/44000 | Loss: 2.9842 | LR: 3.33e-04 | 1386.3ms/step | 14.4 samples/s +Step 17700/44000 | Loss: 3.0095 | LR: 3.33e-04 | 1389.9ms/step | 14.4 samples/s +Step 17710/44000 | Loss: 2.9829 | LR: 3.33e-04 | 1389.2ms/step | 14.4 samples/s +Step 17720/44000 | Loss: 2.9767 | LR: 3.33e-04 | 1387.4ms/step | 14.4 samples/s +Step 17730/44000 | Loss: 2.9588 | LR: 3.33e-04 | 1388.2ms/step | 14.4 samples/s +Step 17740/44000 | Loss: 2.9717 | LR: 3.33e-04 | 1392.4ms/step | 14.4 samples/s +Step 17750/44000 | Loss: 3.0061 | LR: 3.32e-04 | 1391.3ms/step | 14.4 samples/s +Validation at step 17750: Loss = 3.0039, Perplexity = 20.17 +Step 17760/44000 | Loss: 2.9858 | LR: 3.32e-04 | 3593.4ms/step | 5.6 samples/s +Step 17770/44000 | Loss: 3.0009 | LR: 3.32e-04 | 1393.7ms/step | 14.3 samples/s +Step 17780/44000 | Loss: 3.0066 | LR: 3.32e-04 | 1396.2ms/step | 14.3 samples/s +Step 17790/44000 | Loss: 2.9848 | LR: 3.32e-04 | 1400.2ms/step | 14.3 samples/s +Step 17800/44000 | Loss: 2.9876 | LR: 3.32e-04 | 1400.2ms/step | 14.3 samples/s +Step 17810/44000 | Loss: 2.9880 | LR: 3.31e-04 | 1398.0ms/step | 14.3 samples/s +Step 17820/44000 | Loss: 3.0177 | LR: 3.31e-04 | 1395.9ms/step | 14.3 samples/s +Step 17830/44000 | Loss: 2.9794 | LR: 3.31e-04 | 1393.8ms/step | 14.3 samples/s +Step 17840/44000 | Loss: 2.9958 | LR: 3.31e-04 | 1392.4ms/step | 14.4 samples/s +Step 17850/44000 | Loss: 2.9829 | LR: 3.31e-04 | 1391.7ms/step | 14.4 samples/s +Step 17860/44000 | Loss: 2.9691 | LR: 3.31e-04 | 1394.2ms/step | 14.3 samples/s +Step 17870/44000 | Loss: 2.9861 | LR: 3.30e-04 | 1393.4ms/step | 14.4 samples/s +Step 17880/44000 | Loss: 2.9750 | LR: 3.30e-04 | 1396.4ms/step | 14.3 samples/s +Step 17890/44000 | Loss: 2.9568 | LR: 3.30e-04 | 1390.3ms/step | 14.4 samples/s +Step 17900/44000 | Loss: 2.9989 | LR: 3.30e-04 | 1391.9ms/step | 14.4 samples/s +Step 17910/44000 | Loss: 3.0055 | LR: 3.30e-04 | 1396.1ms/step | 14.3 samples/s +Step 17920/44000 | Loss: 2.9802 | LR: 3.29e-04 | 1393.2ms/step | 14.4 samples/s +Step 17930/44000 | Loss: 2.9830 | LR: 3.29e-04 | 1394.7ms/step | 14.3 samples/s +Step 17940/44000 | Loss: 2.9840 | LR: 3.29e-04 | 1400.2ms/step | 14.3 samples/s +Step 17950/44000 | Loss: 2.9826 | LR: 3.29e-04 | 1400.0ms/step | 14.3 samples/s +Step 17960/44000 | Loss: 2.9974 | LR: 3.29e-04 | 1399.9ms/step | 14.3 samples/s +Step 17970/44000 | Loss: 3.0018 | LR: 3.29e-04 | 1400.1ms/step | 14.3 samples/s +Step 17980/44000 | Loss: 2.9597 | LR: 3.28e-04 | 1400.2ms/step | 14.3 samples/s +Step 17990/44000 | Loss: 2.9930 | LR: 3.28e-04 | 1399.8ms/step | 14.3 samples/s +Step 18000/44000 | Loss: 3.0003 | LR: 3.28e-04 | 1400.1ms/step | 14.3 samples/s +Validation at step 18000: Loss = 3.0005, Perplexity = 20.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-18000 +Step 18010/44000 | Loss: 2.9786 | LR: 3.28e-04 | 3936.0ms/step | 5.1 samples/s +Step 18020/44000 | Loss: 2.9768 | LR: 3.28e-04 | 1399.8ms/step | 14.3 samples/s +Step 18030/44000 | Loss: 2.9535 | LR: 3.28e-04 | 1400.0ms/step | 14.3 samples/s +Step 18040/44000 | Loss: 3.0073 | LR: 3.27e-04 | 1400.1ms/step | 14.3 samples/s +Step 18050/44000 | Loss: 2.9737 | LR: 3.27e-04 | 1400.1ms/step | 14.3 samples/s +Step 18060/44000 | Loss: 2.9793 | LR: 3.27e-04 | 1399.5ms/step | 14.3 samples/s +Step 18070/44000 | Loss: 2.9618 | LR: 3.27e-04 | 1397.8ms/step | 14.3 samples/s +Step 18080/44000 | Loss: 2.9741 | LR: 3.27e-04 | 1391.9ms/step | 14.4 samples/s +Step 18090/44000 | Loss: 2.9860 | LR: 3.27e-04 | 1390.6ms/step | 14.4 samples/s +Step 18100/44000 | Loss: 2.9644 | LR: 3.26e-04 | 1393.7ms/step | 14.4 samples/s +Step 18110/44000 | Loss: 3.0022 | LR: 3.26e-04 | 1396.0ms/step | 14.3 samples/s +Step 18120/44000 | Loss: 2.9990 | LR: 3.26e-04 | 1391.5ms/step | 14.4 samples/s +Step 18130/44000 | Loss: 2.9861 | LR: 3.26e-04 | 1392.9ms/step | 14.4 samples/s +Step 18140/44000 | Loss: 2.9730 | LR: 3.26e-04 | 1389.8ms/step | 14.4 samples/s +Step 18150/44000 | Loss: 2.9610 | LR: 3.26e-04 | 1390.6ms/step | 14.4 samples/s +Step 18160/44000 | Loss: 2.9863 | LR: 3.25e-04 | 1393.5ms/step | 14.4 samples/s +Step 18170/44000 | Loss: 2.9891 | LR: 3.25e-04 | 1392.1ms/step | 14.4 samples/s +Step 18180/44000 | Loss: 2.9755 | LR: 3.25e-04 | 1390.6ms/step | 14.4 samples/s +Step 18190/44000 | Loss: 3.0018 | LR: 3.25e-04 | 1393.1ms/step | 14.4 samples/s +Step 18200/44000 | Loss: 2.9898 | LR: 3.25e-04 | 1396.3ms/step | 14.3 samples/s +Step 18210/44000 | Loss: 2.9834 | LR: 3.24e-04 | 1400.0ms/step | 14.3 samples/s +Step 18220/44000 | Loss: 2.9742 | LR: 3.24e-04 | 1399.8ms/step | 14.3 samples/s +Step 18230/44000 | Loss: 2.9759 | LR: 3.24e-04 | 1400.2ms/step | 14.3 samples/s +Step 18240/44000 | Loss: 2.9745 | LR: 3.24e-04 | 1400.1ms/step | 14.3 samples/s +Step 18250/44000 | Loss: 2.9688 | LR: 3.24e-04 | 1399.8ms/step | 14.3 samples/s +Validation at step 18250: Loss = 2.9973, Perplexity = 20.03 +Step 18260/44000 | Loss: 2.9847 | LR: 3.24e-04 | 3606.5ms/step | 5.5 samples/s +Step 18270/44000 | Loss: 2.9879 | LR: 3.23e-04 | 1391.2ms/step | 14.4 samples/s +Step 18280/44000 | Loss: 2.9983 | LR: 3.23e-04 | 1394.0ms/step | 14.3 samples/s +Step 18290/44000 | Loss: 2.9667 | LR: 3.23e-04 | 1396.3ms/step | 14.3 samples/s +Step 18300/44000 | Loss: 2.9772 | LR: 3.23e-04 | 1390.1ms/step | 14.4 samples/s +Step 18310/44000 | Loss: 3.0112 | LR: 3.23e-04 | 1390.1ms/step | 14.4 samples/s +Step 18320/44000 | Loss: 2.9610 | LR: 3.23e-04 | 1392.1ms/step | 14.4 samples/s +Step 18330/44000 | Loss: 3.0157 | LR: 3.22e-04 | 1391.2ms/step | 14.4 samples/s +Step 18340/44000 | Loss: 2.9942 | LR: 3.22e-04 | 1394.6ms/step | 14.3 samples/s +Step 18350/44000 | Loss: 2.9934 | LR: 3.22e-04 | 1396.0ms/step | 14.3 samples/s +Step 18360/44000 | Loss: 3.0120 | LR: 3.22e-04 | 1395.7ms/step | 14.3 samples/s +Step 18370/44000 | Loss: 2.9751 | LR: 3.22e-04 | 1394.1ms/step | 14.3 samples/s +Step 18380/44000 | Loss: 2.9717 | LR: 3.22e-04 | 1392.4ms/step | 14.4 samples/s +Step 18390/44000 | Loss: 3.0051 | LR: 3.21e-04 | 1397.6ms/step | 14.3 samples/s +Step 18400/44000 | Loss: 2.9976 | LR: 3.21e-04 | 1396.3ms/step | 14.3 samples/s +Step 18410/44000 | Loss: 2.9853 | LR: 3.21e-04 | 1398.0ms/step | 14.3 samples/s +Step 18420/44000 | Loss: 2.9576 | LR: 3.21e-04 | 1400.0ms/step | 14.3 samples/s +Step 18430/44000 | Loss: 2.9966 | LR: 3.21e-04 | 1398.0ms/step | 14.3 samples/s +Step 18440/44000 | Loss: 2.9749 | LR: 3.20e-04 | 1400.0ms/step | 14.3 samples/s +Step 18450/44000 | Loss: 2.9981 | LR: 3.20e-04 | 1395.5ms/step | 14.3 samples/s +Step 18460/44000 | Loss: 3.0110 | LR: 3.20e-04 | 1388.1ms/step | 14.4 samples/s +Step 18470/44000 | Loss: 2.9553 | LR: 3.20e-04 | 1386.1ms/step | 14.4 samples/s +Step 18480/44000 | Loss: 2.9900 | LR: 3.20e-04 | 1390.5ms/step | 14.4 samples/s +Step 18490/44000 | Loss: 2.9917 | LR: 3.20e-04 | 1391.9ms/step | 14.4 samples/s +Step 18500/44000 | Loss: 2.9859 | LR: 3.19e-04 | 1392.1ms/step | 14.4 samples/s +Validation at step 18500: Loss = 2.9960, Perplexity = 20.01 +Step 18510/44000 | Loss: 2.9607 | LR: 3.19e-04 | 3597.3ms/step | 5.6 samples/s +Step 18520/44000 | Loss: 2.9637 | LR: 3.19e-04 | 1398.7ms/step | 14.3 samples/s +Step 18530/44000 | Loss: 2.9571 | LR: 3.19e-04 | 1399.9ms/step | 14.3 samples/s +Step 18540/44000 | Loss: 3.0085 | LR: 3.19e-04 | 1398.1ms/step | 14.3 samples/s +Step 18550/44000 | Loss: 2.9799 | LR: 3.19e-04 | 1390.0ms/step | 14.4 samples/s +Step 18560/44000 | Loss: 2.9475 | LR: 3.18e-04 | 1391.5ms/step | 14.4 samples/s +Step 18570/44000 | Loss: 2.9840 | LR: 3.18e-04 | 1390.3ms/step | 14.4 samples/s +Step 18580/44000 | Loss: 2.9526 | LR: 3.18e-04 | 1392.0ms/step | 14.4 samples/s +Step 18590/44000 | Loss: 2.9718 | LR: 3.18e-04 | 1392.3ms/step | 14.4 samples/s +Step 18600/44000 | Loss: 2.9948 | LR: 3.18e-04 | 1400.2ms/step | 14.3 samples/s +Step 18610/44000 | Loss: 3.0054 | LR: 3.18e-04 | 1399.9ms/step | 14.3 samples/s +Step 18620/44000 | Loss: 2.9663 | LR: 3.17e-04 | 1399.4ms/step | 14.3 samples/s +Step 18630/44000 | Loss: 2.9688 | LR: 3.17e-04 | 1394.0ms/step | 14.3 samples/s +Step 18640/44000 | Loss: 2.9722 | LR: 3.17e-04 | 1391.1ms/step | 14.4 samples/s +Step 18650/44000 | Loss: 2.9589 | LR: 3.17e-04 | 1389.4ms/step | 14.4 samples/s +Step 18660/44000 | Loss: 2.9343 | LR: 3.17e-04 | 1387.7ms/step | 14.4 samples/s +Step 18670/44000 | Loss: 2.9550 | LR: 3.16e-04 | 1392.3ms/step | 14.4 samples/s +Step 18680/44000 | Loss: 2.9717 | LR: 3.16e-04 | 1393.7ms/step | 14.4 samples/s +Step 18690/44000 | Loss: 2.9754 | LR: 3.16e-04 | 1398.2ms/step | 14.3 samples/s +Step 18700/44000 | Loss: 2.9478 | LR: 3.16e-04 | 1392.0ms/step | 14.4 samples/s +Step 18710/44000 | Loss: 2.9823 | LR: 3.16e-04 | 1400.2ms/step | 14.3 samples/s +Step 18720/44000 | Loss: 2.9899 | LR: 3.16e-04 | 1400.0ms/step | 14.3 samples/s +Step 18730/44000 | Loss: 2.9912 | LR: 3.15e-04 | 1391.0ms/step | 14.4 samples/s +Step 18740/44000 | Loss: 2.9927 | LR: 3.15e-04 | 1388.6ms/step | 14.4 samples/s +Step 18750/44000 | Loss: 2.9974 | LR: 3.15e-04 | 1385.7ms/step | 14.4 samples/s +Validation at step 18750: Loss = 2.9915, Perplexity = 19.92 +Step 18760/44000 | Loss: 2.9761 | LR: 3.15e-04 | 3600.8ms/step | 5.6 samples/s +Step 18770/44000 | Loss: 2.9774 | LR: 3.15e-04 | 1399.8ms/step | 14.3 samples/s +Step 18780/44000 | Loss: 2.9716 | LR: 3.15e-04 | 1398.2ms/step | 14.3 samples/s +Step 18790/44000 | Loss: 2.9680 | LR: 3.14e-04 | 1389.8ms/step | 14.4 samples/s +Step 18800/44000 | Loss: 2.9909 | LR: 3.14e-04 | 1396.5ms/step | 14.3 samples/s +Step 18810/44000 | Loss: 2.9829 | LR: 3.14e-04 | 1397.8ms/step | 14.3 samples/s +Step 18820/44000 | Loss: 2.9743 | LR: 3.14e-04 | 1400.0ms/step | 14.3 samples/s +Step 18830/44000 | Loss: 2.9631 | LR: 3.14e-04 | 1394.8ms/step | 14.3 samples/s +Step 18840/44000 | Loss: 2.9892 | LR: 3.13e-04 | 1399.1ms/step | 14.3 samples/s +Step 18850/44000 | Loss: 2.9470 | LR: 3.13e-04 | 1390.5ms/step | 14.4 samples/s +Step 18860/44000 | Loss: 2.9780 | LR: 3.13e-04 | 1389.1ms/step | 14.4 samples/s +Step 18870/44000 | Loss: 2.9910 | LR: 3.13e-04 | 1390.5ms/step | 14.4 samples/s +Step 18880/44000 | Loss: 2.9609 | LR: 3.13e-04 | 1391.3ms/step | 14.4 samples/s +Step 18890/44000 | Loss: 2.9782 | LR: 3.13e-04 | 1394.4ms/step | 14.3 samples/s +Step 18900/44000 | Loss: 2.9392 | LR: 3.12e-04 | 1394.4ms/step | 14.3 samples/s +Step 18910/44000 | Loss: 2.9795 | LR: 3.12e-04 | 1397.4ms/step | 14.3 samples/s +Step 18920/44000 | Loss: 2.9825 | LR: 3.12e-04 | 1398.5ms/step | 14.3 samples/s +Step 18930/44000 | Loss: 2.9792 | LR: 3.12e-04 | 1400.0ms/step | 14.3 samples/s +Step 18940/44000 | Loss: 2.9878 | LR: 3.12e-04 | 1398.0ms/step | 14.3 samples/s +Step 18950/44000 | Loss: 2.9477 | LR: 3.12e-04 | 1397.9ms/step | 14.3 samples/s +Step 18960/44000 | Loss: 2.9640 | LR: 3.11e-04 | 1395.9ms/step | 14.3 samples/s +Step 18970/44000 | Loss: 2.9825 | LR: 3.11e-04 | 1400.2ms/step | 14.3 samples/s +Step 18980/44000 | Loss: 2.9580 | LR: 3.11e-04 | 1398.1ms/step | 14.3 samples/s +Step 18990/44000 | Loss: 2.9751 | LR: 3.11e-04 | 1393.4ms/step | 14.4 samples/s +Step 19000/44000 | Loss: 2.9714 | LR: 3.11e-04 | 1393.8ms/step | 14.3 samples/s +Validation at step 19000: Loss = 2.9884, Perplexity = 19.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-19000 +Step 19010/44000 | Loss: 2.9437 | LR: 3.11e-04 | 3900.6ms/step | 5.1 samples/s +Step 19020/44000 | Loss: 2.9816 | LR: 3.10e-04 | 1390.8ms/step | 14.4 samples/s +Step 19030/44000 | Loss: 2.9606 | LR: 3.10e-04 | 1394.9ms/step | 14.3 samples/s +Step 19040/44000 | Loss: 2.9731 | LR: 3.10e-04 | 1394.3ms/step | 14.3 samples/s +Step 19050/44000 | Loss: 2.9893 | LR: 3.10e-04 | 1394.1ms/step | 14.3 samples/s +Step 19060/44000 | Loss: 3.0042 | LR: 3.10e-04 | 1395.8ms/step | 14.3 samples/s +Step 19070/44000 | Loss: 2.9794 | LR: 3.09e-04 | 1398.0ms/step | 14.3 samples/s +Step 19080/44000 | Loss: 2.9482 | LR: 3.09e-04 | 1394.1ms/step | 14.3 samples/s +Step 19090/44000 | Loss: 2.9668 | LR: 3.09e-04 | 1392.1ms/step | 14.4 samples/s +Step 19100/44000 | Loss: 2.9521 | LR: 3.09e-04 | 1396.2ms/step | 14.3 samples/s +Step 19110/44000 | Loss: 2.9823 | LR: 3.09e-04 | 1399.8ms/step | 14.3 samples/s +Step 19120/44000 | Loss: 2.9746 | LR: 3.09e-04 | 1397.8ms/step | 14.3 samples/s +Step 19130/44000 | Loss: 2.9889 | LR: 3.08e-04 | 1398.4ms/step | 14.3 samples/s +Step 19140/44000 | Loss: 2.9555 | LR: 3.08e-04 | 1393.8ms/step | 14.3 samples/s +Step 19150/44000 | Loss: 2.9673 | LR: 3.08e-04 | 1393.5ms/step | 14.4 samples/s +Step 19160/44000 | Loss: 2.9702 | LR: 3.08e-04 | 1392.6ms/step | 14.4 samples/s +Step 19170/44000 | Loss: 2.9318 | LR: 3.08e-04 | 1397.9ms/step | 14.3 samples/s +Step 19180/44000 | Loss: 2.9924 | LR: 3.08e-04 | 1399.4ms/step | 14.3 samples/s +Step 19190/44000 | Loss: 2.9455 | LR: 3.07e-04 | 1394.4ms/step | 14.3 samples/s +Step 19200/44000 | Loss: 2.9516 | LR: 3.07e-04 | 1398.2ms/step | 14.3 samples/s +Step 19210/44000 | Loss: 2.9786 | LR: 3.07e-04 | 1398.2ms/step | 14.3 samples/s +Step 19220/44000 | Loss: 2.9635 | LR: 3.07e-04 | 1400.3ms/step | 14.3 samples/s +Step 19230/44000 | Loss: 3.0266 | LR: 3.07e-04 | 1399.7ms/step | 14.3 samples/s +Step 19240/44000 | Loss: 2.9773 | LR: 3.06e-04 | 1400.1ms/step | 14.3 samples/s +Step 19250/44000 | Loss: 2.9623 | LR: 3.06e-04 | 1399.9ms/step | 14.3 samples/s +Validation at step 19250: Loss = 2.9863, Perplexity = 19.81 +Step 19260/44000 | Loss: 2.9730 | LR: 3.06e-04 | 3626.0ms/step | 5.5 samples/s +Step 19270/44000 | Loss: 2.9608 | LR: 3.06e-04 | 1400.0ms/step | 14.3 samples/s +Step 19280/44000 | Loss: 2.9482 | LR: 3.06e-04 | 1400.1ms/step | 14.3 samples/s +Step 19290/44000 | Loss: 2.9676 | LR: 3.06e-04 | 1400.1ms/step | 14.3 samples/s +Step 19300/44000 | Loss: 2.9579 | LR: 3.05e-04 | 1397.7ms/step | 14.3 samples/s +Step 19310/44000 | Loss: 2.9844 | LR: 3.05e-04 | 1393.8ms/step | 14.3 samples/s +Step 19320/44000 | Loss: 2.9828 | LR: 3.05e-04 | 1390.0ms/step | 14.4 samples/s +Step 19330/44000 | Loss: 2.9943 | LR: 3.05e-04 | 1396.4ms/step | 14.3 samples/s +Step 19340/44000 | Loss: 2.9869 | LR: 3.05e-04 | 1398.0ms/step | 14.3 samples/s +Step 19350/44000 | Loss: 2.9735 | LR: 3.04e-04 | 1399.8ms/step | 14.3 samples/s +Step 19360/44000 | Loss: 2.9869 | LR: 3.04e-04 | 1398.0ms/step | 14.3 samples/s +Step 19370/44000 | Loss: 2.9798 | LR: 3.04e-04 | 1390.2ms/step | 14.4 samples/s +Step 19380/44000 | Loss: 2.9449 | LR: 3.04e-04 | 1389.6ms/step | 14.4 samples/s +Step 19390/44000 | Loss: 2.9519 | LR: 3.04e-04 | 1388.4ms/step | 14.4 samples/s +Step 19400/44000 | Loss: 2.9748 | LR: 3.04e-04 | 1395.7ms/step | 14.3 samples/s +Step 19410/44000 | Loss: 2.9632 | LR: 3.03e-04 | 1396.1ms/step | 14.3 samples/s +Step 19420/44000 | Loss: 2.9795 | LR: 3.03e-04 | 1400.1ms/step | 14.3 samples/s +Step 19430/44000 | Loss: 2.9406 | LR: 3.03e-04 | 1398.8ms/step | 14.3 samples/s +Step 19440/44000 | Loss: 2.9918 | LR: 3.03e-04 | 1391.4ms/step | 14.4 samples/s +Step 19450/44000 | Loss: 2.9647 | LR: 3.03e-04 | 1390.0ms/step | 14.4 samples/s +Step 19460/44000 | Loss: 2.9776 | LR: 3.03e-04 | 1388.4ms/step | 14.4 samples/s +Step 19470/44000 | Loss: 2.9869 | LR: 3.02e-04 | 1388.6ms/step | 14.4 samples/s +Step 19480/44000 | Loss: 2.9434 | LR: 3.02e-04 | 1390.9ms/step | 14.4 samples/s +Step 19490/44000 | Loss: 2.9938 | LR: 3.02e-04 | 1393.2ms/step | 14.4 samples/s +Step 19500/44000 | Loss: 2.9553 | LR: 3.02e-04 | 1394.4ms/step | 14.3 samples/s +Validation at step 19500: Loss = 2.9820, Perplexity = 19.73 +Step 19510/44000 | Loss: 2.9783 | LR: 3.02e-04 | 3585.0ms/step | 5.6 samples/s +Step 19520/44000 | Loss: 2.9589 | LR: 3.01e-04 | 1389.4ms/step | 14.4 samples/s +Step 19530/44000 | Loss: 2.9367 | LR: 3.01e-04 | 1393.4ms/step | 14.4 samples/s +Step 19540/44000 | Loss: 2.9669 | LR: 3.01e-04 | 1394.2ms/step | 14.3 samples/s +Step 19550/44000 | Loss: 2.9570 | LR: 3.01e-04 | 1389.8ms/step | 14.4 samples/s +Step 19560/44000 | Loss: 2.9623 | LR: 3.01e-04 | 1388.0ms/step | 14.4 samples/s +Step 19570/44000 | Loss: 2.9761 | LR: 3.01e-04 | 1387.8ms/step | 14.4 samples/s +Step 19580/44000 | Loss: 2.9867 | LR: 3.00e-04 | 1390.6ms/step | 14.4 samples/s +Step 19590/44000 | Loss: 2.9710 | LR: 3.00e-04 | 1394.2ms/step | 14.3 samples/s +Step 19600/44000 | Loss: 2.9470 | LR: 3.00e-04 | 1396.3ms/step | 14.3 samples/s +Step 19610/44000 | Loss: 2.9670 | LR: 3.00e-04 | 1398.1ms/step | 14.3 samples/s +Step 19620/44000 | Loss: 2.9640 | LR: 3.00e-04 | 1393.9ms/step | 14.3 samples/s +Step 19630/44000 | Loss: 2.9898 | LR: 3.00e-04 | 1395.8ms/step | 14.3 samples/s +Step 19640/44000 | Loss: 2.9840 | LR: 2.99e-04 | 1399.9ms/step | 14.3 samples/s +Step 19650/44000 | Loss: 2.9538 | LR: 2.99e-04 | 1391.4ms/step | 14.4 samples/s +Step 19660/44000 | Loss: 2.9676 | LR: 2.99e-04 | 1391.0ms/step | 14.4 samples/s +Step 19670/44000 | Loss: 2.9785 | LR: 2.99e-04 | 1393.7ms/step | 14.3 samples/s +Step 19680/44000 | Loss: 2.9905 | LR: 2.99e-04 | 1393.7ms/step | 14.4 samples/s +Step 19690/44000 | Loss: 2.9515 | LR: 2.98e-04 | 1391.4ms/step | 14.4 samples/s +Step 19700/44000 | Loss: 2.9848 | LR: 2.98e-04 | 1393.0ms/step | 14.4 samples/s +Step 19710/44000 | Loss: 2.9794 | LR: 2.98e-04 | 1396.3ms/step | 14.3 samples/s +Step 19720/44000 | Loss: 2.9684 | LR: 2.98e-04 | 1400.1ms/step | 14.3 samples/s +Step 19730/44000 | Loss: 2.9931 | LR: 2.98e-04 | 1397.9ms/step | 14.3 samples/s +Step 19740/44000 | Loss: 2.9659 | LR: 2.98e-04 | 1391.6ms/step | 14.4 samples/s +Step 19750/44000 | Loss: 2.9473 | LR: 2.97e-04 | 1392.0ms/step | 14.4 samples/s +Validation at step 19750: Loss = 2.9801, Perplexity = 19.69 +Step 19760/44000 | Loss: 2.9829 | LR: 2.97e-04 | 3594.2ms/step | 5.6 samples/s +Step 19770/44000 | Loss: 2.9893 | LR: 2.97e-04 | 1394.1ms/step | 14.3 samples/s +Step 19780/44000 | Loss: 2.9849 | LR: 2.97e-04 | 1397.7ms/step | 14.3 samples/s +Step 19790/44000 | Loss: 2.9636 | LR: 2.97e-04 | 1398.1ms/step | 14.3 samples/s +Step 19800/44000 | Loss: 2.9906 | LR: 2.96e-04 | 1400.4ms/step | 14.3 samples/s +Step 19810/44000 | Loss: 2.9506 | LR: 2.96e-04 | 1399.7ms/step | 14.3 samples/s +Step 19820/44000 | Loss: 2.9403 | LR: 2.96e-04 | 1395.7ms/step | 14.3 samples/s +Step 19830/44000 | Loss: 2.9894 | LR: 2.96e-04 | 1398.5ms/step | 14.3 samples/s +Step 19840/44000 | Loss: 2.9627 | LR: 2.96e-04 | 1397.8ms/step | 14.3 samples/s +Step 19850/44000 | Loss: 2.9622 | LR: 2.96e-04 | 1400.3ms/step | 14.3 samples/s +Step 19860/44000 | Loss: 2.9920 | LR: 2.95e-04 | 1400.1ms/step | 14.3 samples/s +Step 19870/44000 | Loss: 2.9863 | LR: 2.95e-04 | 1399.5ms/step | 14.3 samples/s +Step 19880/44000 | Loss: 2.9740 | LR: 2.95e-04 | 1398.3ms/step | 14.3 samples/s +Step 19890/44000 | Loss: 2.9745 | LR: 2.95e-04 | 1397.9ms/step | 14.3 samples/s +Step 19900/44000 | Loss: 2.9599 | LR: 2.95e-04 | 1398.1ms/step | 14.3 samples/s +Step 19910/44000 | Loss: 2.9706 | LR: 2.95e-04 | 1394.9ms/step | 14.3 samples/s +Step 19920/44000 | Loss: 2.9625 | LR: 2.94e-04 | 1397.0ms/step | 14.3 samples/s +Step 19930/44000 | Loss: 2.9743 | LR: 2.94e-04 | 1395.7ms/step | 14.3 samples/s +Step 19940/44000 | Loss: 2.9382 | LR: 2.94e-04 | 1391.3ms/step | 14.4 samples/s +Step 19950/44000 | Loss: 2.9571 | LR: 2.94e-04 | 1392.7ms/step | 14.4 samples/s +Step 19960/44000 | Loss: 2.9977 | LR: 2.94e-04 | 1389.8ms/step | 14.4 samples/s +Step 19970/44000 | Loss: 2.9853 | LR: 2.93e-04 | 1390.7ms/step | 14.4 samples/s +Step 19980/44000 | Loss: 2.9818 | LR: 2.93e-04 | 1393.2ms/step | 14.4 samples/s +Step 19990/44000 | Loss: 2.9719 | LR: 2.93e-04 | 1390.0ms/step | 14.4 samples/s +Step 20000/44000 | Loss: 2.9769 | LR: 2.93e-04 | 1392.3ms/step | 14.4 samples/s +Validation at step 20000: Loss = 2.9774, Perplexity = 19.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-20000 +Step 20010/44000 | Loss: 2.9540 | LR: 2.93e-04 | 3920.1ms/step | 5.1 samples/s +Step 20020/44000 | Loss: 2.9765 | LR: 2.93e-04 | 1390.1ms/step | 14.4 samples/s +Step 20030/44000 | Loss: 2.9502 | LR: 2.92e-04 | 1389.9ms/step | 14.4 samples/s +Step 20040/44000 | Loss: 2.9529 | LR: 2.92e-04 | 1392.0ms/step | 14.4 samples/s +Step 20050/44000 | Loss: 2.9692 | LR: 2.92e-04 | 1392.8ms/step | 14.4 samples/s +Step 20060/44000 | Loss: 2.9366 | LR: 2.92e-04 | 1390.0ms/step | 14.4 samples/s +Step 20070/44000 | Loss: 2.9628 | LR: 2.92e-04 | 1392.7ms/step | 14.4 samples/s +Step 20080/44000 | Loss: 2.9448 | LR: 2.91e-04 | 1389.8ms/step | 14.4 samples/s +Step 20090/44000 | Loss: 2.9555 | LR: 2.91e-04 | 1396.6ms/step | 14.3 samples/s +Step 20100/44000 | Loss: 2.9724 | LR: 2.91e-04 | 1400.3ms/step | 14.3 samples/s +Step 20110/44000 | Loss: 2.9636 | LR: 2.91e-04 | 1395.5ms/step | 14.3 samples/s +Step 20120/44000 | Loss: 2.9744 | LR: 2.91e-04 | 1388.4ms/step | 14.4 samples/s +Step 20130/44000 | Loss: 2.9707 | LR: 2.91e-04 | 1393.7ms/step | 14.3 samples/s +Step 20140/44000 | Loss: 2.9720 | LR: 2.90e-04 | 1392.2ms/step | 14.4 samples/s +Step 20150/44000 | Loss: 2.9600 | LR: 2.90e-04 | 1394.1ms/step | 14.3 samples/s +Step 20160/44000 | Loss: 2.9741 | LR: 2.90e-04 | 1398.1ms/step | 14.3 samples/s +Step 20170/44000 | Loss: 2.9300 | LR: 2.90e-04 | 1399.9ms/step | 14.3 samples/s +Step 20180/44000 | Loss: 2.9495 | LR: 2.90e-04 | 1395.7ms/step | 14.3 samples/s +Step 20190/44000 | Loss: 2.9808 | LR: 2.90e-04 | 1390.3ms/step | 14.4 samples/s +Step 20200/44000 | Loss: 2.9388 | LR: 2.89e-04 | 1391.2ms/step | 14.4 samples/s +Step 20210/44000 | Loss: 2.9297 | LR: 2.89e-04 | 1390.7ms/step | 14.4 samples/s +Step 20220/44000 | Loss: 2.9815 | LR: 2.89e-04 | 1394.2ms/step | 14.3 samples/s +Step 20230/44000 | Loss: 2.9681 | LR: 2.89e-04 | 1390.5ms/step | 14.4 samples/s +Step 20240/44000 | Loss: 2.9693 | LR: 2.89e-04 | 1395.7ms/step | 14.3 samples/s +Step 20250/44000 | Loss: 2.9537 | LR: 2.88e-04 | 1395.9ms/step | 14.3 samples/s +Validation at step 20250: Loss = 2.9754, Perplexity = 19.60 +Step 20260/44000 | Loss: 2.9221 | LR: 2.88e-04 | 3605.8ms/step | 5.5 samples/s +Step 20270/44000 | Loss: 2.9743 | LR: 2.88e-04 | 1396.1ms/step | 14.3 samples/s +Step 20280/44000 | Loss: 2.9686 | LR: 2.88e-04 | 1400.0ms/step | 14.3 samples/s +Step 20290/44000 | Loss: 2.9617 | LR: 2.88e-04 | 1399.9ms/step | 14.3 samples/s +Step 20300/44000 | Loss: 2.9852 | LR: 2.88e-04 | 1394.0ms/step | 14.3 samples/s +Step 20310/44000 | Loss: 2.9509 | LR: 2.87e-04 | 1396.3ms/step | 14.3 samples/s +Step 20320/44000 | Loss: 2.9594 | LR: 2.87e-04 | 1399.8ms/step | 14.3 samples/s +Step 20330/44000 | Loss: 2.9500 | LR: 2.87e-04 | 1391.6ms/step | 14.4 samples/s +Step 20340/44000 | Loss: 2.9391 | LR: 2.87e-04 | 1388.5ms/step | 14.4 samples/s +Step 20350/44000 | Loss: 2.9663 | LR: 2.87e-04 | 1388.9ms/step | 14.4 samples/s +Step 20360/44000 | Loss: 2.9794 | LR: 2.86e-04 | 1391.2ms/step | 14.4 samples/s +Step 20370/44000 | Loss: 2.9334 | LR: 2.86e-04 | 1395.9ms/step | 14.3 samples/s +Step 20380/44000 | Loss: 2.9430 | LR: 2.86e-04 | 1395.6ms/step | 14.3 samples/s +Step 20390/44000 | Loss: 2.9606 | LR: 2.86e-04 | 1390.4ms/step | 14.4 samples/s +Step 20400/44000 | Loss: 2.9758 | LR: 2.86e-04 | 1389.5ms/step | 14.4 samples/s +Step 20410/44000 | Loss: 2.9510 | LR: 2.86e-04 | 1392.6ms/step | 14.4 samples/s +Step 20420/44000 | Loss: 2.9748 | LR: 2.85e-04 | 1394.1ms/step | 14.3 samples/s +Step 20430/44000 | Loss: 2.9379 | LR: 2.85e-04 | 1392.1ms/step | 14.4 samples/s +Step 20440/44000 | Loss: 2.9687 | LR: 2.85e-04 | 1397.7ms/step | 14.3 samples/s +Step 20450/44000 | Loss: 2.9585 | LR: 2.85e-04 | 1398.4ms/step | 14.3 samples/s +Step 20460/44000 | Loss: 2.9625 | LR: 2.85e-04 | 1400.1ms/step | 14.3 samples/s +Step 20470/44000 | Loss: 2.9794 | LR: 2.84e-04 | 1399.5ms/step | 14.3 samples/s +Step 20480/44000 | Loss: 2.9563 | LR: 2.84e-04 | 1400.4ms/step | 14.3 samples/s +Step 20490/44000 | Loss: 2.9578 | LR: 2.84e-04 | 1400.0ms/step | 14.3 samples/s +Step 20500/44000 | Loss: 2.9375 | LR: 2.84e-04 | 1400.1ms/step | 14.3 samples/s +Validation at step 20500: Loss = 2.9718, Perplexity = 19.53 +Step 20510/44000 | Loss: 2.9845 | LR: 2.84e-04 | 3623.6ms/step | 5.5 samples/s +Step 20520/44000 | Loss: 2.9389 | LR: 2.84e-04 | 1400.2ms/step | 14.3 samples/s +Step 20530/44000 | Loss: 2.9304 | LR: 2.83e-04 | 1400.1ms/step | 14.3 samples/s +Step 20540/44000 | Loss: 2.9582 | LR: 2.83e-04 | 1400.0ms/step | 14.3 samples/s +Step 20550/44000 | Loss: 2.9715 | LR: 2.83e-04 | 1400.1ms/step | 14.3 samples/s +Step 20560/44000 | Loss: 2.9753 | LR: 2.83e-04 | 1399.9ms/step | 14.3 samples/s +Step 20570/44000 | Loss: 2.9695 | LR: 2.83e-04 | 1400.0ms/step | 14.3 samples/s +Step 20580/44000 | Loss: 2.9405 | LR: 2.83e-04 | 1399.6ms/step | 14.3 samples/s +Step 20590/44000 | Loss: 2.9807 | LR: 2.82e-04 | 1400.2ms/step | 14.3 samples/s +Step 20600/44000 | Loss: 2.9576 | LR: 2.82e-04 | 1396.7ms/step | 14.3 samples/s +Step 20610/44000 | Loss: 2.9647 | LR: 2.82e-04 | 1390.6ms/step | 14.4 samples/s +Step 20620/44000 | Loss: 2.9583 | LR: 2.82e-04 | 1390.5ms/step | 14.4 samples/s +Step 20630/44000 | Loss: 2.9674 | LR: 2.82e-04 | 1392.0ms/step | 14.4 samples/s +Step 20640/44000 | Loss: 2.9259 | LR: 2.81e-04 | 1393.2ms/step | 14.4 samples/s +Step 20650/44000 | Loss: 2.9726 | LR: 2.81e-04 | 1394.9ms/step | 14.3 samples/s +Step 20660/44000 | Loss: 2.9261 | LR: 2.81e-04 | 1393.3ms/step | 14.4 samples/s +Step 20670/44000 | Loss: 2.9732 | LR: 2.81e-04 | 1392.5ms/step | 14.4 samples/s +Step 20680/44000 | Loss: 2.9630 | LR: 2.81e-04 | 1390.2ms/step | 14.4 samples/s +Step 20690/44000 | Loss: 2.9510 | LR: 2.81e-04 | 1391.8ms/step | 14.4 samples/s +Step 20700/44000 | Loss: 2.9622 | LR: 2.80e-04 | 1388.3ms/step | 14.4 samples/s +Step 20710/44000 | Loss: 2.9689 | LR: 2.80e-04 | 1393.9ms/step | 14.3 samples/s +Step 20720/44000 | Loss: 2.9690 | LR: 2.80e-04 | 1393.3ms/step | 14.4 samples/s +Step 20730/44000 | Loss: 2.9839 | LR: 2.80e-04 | 1392.8ms/step | 14.4 samples/s +Step 20740/44000 | Loss: 2.9673 | LR: 2.80e-04 | 1391.8ms/step | 14.4 samples/s +Step 20750/44000 | Loss: 2.9628 | LR: 2.79e-04 | 1391.7ms/step | 14.4 samples/s +Validation at step 20750: Loss = 2.9694, Perplexity = 19.48 +Step 20760/44000 | Loss: 2.9677 | LR: 2.79e-04 | 3591.6ms/step | 5.6 samples/s +Step 20770/44000 | Loss: 2.9512 | LR: 2.79e-04 | 1396.2ms/step | 14.3 samples/s +Step 20780/44000 | Loss: 2.9293 | LR: 2.79e-04 | 1395.0ms/step | 14.3 samples/s +Step 20790/44000 | Loss: 2.9488 | LR: 2.79e-04 | 1400.1ms/step | 14.3 samples/s +Step 20800/44000 | Loss: 2.9482 | LR: 2.79e-04 | 1399.7ms/step | 14.3 samples/s +Step 20810/44000 | Loss: 2.9665 | LR: 2.78e-04 | 1397.8ms/step | 14.3 samples/s +Step 20820/44000 | Loss: 2.9623 | LR: 2.78e-04 | 1398.2ms/step | 14.3 samples/s +Step 20830/44000 | Loss: 2.9656 | LR: 2.78e-04 | 1393.2ms/step | 14.4 samples/s +Step 20840/44000 | Loss: 2.9523 | LR: 2.78e-04 | 1394.7ms/step | 14.3 samples/s +Step 20850/44000 | Loss: 2.9547 | LR: 2.78e-04 | 1396.2ms/step | 14.3 samples/s +Step 20860/44000 | Loss: 2.9741 | LR: 2.77e-04 | 1394.1ms/step | 14.3 samples/s +Step 20870/44000 | Loss: 2.9786 | LR: 2.77e-04 | 1397.8ms/step | 14.3 samples/s +Step 20880/44000 | Loss: 2.9488 | LR: 2.77e-04 | 1400.1ms/step | 14.3 samples/s +Step 20890/44000 | Loss: 2.9347 | LR: 2.77e-04 | 1398.1ms/step | 14.3 samples/s +Step 20900/44000 | Loss: 2.9357 | LR: 2.77e-04 | 1400.0ms/step | 14.3 samples/s +Step 20910/44000 | Loss: 2.9683 | LR: 2.77e-04 | 1398.2ms/step | 14.3 samples/s +Step 20920/44000 | Loss: 2.9299 | LR: 2.76e-04 | 1400.0ms/step | 14.3 samples/s +Step 20930/44000 | Loss: 2.9410 | LR: 2.76e-04 | 1399.8ms/step | 14.3 samples/s +Step 20940/44000 | Loss: 2.9638 | LR: 2.76e-04 | 1400.2ms/step | 14.3 samples/s +Step 20950/44000 | Loss: 2.9286 | LR: 2.76e-04 | 1397.8ms/step | 14.3 samples/s +Step 20960/44000 | Loss: 2.9448 | LR: 2.76e-04 | 1394.5ms/step | 14.3 samples/s +Step 20970/44000 | Loss: 2.9279 | LR: 2.75e-04 | 1393.6ms/step | 14.4 samples/s +Step 20980/44000 | Loss: 2.9320 | LR: 2.75e-04 | 1393.5ms/step | 14.4 samples/s +Step 20990/44000 | Loss: 2.9407 | LR: 2.75e-04 | 1390.4ms/step | 14.4 samples/s +Step 21000/44000 | Loss: 2.9398 | LR: 2.75e-04 | 1387.9ms/step | 14.4 samples/s +Validation at step 21000: Loss = 2.9677, Perplexity = 19.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-21000 +Step 21010/44000 | Loss: 2.9468 | LR: 2.75e-04 | 3888.6ms/step | 5.1 samples/s +Step 21020/44000 | Loss: 2.9464 | LR: 2.75e-04 | 1389.7ms/step | 14.4 samples/s +Step 21030/44000 | Loss: 2.9853 | LR: 2.74e-04 | 1398.2ms/step | 14.3 samples/s +Step 21040/44000 | Loss: 2.9289 | LR: 2.74e-04 | 1393.6ms/step | 14.4 samples/s +Step 21050/44000 | Loss: 2.9472 | LR: 2.74e-04 | 1398.3ms/step | 14.3 samples/s +Step 21060/44000 | Loss: 2.9330 | LR: 2.74e-04 | 1399.8ms/step | 14.3 samples/s +Step 21070/44000 | Loss: 2.9755 | LR: 2.74e-04 | 1400.1ms/step | 14.3 samples/s +Step 21080/44000 | Loss: 2.9209 | LR: 2.73e-04 | 1400.0ms/step | 14.3 samples/s +Step 21090/44000 | Loss: 2.9792 | LR: 2.73e-04 | 1400.2ms/step | 14.3 samples/s +Step 21100/44000 | Loss: 2.9631 | LR: 2.73e-04 | 1399.9ms/step | 14.3 samples/s +Step 21110/44000 | Loss: 2.9523 | LR: 2.73e-04 | 1400.0ms/step | 14.3 samples/s +Step 21120/44000 | Loss: 2.9642 | LR: 2.73e-04 | 1400.0ms/step | 14.3 samples/s +Step 21130/44000 | Loss: 2.9859 | LR: 2.73e-04 | 1398.1ms/step | 14.3 samples/s +Step 21140/44000 | Loss: 2.9461 | LR: 2.72e-04 | 1399.9ms/step | 14.3 samples/s +Step 21150/44000 | Loss: 2.9563 | LR: 2.72e-04 | 1396.1ms/step | 14.3 samples/s +Step 21160/44000 | Loss: 2.9385 | LR: 2.72e-04 | 1395.8ms/step | 14.3 samples/s +Step 21170/44000 | Loss: 2.9519 | LR: 2.72e-04 | 1393.6ms/step | 14.4 samples/s +Step 21180/44000 | Loss: 2.9605 | LR: 2.72e-04 | 1390.7ms/step | 14.4 samples/s +Step 21190/44000 | Loss: 2.9020 | LR: 2.71e-04 | 1391.9ms/step | 14.4 samples/s +Step 21200/44000 | Loss: 2.9714 | LR: 2.71e-04 | 1390.0ms/step | 14.4 samples/s +Step 21210/44000 | Loss: 2.9531 | LR: 2.71e-04 | 1393.6ms/step | 14.4 samples/s +Step 21220/44000 | Loss: 2.9431 | LR: 2.71e-04 | 1390.3ms/step | 14.4 samples/s +Step 21230/44000 | Loss: 2.9558 | LR: 2.71e-04 | 1388.2ms/step | 14.4 samples/s +Step 21240/44000 | Loss: 2.9459 | LR: 2.71e-04 | 1391.5ms/step | 14.4 samples/s +Step 21250/44000 | Loss: 2.9393 | LR: 2.70e-04 | 1388.4ms/step | 14.4 samples/s +Validation at step 21250: Loss = 2.9644, Perplexity = 19.38 +Step 21260/44000 | Loss: 2.9816 | LR: 2.70e-04 | 3589.0ms/step | 5.6 samples/s +Step 21270/44000 | Loss: 2.9568 | LR: 2.70e-04 | 1385.8ms/step | 14.4 samples/s +Step 21280/44000 | Loss: 2.9482 | LR: 2.70e-04 | 1388.5ms/step | 14.4 samples/s +Step 21290/44000 | Loss: 2.9438 | LR: 2.70e-04 | 1388.6ms/step | 14.4 samples/s +Step 21300/44000 | Loss: 2.9542 | LR: 2.70e-04 | 1392.3ms/step | 14.4 samples/s +Step 21310/44000 | Loss: 2.9553 | LR: 2.69e-04 | 1393.2ms/step | 14.4 samples/s +Step 21320/44000 | Loss: 2.9664 | LR: 2.69e-04 | 1392.9ms/step | 14.4 samples/s +Step 21330/44000 | Loss: 2.9506 | LR: 2.69e-04 | 1397.1ms/step | 14.3 samples/s +Step 21340/44000 | Loss: 2.9450 | LR: 2.69e-04 | 1392.9ms/step | 14.4 samples/s +Step 21350/44000 | Loss: 2.9781 | LR: 2.69e-04 | 1392.0ms/step | 14.4 samples/s +Step 21360/44000 | Loss: 2.9493 | LR: 2.68e-04 | 1394.0ms/step | 14.3 samples/s +Step 21370/44000 | Loss: 2.9524 | LR: 2.68e-04 | 1396.0ms/step | 14.3 samples/s +Step 21380/44000 | Loss: 2.9407 | LR: 2.68e-04 | 1391.6ms/step | 14.4 samples/s +Step 21390/44000 | Loss: 2.9704 | LR: 2.68e-04 | 1390.5ms/step | 14.4 samples/s +Step 21400/44000 | Loss: 2.9320 | LR: 2.68e-04 | 1386.7ms/step | 14.4 samples/s +Step 21410/44000 | Loss: 2.9729 | LR: 2.68e-04 | 1391.3ms/step | 14.4 samples/s +Step 21420/44000 | Loss: 2.9216 | LR: 2.67e-04 | 1395.8ms/step | 14.3 samples/s +Step 21430/44000 | Loss: 2.9377 | LR: 2.67e-04 | 1396.2ms/step | 14.3 samples/s +Step 21440/44000 | Loss: 2.9305 | LR: 2.67e-04 | 1393.9ms/step | 14.3 samples/s +Step 21450/44000 | Loss: 2.9469 | LR: 2.67e-04 | 1396.1ms/step | 14.3 samples/s +Step 21460/44000 | Loss: 2.9535 | LR: 2.67e-04 | 1397.9ms/step | 14.3 samples/s +Step 21470/44000 | Loss: 2.9576 | LR: 2.66e-04 | 1397.9ms/step | 14.3 samples/s +Step 21480/44000 | Loss: 2.9380 | LR: 2.66e-04 | 1397.4ms/step | 14.3 samples/s +Step 21490/44000 | Loss: 2.9545 | LR: 2.66e-04 | 1394.6ms/step | 14.3 samples/s +Step 21500/44000 | Loss: 2.9526 | LR: 2.66e-04 | 1396.2ms/step | 14.3 samples/s +Validation at step 21500: Loss = 2.9619, Perplexity = 19.33 +Step 21510/44000 | Loss: 2.9565 | LR: 2.66e-04 | 3595.9ms/step | 5.6 samples/s +Step 21520/44000 | Loss: 2.9422 | LR: 2.66e-04 | 1392.2ms/step | 14.4 samples/s +Step 21530/44000 | Loss: 2.9399 | LR: 2.65e-04 | 1390.0ms/step | 14.4 samples/s +Step 21540/44000 | Loss: 2.9500 | LR: 2.65e-04 | 1389.8ms/step | 14.4 samples/s +Step 21550/44000 | Loss: 2.9504 | LR: 2.65e-04 | 1390.7ms/step | 14.4 samples/s +Step 21560/44000 | Loss: 2.9540 | LR: 2.65e-04 | 1397.1ms/step | 14.3 samples/s +Step 21570/44000 | Loss: 2.9337 | LR: 2.65e-04 | 1391.9ms/step | 14.4 samples/s +Step 21580/44000 | Loss: 2.9266 | LR: 2.64e-04 | 1391.2ms/step | 14.4 samples/s +Step 21590/44000 | Loss: 2.9423 | LR: 2.64e-04 | 1395.3ms/step | 14.3 samples/s +Step 21600/44000 | Loss: 2.9430 | LR: 2.64e-04 | 1396.0ms/step | 14.3 samples/s +Step 21610/44000 | Loss: 2.9397 | LR: 2.64e-04 | 1395.9ms/step | 14.3 samples/s +Step 21620/44000 | Loss: 2.9614 | LR: 2.64e-04 | 1396.9ms/step | 14.3 samples/s +Step 21630/44000 | Loss: 2.9315 | LR: 2.64e-04 | 1395.3ms/step | 14.3 samples/s +Step 21640/44000 | Loss: 2.9222 | LR: 2.63e-04 | 1394.0ms/step | 14.3 samples/s +Step 21650/44000 | Loss: 2.9494 | LR: 2.63e-04 | 1394.9ms/step | 14.3 samples/s +Step 21660/44000 | Loss: 2.9854 | LR: 2.63e-04 | 1393.0ms/step | 14.4 samples/s +Step 21670/44000 | Loss: 2.9301 | LR: 2.63e-04 | 1391.5ms/step | 14.4 samples/s +Step 21680/44000 | Loss: 2.9531 | LR: 2.63e-04 | 1390.4ms/step | 14.4 samples/s +Step 21690/44000 | Loss: 2.9822 | LR: 2.62e-04 | 1393.8ms/step | 14.3 samples/s +Step 21700/44000 | Loss: 2.9649 | LR: 2.62e-04 | 1396.3ms/step | 14.3 samples/s +Step 21710/44000 | Loss: 2.9467 | LR: 2.62e-04 | 1398.3ms/step | 14.3 samples/s +Step 21720/44000 | Loss: 2.9198 | LR: 2.62e-04 | 1399.8ms/step | 14.3 samples/s +Step 21730/44000 | Loss: 2.9433 | LR: 2.62e-04 | 1400.1ms/step | 14.3 samples/s +Step 21740/44000 | Loss: 2.9343 | LR: 2.62e-04 | 1400.0ms/step | 14.3 samples/s +Step 21750/44000 | Loss: 2.9386 | LR: 2.61e-04 | 1400.0ms/step | 14.3 samples/s +Validation at step 21750: Loss = 2.9593, Perplexity = 19.28 +Step 21760/44000 | Loss: 2.9590 | LR: 2.61e-04 | 3614.1ms/step | 5.5 samples/s +Step 21770/44000 | Loss: 2.9442 | LR: 2.61e-04 | 1400.1ms/step | 14.3 samples/s +Step 21780/44000 | Loss: 2.9456 | LR: 2.61e-04 | 1399.8ms/step | 14.3 samples/s +Step 21790/44000 | Loss: 2.9097 | LR: 2.61e-04 | 1400.1ms/step | 14.3 samples/s +Step 21800/44000 | Loss: 2.9366 | LR: 2.60e-04 | 1399.9ms/step | 14.3 samples/s +Step 21810/44000 | Loss: 2.9333 | LR: 2.60e-04 | 1399.9ms/step | 14.3 samples/s +Step 21820/44000 | Loss: 2.9535 | LR: 2.60e-04 | 1400.0ms/step | 14.3 samples/s +Step 21830/44000 | Loss: 2.9225 | LR: 2.60e-04 | 1395.8ms/step | 14.3 samples/s +Step 21840/44000 | Loss: 2.9482 | LR: 2.60e-04 | 1389.3ms/step | 14.4 samples/s +Step 21850/44000 | Loss: 2.9447 | LR: 2.60e-04 | 1390.7ms/step | 14.4 samples/s +Step 21860/44000 | Loss: 2.9474 | LR: 2.59e-04 | 1390.0ms/step | 14.4 samples/s +Step 21870/44000 | Loss: 2.9598 | LR: 2.59e-04 | 1396.2ms/step | 14.3 samples/s +Step 21880/44000 | Loss: 2.9282 | LR: 2.59e-04 | 1398.1ms/step | 14.3 samples/s +Step 21890/44000 | Loss: 2.9321 | LR: 2.59e-04 | 1398.0ms/step | 14.3 samples/s +Step 21900/44000 | Loss: 2.9397 | LR: 2.59e-04 | 1400.1ms/step | 14.3 samples/s +Step 21910/44000 | Loss: 2.9389 | LR: 2.58e-04 | 1397.8ms/step | 14.3 samples/s +Step 21920/44000 | Loss: 2.9480 | LR: 2.58e-04 | 1392.4ms/step | 14.4 samples/s +Step 21930/44000 | Loss: 2.9730 | LR: 2.58e-04 | 1389.4ms/step | 14.4 samples/s +Step 21940/44000 | Loss: 2.9143 | LR: 2.58e-04 | 1396.1ms/step | 14.3 samples/s +Step 21950/44000 | Loss: 2.9457 | LR: 2.58e-04 | 1396.1ms/step | 14.3 samples/s +Step 21960/44000 | Loss: 2.9353 | LR: 2.58e-04 | 1394.0ms/step | 14.3 samples/s +Step 21970/44000 | Loss: 2.9303 | LR: 2.57e-04 | 1390.1ms/step | 14.4 samples/s +Step 21980/44000 | Loss: 2.9511 | LR: 2.57e-04 | 1389.9ms/step | 14.4 samples/s +Step 21990/44000 | Loss: 2.9546 | LR: 2.57e-04 | 1390.3ms/step | 14.4 samples/s +Step 22000/44000 | Loss: 2.9424 | LR: 2.57e-04 | 1391.6ms/step | 14.4 samples/s +Validation at step 22000: Loss = 2.9569, Perplexity = 19.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-22000 +Step 22010/44000 | Loss: 2.9438 | LR: 2.57e-04 | 3912.1ms/step | 5.1 samples/s +Step 22020/44000 | Loss: 2.9575 | LR: 2.56e-04 | 1394.4ms/step | 14.3 samples/s +Step 22030/44000 | Loss: 2.9516 | LR: 2.56e-04 | 1397.7ms/step | 14.3 samples/s +Step 22040/44000 | Loss: 2.9668 | LR: 2.56e-04 | 1395.2ms/step | 14.3 samples/s +Step 22050/44000 | Loss: 2.9373 | LR: 2.56e-04 | 1392.9ms/step | 14.4 samples/s +Step 22060/44000 | Loss: 2.9369 | LR: 2.56e-04 | 1389.2ms/step | 14.4 samples/s +Step 22070/44000 | Loss: 2.9411 | LR: 2.56e-04 | 1388.9ms/step | 14.4 samples/s +Step 22080/44000 | Loss: 2.9324 | LR: 2.55e-04 | 1392.3ms/step | 14.4 samples/s +Step 22090/44000 | Loss: 2.9621 | LR: 2.55e-04 | 1396.4ms/step | 14.3 samples/s +Step 22100/44000 | Loss: 2.9548 | LR: 2.55e-04 | 1397.4ms/step | 14.3 samples/s +Step 22110/44000 | Loss: 2.9386 | LR: 2.55e-04 | 1398.8ms/step | 14.3 samples/s +Step 22120/44000 | Loss: 2.9207 | LR: 2.55e-04 | 1393.8ms/step | 14.3 samples/s +Step 22130/44000 | Loss: 2.9307 | LR: 2.54e-04 | 1392.4ms/step | 14.4 samples/s +Step 22140/44000 | Loss: 2.9402 | LR: 2.54e-04 | 1398.7ms/step | 14.3 samples/s +Step 22150/44000 | Loss: 2.9206 | LR: 2.54e-04 | 1394.3ms/step | 14.3 samples/s +Step 22160/44000 | Loss: 2.9429 | LR: 2.54e-04 | 1394.2ms/step | 14.3 samples/s +Step 22170/44000 | Loss: 2.9597 | LR: 2.54e-04 | 1394.5ms/step | 14.3 samples/s +Step 22180/44000 | Loss: 2.9418 | LR: 2.54e-04 | 1397.4ms/step | 14.3 samples/s +Step 22190/44000 | Loss: 2.9422 | LR: 2.53e-04 | 1399.8ms/step | 14.3 samples/s +Step 22200/44000 | Loss: 2.9394 | LR: 2.53e-04 | 1396.0ms/step | 14.3 samples/s +Step 22210/44000 | Loss: 2.9431 | LR: 2.53e-04 | 1394.0ms/step | 14.3 samples/s +Step 22220/44000 | Loss: 2.9398 | LR: 2.53e-04 | 1392.2ms/step | 14.4 samples/s +Step 22230/44000 | Loss: 2.9574 | LR: 2.53e-04 | 1390.3ms/step | 14.4 samples/s +Step 22240/44000 | Loss: 2.9689 | LR: 2.52e-04 | 1392.7ms/step | 14.4 samples/s +Step 22250/44000 | Loss: 2.9551 | LR: 2.52e-04 | 1392.5ms/step | 14.4 samples/s +Validation at step 22250: Loss = 2.9548, Perplexity = 19.20 +Step 22260/44000 | Loss: 2.9236 | LR: 2.52e-04 | 3602.4ms/step | 5.6 samples/s +Step 22270/44000 | Loss: 2.9583 | LR: 2.52e-04 | 1392.9ms/step | 14.4 samples/s +Step 22280/44000 | Loss: 2.9370 | LR: 2.52e-04 | 1387.5ms/step | 14.4 samples/s +Step 22290/44000 | Loss: 2.9366 | LR: 2.52e-04 | 1387.6ms/step | 14.4 samples/s +Step 22300/44000 | Loss: 2.9729 | LR: 2.51e-04 | 1391.2ms/step | 14.4 samples/s +Step 22310/44000 | Loss: 2.9336 | LR: 2.51e-04 | 1387.5ms/step | 14.4 samples/s +Step 22320/44000 | Loss: 2.9187 | LR: 2.51e-04 | 1391.4ms/step | 14.4 samples/s +Step 22330/44000 | Loss: 2.9219 | LR: 2.51e-04 | 1394.1ms/step | 14.3 samples/s +Step 22340/44000 | Loss: 2.9698 | LR: 2.51e-04 | 1395.8ms/step | 14.3 samples/s +Step 22350/44000 | Loss: 2.9325 | LR: 2.50e-04 | 1392.8ms/step | 14.4 samples/s +Step 22360/44000 | Loss: 2.9336 | LR: 2.50e-04 | 1393.1ms/step | 14.4 samples/s +Step 22370/44000 | Loss: 2.9374 | LR: 2.50e-04 | 1396.6ms/step | 14.3 samples/s +Step 22380/44000 | Loss: 2.9399 | LR: 2.50e-04 | 1395.8ms/step | 14.3 samples/s +Step 22390/44000 | Loss: 2.9272 | LR: 2.50e-04 | 1393.6ms/step | 14.4 samples/s +Step 22400/44000 | Loss: 2.9124 | LR: 2.50e-04 | 1392.4ms/step | 14.4 samples/s +Step 22410/44000 | Loss: 2.9298 | LR: 2.49e-04 | 1396.3ms/step | 14.3 samples/s +Step 22420/44000 | Loss: 2.9321 | LR: 2.49e-04 | 1397.9ms/step | 14.3 samples/s +Step 22430/44000 | Loss: 2.9473 | LR: 2.49e-04 | 1400.2ms/step | 14.3 samples/s +Step 22440/44000 | Loss: 2.9359 | LR: 2.49e-04 | 1399.6ms/step | 14.3 samples/s +Step 22450/44000 | Loss: 2.9521 | LR: 2.49e-04 | 1396.1ms/step | 14.3 samples/s +Step 22460/44000 | Loss: 2.9304 | LR: 2.48e-04 | 1390.2ms/step | 14.4 samples/s +Step 22470/44000 | Loss: 2.9071 | LR: 2.48e-04 | 1389.3ms/step | 14.4 samples/s +Step 22480/44000 | Loss: 2.9214 | LR: 2.48e-04 | 1389.1ms/step | 14.4 samples/s +Step 22490/44000 | Loss: 2.9277 | LR: 2.48e-04 | 1390.9ms/step | 14.4 samples/s +Step 22500/44000 | Loss: 2.9108 | LR: 2.48e-04 | 1396.6ms/step | 14.3 samples/s +Validation at step 22500: Loss = 2.9519, Perplexity = 19.14 +Step 22510/44000 | Loss: 2.9215 | LR: 2.48e-04 | 3601.8ms/step | 5.6 samples/s +Step 22520/44000 | Loss: 2.9669 | LR: 2.47e-04 | 1400.3ms/step | 14.3 samples/s +Step 22530/44000 | Loss: 2.9077 | LR: 2.47e-04 | 1399.9ms/step | 14.3 samples/s +Step 22540/44000 | Loss: 2.9398 | LR: 2.47e-04 | 1399.9ms/step | 14.3 samples/s +Step 22550/44000 | Loss: 2.9064 | LR: 2.47e-04 | 1397.4ms/step | 14.3 samples/s +Step 22560/44000 | Loss: 2.9333 | LR: 2.47e-04 | 1392.4ms/step | 14.4 samples/s +Step 22570/44000 | Loss: 2.9106 | LR: 2.46e-04 | 1388.3ms/step | 14.4 samples/s +Step 22580/44000 | Loss: 2.9481 | LR: 2.46e-04 | 1389.9ms/step | 14.4 samples/s +Step 22590/44000 | Loss: 2.9453 | LR: 2.46e-04 | 1387.3ms/step | 14.4 samples/s +Step 22600/44000 | Loss: 2.9175 | LR: 2.46e-04 | 1388.5ms/step | 14.4 samples/s +Step 22610/44000 | Loss: 2.9321 | LR: 2.46e-04 | 1392.0ms/step | 14.4 samples/s +Step 22620/44000 | Loss: 2.9501 | LR: 2.46e-04 | 1394.3ms/step | 14.3 samples/s +Step 22630/44000 | Loss: 2.9303 | LR: 2.45e-04 | 1400.1ms/step | 14.3 samples/s +Step 22640/44000 | Loss: 2.9435 | LR: 2.45e-04 | 1393.8ms/step | 14.3 samples/s +Step 22650/44000 | Loss: 2.9327 | LR: 2.45e-04 | 1392.0ms/step | 14.4 samples/s +Step 22660/44000 | Loss: 2.9656 | LR: 2.45e-04 | 1400.0ms/step | 14.3 samples/s +Step 22670/44000 | Loss: 2.9386 | LR: 2.45e-04 | 1397.9ms/step | 14.3 samples/s +Step 22680/44000 | Loss: 2.9179 | LR: 2.44e-04 | 1400.2ms/step | 14.3 samples/s +Step 22690/44000 | Loss: 2.9484 | LR: 2.44e-04 | 1399.5ms/step | 14.3 samples/s +Step 22700/44000 | Loss: 2.9512 | LR: 2.44e-04 | 1392.3ms/step | 14.4 samples/s +Step 22710/44000 | Loss: 2.9072 | LR: 2.44e-04 | 1396.3ms/step | 14.3 samples/s +Step 22720/44000 | Loss: 2.9355 | LR: 2.44e-04 | 1399.0ms/step | 14.3 samples/s +Step 22730/44000 | Loss: 2.9517 | LR: 2.44e-04 | 1394.8ms/step | 14.3 samples/s +Step 22740/44000 | Loss: 2.9371 | LR: 2.43e-04 | 1391.9ms/step | 14.4 samples/s +Step 22750/44000 | Loss: 2.9153 | LR: 2.43e-04 | 1390.8ms/step | 14.4 samples/s +Validation at step 22750: Loss = 2.9493, Perplexity = 19.09 +Step 22760/44000 | Loss: 2.9486 | LR: 2.43e-04 | 3591.5ms/step | 5.6 samples/s +Step 22770/44000 | Loss: 2.9313 | LR: 2.43e-04 | 1391.6ms/step | 14.4 samples/s +Step 22780/44000 | Loss: 2.9400 | LR: 2.43e-04 | 1390.6ms/step | 14.4 samples/s +Step 22790/44000 | Loss: 2.9561 | LR: 2.42e-04 | 1390.7ms/step | 14.4 samples/s +Step 22800/44000 | Loss: 2.9561 | LR: 2.42e-04 | 1390.2ms/step | 14.4 samples/s +Step 22810/44000 | Loss: 2.9237 | LR: 2.42e-04 | 1392.9ms/step | 14.4 samples/s +Step 22820/44000 | Loss: 2.9353 | LR: 2.42e-04 | 1398.3ms/step | 14.3 samples/s +Step 22830/44000 | Loss: 2.9611 | LR: 2.42e-04 | 1400.0ms/step | 14.3 samples/s +Step 22840/44000 | Loss: 2.9452 | LR: 2.42e-04 | 1400.0ms/step | 14.3 samples/s +Step 22850/44000 | Loss: 2.9187 | LR: 2.41e-04 | 1397.9ms/step | 14.3 samples/s +Step 22860/44000 | Loss: 2.9580 | LR: 2.41e-04 | 1392.3ms/step | 14.4 samples/s +Step 22870/44000 | Loss: 2.9301 | LR: 2.41e-04 | 1389.6ms/step | 14.4 samples/s +Step 22880/44000 | Loss: 2.9291 | LR: 2.41e-04 | 1388.6ms/step | 14.4 samples/s +Step 22890/44000 | Loss: 2.9185 | LR: 2.41e-04 | 1391.4ms/step | 14.4 samples/s +Step 22900/44000 | Loss: 2.9638 | LR: 2.40e-04 | 1400.1ms/step | 14.3 samples/s +Step 22910/44000 | Loss: 2.9492 | LR: 2.40e-04 | 1400.2ms/step | 14.3 samples/s +Step 22920/44000 | Loss: 2.9371 | LR: 2.40e-04 | 1399.6ms/step | 14.3 samples/s +Step 22930/44000 | Loss: 2.9189 | LR: 2.40e-04 | 1393.6ms/step | 14.4 samples/s +Step 22940/44000 | Loss: 2.9210 | LR: 2.40e-04 | 1396.5ms/step | 14.3 samples/s +Step 22950/44000 | Loss: 2.9281 | LR: 2.40e-04 | 1398.2ms/step | 14.3 samples/s +Step 22960/44000 | Loss: 2.9707 | LR: 2.39e-04 | 1398.0ms/step | 14.3 samples/s +Step 22970/44000 | Loss: 2.9169 | LR: 2.39e-04 | 1398.1ms/step | 14.3 samples/s +Step 22980/44000 | Loss: 2.9323 | LR: 2.39e-04 | 1397.8ms/step | 14.3 samples/s +Step 22990/44000 | Loss: 2.9207 | LR: 2.39e-04 | 1400.5ms/step | 14.3 samples/s +Step 23000/44000 | Loss: 2.9194 | LR: 2.39e-04 | 1399.7ms/step | 14.3 samples/s +Validation at step 23000: Loss = 2.9476, Perplexity = 19.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-23000 +Step 23010/44000 | Loss: 2.9232 | LR: 2.38e-04 | 3920.1ms/step | 5.1 samples/s +Step 23020/44000 | Loss: 2.9332 | LR: 2.38e-04 | 1399.9ms/step | 14.3 samples/s +Step 23030/44000 | Loss: 2.9396 | LR: 2.38e-04 | 1400.0ms/step | 14.3 samples/s +Step 23040/44000 | Loss: 2.9524 | LR: 2.38e-04 | 1400.1ms/step | 14.3 samples/s +Step 23050/44000 | Loss: 2.9719 | LR: 2.38e-04 | 1399.9ms/step | 14.3 samples/s +Step 23060/44000 | Loss: 2.9354 | LR: 2.38e-04 | 1398.0ms/step | 14.3 samples/s +Step 23070/44000 | Loss: 2.9317 | LR: 2.37e-04 | 1400.1ms/step | 14.3 samples/s +Step 23080/44000 | Loss: 2.9335 | LR: 2.37e-04 | 1399.6ms/step | 14.3 samples/s +Step 23090/44000 | Loss: 2.9444 | LR: 2.37e-04 | 1400.1ms/step | 14.3 samples/s +Step 23100/44000 | Loss: 2.9320 | LR: 2.37e-04 | 1398.2ms/step | 14.3 samples/s +Step 23110/44000 | Loss: 2.9186 | LR: 2.37e-04 | 1400.2ms/step | 14.3 samples/s +Step 23120/44000 | Loss: 2.9263 | LR: 2.36e-04 | 1395.6ms/step | 14.3 samples/s +Step 23130/44000 | Loss: 2.9272 | LR: 2.36e-04 | 1392.8ms/step | 14.4 samples/s +Step 23140/44000 | Loss: 2.9044 | LR: 2.36e-04 | 1387.4ms/step | 14.4 samples/s +Step 23150/44000 | Loss: 2.9162 | LR: 2.36e-04 | 1390.0ms/step | 14.4 samples/s +Step 23160/44000 | Loss: 2.9364 | LR: 2.36e-04 | 1395.0ms/step | 14.3 samples/s +Step 23170/44000 | Loss: 2.9261 | LR: 2.36e-04 | 1395.1ms/step | 14.3 samples/s +Step 23180/44000 | Loss: 2.9382 | LR: 2.35e-04 | 1396.6ms/step | 14.3 samples/s +Step 23190/44000 | Loss: 2.9340 | LR: 2.35e-04 | 1396.5ms/step | 14.3 samples/s +Step 23200/44000 | Loss: 2.9322 | LR: 2.35e-04 | 1390.7ms/step | 14.4 samples/s +Step 23210/44000 | Loss: 2.9191 | LR: 2.35e-04 | 1391.6ms/step | 14.4 samples/s +Step 23220/44000 | Loss: 2.9361 | LR: 2.35e-04 | 1394.1ms/step | 14.3 samples/s +Step 23230/44000 | Loss: 2.9133 | LR: 2.34e-04 | 1394.3ms/step | 14.3 samples/s +Step 23240/44000 | Loss: 2.9347 | LR: 2.34e-04 | 1393.7ms/step | 14.4 samples/s +Step 23250/44000 | Loss: 2.9380 | LR: 2.34e-04 | 1388.6ms/step | 14.4 samples/s +Validation at step 23250: Loss = 2.9449, Perplexity = 19.01 +Step 23260/44000 | Loss: 2.9092 | LR: 2.34e-04 | 3591.5ms/step | 5.6 samples/s +Step 23270/44000 | Loss: 2.9320 | LR: 2.34e-04 | 1394.3ms/step | 14.3 samples/s +Step 23280/44000 | Loss: 2.9465 | LR: 2.34e-04 | 1392.1ms/step | 14.4 samples/s +Step 23290/44000 | Loss: 2.9272 | LR: 2.33e-04 | 1395.7ms/step | 14.3 samples/s +Step 23300/44000 | Loss: 2.9172 | LR: 2.33e-04 | 1392.2ms/step | 14.4 samples/s +Step 23310/44000 | Loss: 2.8965 | LR: 2.33e-04 | 1394.3ms/step | 14.3 samples/s +Step 23320/44000 | Loss: 2.9241 | LR: 2.33e-04 | 1395.6ms/step | 14.3 samples/s +Step 23330/44000 | Loss: 2.9144 | LR: 2.33e-04 | 1391.9ms/step | 14.4 samples/s +Step 23340/44000 | Loss: 2.9411 | LR: 2.32e-04 | 1392.5ms/step | 14.4 samples/s +Step 23350/44000 | Loss: 2.9253 | LR: 2.32e-04 | 1391.9ms/step | 14.4 samples/s +Step 23360/44000 | Loss: 2.9671 | LR: 2.32e-04 | 1399.3ms/step | 14.3 samples/s +Step 23370/44000 | Loss: 2.9194 | LR: 2.32e-04 | 1392.3ms/step | 14.4 samples/s +Step 23380/44000 | Loss: 2.9126 | LR: 2.32e-04 | 1394.1ms/step | 14.3 samples/s +Step 23390/44000 | Loss: 2.8998 | LR: 2.32e-04 | 1398.4ms/step | 14.3 samples/s +Step 23400/44000 | Loss: 2.9377 | LR: 2.31e-04 | 1400.0ms/step | 14.3 samples/s +Step 23410/44000 | Loss: 2.9249 | LR: 2.31e-04 | 1399.8ms/step | 14.3 samples/s +Step 23420/44000 | Loss: 2.9467 | LR: 2.31e-04 | 1393.9ms/step | 14.3 samples/s +Step 23430/44000 | Loss: 2.9172 | LR: 2.31e-04 | 1394.3ms/step | 14.3 samples/s +Step 23440/44000 | Loss: 2.9153 | LR: 2.31e-04 | 1399.8ms/step | 14.3 samples/s +Step 23450/44000 | Loss: 2.9347 | LR: 2.30e-04 | 1393.6ms/step | 14.4 samples/s +Step 23460/44000 | Loss: 2.9180 | LR: 2.30e-04 | 1394.1ms/step | 14.3 samples/s +Step 23470/44000 | Loss: 2.9397 | LR: 2.30e-04 | 1394.2ms/step | 14.3 samples/s +Step 23480/44000 | Loss: 2.9142 | LR: 2.30e-04 | 1393.8ms/step | 14.3 samples/s +Step 23490/44000 | Loss: 2.9311 | LR: 2.30e-04 | 1391.7ms/step | 14.4 samples/s +Step 23500/44000 | Loss: 2.9170 | LR: 2.30e-04 | 1392.5ms/step | 14.4 samples/s +Validation at step 23500: Loss = 2.9434, Perplexity = 18.98 +Step 23510/44000 | Loss: 2.9331 | LR: 2.29e-04 | 3599.6ms/step | 5.6 samples/s +Step 23520/44000 | Loss: 2.9240 | LR: 2.29e-04 | 1394.2ms/step | 14.3 samples/s +Step 23530/44000 | Loss: 2.9217 | LR: 2.29e-04 | 1396.6ms/step | 14.3 samples/s +Step 23540/44000 | Loss: 2.9069 | LR: 2.29e-04 | 1393.9ms/step | 14.3 samples/s +Step 23550/44000 | Loss: 2.9053 | LR: 2.29e-04 | 1392.9ms/step | 14.4 samples/s +Step 23560/44000 | Loss: 2.9230 | LR: 2.29e-04 | 1393.1ms/step | 14.4 samples/s +Step 23570/44000 | Loss: 2.9511 | LR: 2.28e-04 | 1391.8ms/step | 14.4 samples/s +Step 23580/44000 | Loss: 2.9010 | LR: 2.28e-04 | 1393.8ms/step | 14.3 samples/s +Step 23590/44000 | Loss: 2.9414 | LR: 2.28e-04 | 1394.4ms/step | 14.3 samples/s +Step 23600/44000 | Loss: 2.9572 | LR: 2.28e-04 | 1397.9ms/step | 14.3 samples/s +Step 23610/44000 | Loss: 2.9162 | LR: 2.28e-04 | 1392.6ms/step | 14.4 samples/s +Step 23620/44000 | Loss: 2.9220 | LR: 2.27e-04 | 1397.3ms/step | 14.3 samples/s +Step 23630/44000 | Loss: 2.9267 | LR: 2.27e-04 | 1394.2ms/step | 14.3 samples/s +Step 23640/44000 | Loss: 2.9416 | LR: 2.27e-04 | 1396.7ms/step | 14.3 samples/s +Step 23650/44000 | Loss: 2.9486 | LR: 2.27e-04 | 1395.4ms/step | 14.3 samples/s +Step 23660/44000 | Loss: 2.9289 | LR: 2.27e-04 | 1397.8ms/step | 14.3 samples/s +Step 23670/44000 | Loss: 2.9388 | LR: 2.27e-04 | 1392.5ms/step | 14.4 samples/s +Step 23680/44000 | Loss: 2.9365 | LR: 2.26e-04 | 1389.4ms/step | 14.4 samples/s +Step 23690/44000 | Loss: 2.9378 | LR: 2.26e-04 | 1390.2ms/step | 14.4 samples/s +Step 23700/44000 | Loss: 2.9223 | LR: 2.26e-04 | 1390.5ms/step | 14.4 samples/s +Step 23710/44000 | Loss: 2.9068 | LR: 2.26e-04 | 1390.1ms/step | 14.4 samples/s +Step 23720/44000 | Loss: 2.9129 | LR: 2.26e-04 | 1392.5ms/step | 14.4 samples/s +Step 23730/44000 | Loss: 2.9319 | LR: 2.25e-04 | 1392.5ms/step | 14.4 samples/s +Step 23740/44000 | Loss: 2.9303 | LR: 2.25e-04 | 1396.2ms/step | 14.3 samples/s +Step 23750/44000 | Loss: 2.9137 | LR: 2.25e-04 | 1396.6ms/step | 14.3 samples/s +Validation at step 23750: Loss = 2.9406, Perplexity = 18.93 +Step 23760/44000 | Loss: 2.9420 | LR: 2.25e-04 | 3604.6ms/step | 5.5 samples/s +Step 23770/44000 | Loss: 2.9149 | LR: 2.25e-04 | 1391.2ms/step | 14.4 samples/s +Step 23780/44000 | Loss: 2.9114 | LR: 2.25e-04 | 1392.7ms/step | 14.4 samples/s +Step 23790/44000 | Loss: 2.9061 | LR: 2.24e-04 | 1393.4ms/step | 14.4 samples/s +Step 23800/44000 | Loss: 2.9305 | LR: 2.24e-04 | 1398.0ms/step | 14.3 samples/s +Step 23810/44000 | Loss: 2.9065 | LR: 2.24e-04 | 1396.6ms/step | 14.3 samples/s +Step 23820/44000 | Loss: 2.9497 | LR: 2.24e-04 | 1393.0ms/step | 14.4 samples/s +Step 23830/44000 | Loss: 2.9184 | LR: 2.24e-04 | 1388.3ms/step | 14.4 samples/s +Step 23840/44000 | Loss: 2.9316 | LR: 2.23e-04 | 1391.7ms/step | 14.4 samples/s +Step 23850/44000 | Loss: 2.9341 | LR: 2.23e-04 | 1394.1ms/step | 14.3 samples/s +Step 23860/44000 | Loss: 2.9238 | LR: 2.23e-04 | 1398.1ms/step | 14.3 samples/s +Step 23870/44000 | Loss: 2.9362 | LR: 2.23e-04 | 1398.3ms/step | 14.3 samples/s +Step 23880/44000 | Loss: 2.9204 | LR: 2.23e-04 | 1397.8ms/step | 14.3 samples/s +Step 23890/44000 | Loss: 2.9201 | LR: 2.23e-04 | 1400.0ms/step | 14.3 samples/s +Step 23900/44000 | Loss: 2.9178 | LR: 2.22e-04 | 1397.7ms/step | 14.3 samples/s +Step 23910/44000 | Loss: 2.9159 | LR: 2.22e-04 | 1393.9ms/step | 14.3 samples/s +Step 23920/44000 | Loss: 2.9578 | LR: 2.22e-04 | 1392.4ms/step | 14.4 samples/s +Step 23930/44000 | Loss: 2.9297 | LR: 2.22e-04 | 1393.7ms/step | 14.4 samples/s +Step 23940/44000 | Loss: 2.9422 | LR: 2.22e-04 | 1390.4ms/step | 14.4 samples/s +Step 23950/44000 | Loss: 2.9374 | LR: 2.21e-04 | 1388.2ms/step | 14.4 samples/s +Step 23960/44000 | Loss: 2.9598 | LR: 2.21e-04 | 1391.7ms/step | 14.4 samples/s +Step 23970/44000 | Loss: 2.9390 | LR: 2.21e-04 | 1394.0ms/step | 14.3 samples/s +Step 23980/44000 | Loss: 2.9191 | LR: 2.21e-04 | 1399.8ms/step | 14.3 samples/s +Step 23990/44000 | Loss: 2.8958 | LR: 2.21e-04 | 1398.0ms/step | 14.3 samples/s +Step 24000/44000 | Loss: 2.9192 | LR: 2.21e-04 | 1395.7ms/step | 14.3 samples/s +Validation at step 24000: Loss = 2.9389, Perplexity = 18.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-24000 +Step 24010/44000 | Loss: 2.9296 | LR: 2.20e-04 | 3906.9ms/step | 5.1 samples/s +Step 24020/44000 | Loss: 2.9246 | LR: 2.20e-04 | 1392.5ms/step | 14.4 samples/s +Step 24030/44000 | Loss: 2.9465 | LR: 2.20e-04 | 1397.9ms/step | 14.3 samples/s +Step 24040/44000 | Loss: 2.9590 | LR: 2.20e-04 | 1395.1ms/step | 14.3 samples/s +Step 24050/44000 | Loss: 2.9502 | LR: 2.20e-04 | 1394.4ms/step | 14.3 samples/s +Step 24060/44000 | Loss: 2.9343 | LR: 2.19e-04 | 1393.7ms/step | 14.4 samples/s +Step 24070/44000 | Loss: 2.9515 | LR: 2.19e-04 | 1391.9ms/step | 14.4 samples/s +Step 24080/44000 | Loss: 2.9095 | LR: 2.19e-04 | 1400.3ms/step | 14.3 samples/s +Step 24090/44000 | Loss: 2.9090 | LR: 2.19e-04 | 1397.2ms/step | 14.3 samples/s +Step 24100/44000 | Loss: 2.9282 | LR: 2.19e-04 | 1394.8ms/step | 14.3 samples/s +Step 24110/44000 | Loss: 2.9379 | LR: 2.19e-04 | 1391.0ms/step | 14.4 samples/s +Step 24120/44000 | Loss: 2.8914 | LR: 2.18e-04 | 1385.4ms/step | 14.4 samples/s +Step 24130/44000 | Loss: 2.9392 | LR: 2.18e-04 | 1389.4ms/step | 14.4 samples/s +Step 24140/44000 | Loss: 2.9159 | LR: 2.18e-04 | 1390.3ms/step | 14.4 samples/s +Step 24150/44000 | Loss: 2.9439 | LR: 2.18e-04 | 1393.7ms/step | 14.4 samples/s +Step 24160/44000 | Loss: 2.9544 | LR: 2.18e-04 | 1393.5ms/step | 14.4 samples/s +Step 24170/44000 | Loss: 2.9096 | LR: 2.17e-04 | 1394.4ms/step | 14.3 samples/s +Step 24180/44000 | Loss: 2.9183 | LR: 2.17e-04 | 1392.0ms/step | 14.4 samples/s +Step 24190/44000 | Loss: 2.9392 | LR: 2.17e-04 | 1395.1ms/step | 14.3 samples/s +Step 24200/44000 | Loss: 2.9113 | LR: 2.17e-04 | 1388.8ms/step | 14.4 samples/s +Step 24210/44000 | Loss: 2.9051 | LR: 2.17e-04 | 1391.5ms/step | 14.4 samples/s +Step 24220/44000 | Loss: 2.9238 | LR: 2.17e-04 | 1393.0ms/step | 14.4 samples/s +Step 24230/44000 | Loss: 2.9430 | LR: 2.16e-04 | 1399.9ms/step | 14.3 samples/s +Step 24240/44000 | Loss: 2.9052 | LR: 2.16e-04 | 1396.2ms/step | 14.3 samples/s +Step 24250/44000 | Loss: 2.9317 | LR: 2.16e-04 | 1399.8ms/step | 14.3 samples/s +Validation at step 24250: Loss = 2.9372, Perplexity = 18.86 +Step 24260/44000 | Loss: 2.9016 | LR: 2.16e-04 | 3628.3ms/step | 5.5 samples/s +Step 24270/44000 | Loss: 2.9215 | LR: 2.16e-04 | 1399.7ms/step | 14.3 samples/s +Step 24280/44000 | Loss: 2.9464 | LR: 2.16e-04 | 1400.1ms/step | 14.3 samples/s +Step 24290/44000 | Loss: 2.9287 | LR: 2.15e-04 | 1400.1ms/step | 14.3 samples/s +Step 24300/44000 | Loss: 2.9150 | LR: 2.15e-04 | 1400.0ms/step | 14.3 samples/s +Step 24310/44000 | Loss: 2.9278 | LR: 2.15e-04 | 1400.0ms/step | 14.3 samples/s +Step 24320/44000 | Loss: 2.9255 | LR: 2.15e-04 | 1399.3ms/step | 14.3 samples/s +Step 24330/44000 | Loss: 2.9442 | LR: 2.15e-04 | 1394.5ms/step | 14.3 samples/s +Step 24340/44000 | Loss: 2.9075 | LR: 2.14e-04 | 1396.3ms/step | 14.3 samples/s +Step 24350/44000 | Loss: 2.8993 | LR: 2.14e-04 | 1397.9ms/step | 14.3 samples/s +Step 24360/44000 | Loss: 2.9247 | LR: 2.14e-04 | 1399.8ms/step | 14.3 samples/s +Step 24370/44000 | Loss: 2.8926 | LR: 2.14e-04 | 1400.2ms/step | 14.3 samples/s +Step 24380/44000 | Loss: 2.8864 | LR: 2.14e-04 | 1397.5ms/step | 14.3 samples/s +Step 24390/44000 | Loss: 2.8958 | LR: 2.14e-04 | 1392.4ms/step | 14.4 samples/s +Step 24400/44000 | Loss: 2.9257 | LR: 2.13e-04 | 1394.2ms/step | 14.3 samples/s +Step 24410/44000 | Loss: 2.9245 | LR: 2.13e-04 | 1395.9ms/step | 14.3 samples/s +Step 24420/44000 | Loss: 2.9367 | LR: 2.13e-04 | 1397.0ms/step | 14.3 samples/s +Step 24430/44000 | Loss: 2.9225 | LR: 2.13e-04 | 1394.7ms/step | 14.3 samples/s +Step 24440/44000 | Loss: 2.9290 | LR: 2.13e-04 | 1396.4ms/step | 14.3 samples/s +Step 24450/44000 | Loss: 2.8965 | LR: 2.12e-04 | 1397.8ms/step | 14.3 samples/s +Step 24460/44000 | Loss: 2.9014 | LR: 2.12e-04 | 1400.0ms/step | 14.3 samples/s +Step 24470/44000 | Loss: 2.9284 | LR: 2.12e-04 | 1398.1ms/step | 14.3 samples/s +Step 24480/44000 | Loss: 2.9223 | LR: 2.12e-04 | 1399.8ms/step | 14.3 samples/s +Step 24490/44000 | Loss: 2.9198 | LR: 2.12e-04 | 1398.3ms/step | 14.3 samples/s +Step 24500/44000 | Loss: 2.9353 | LR: 2.12e-04 | 1397.4ms/step | 14.3 samples/s +Validation at step 24500: Loss = 2.9353, Perplexity = 18.83 +Step 24510/44000 | Loss: 2.9244 | LR: 2.11e-04 | 3604.5ms/step | 5.5 samples/s +Step 24520/44000 | Loss: 2.8936 | LR: 2.11e-04 | 1400.0ms/step | 14.3 samples/s +Step 24530/44000 | Loss: 2.9173 | LR: 2.11e-04 | 1394.0ms/step | 14.3 samples/s +Step 24540/44000 | Loss: 2.9383 | LR: 2.11e-04 | 1392.8ms/step | 14.4 samples/s +Step 24550/44000 | Loss: 2.9515 | LR: 2.11e-04 | 1397.2ms/step | 14.3 samples/s +Step 24560/44000 | Loss: 2.9180 | LR: 2.10e-04 | 1400.2ms/step | 14.3 samples/s +Step 24570/44000 | Loss: 2.9854 | LR: 2.10e-04 | 1399.9ms/step | 14.3 samples/s +Step 24580/44000 | Loss: 2.8984 | LR: 2.10e-04 | 1399.9ms/step | 14.3 samples/s +Step 24590/44000 | Loss: 2.9485 | LR: 2.10e-04 | 1400.1ms/step | 14.3 samples/s +Step 24600/44000 | Loss: 2.9181 | LR: 2.10e-04 | 1399.9ms/step | 14.3 samples/s +Step 24610/44000 | Loss: 2.9358 | LR: 2.10e-04 | 1400.2ms/step | 14.3 samples/s +Step 24620/44000 | Loss: 2.9303 | LR: 2.09e-04 | 1400.1ms/step | 14.3 samples/s +Step 24630/44000 | Loss: 2.9515 | LR: 2.09e-04 | 1400.1ms/step | 14.3 samples/s +Step 24640/44000 | Loss: 2.9203 | LR: 2.09e-04 | 1399.6ms/step | 14.3 samples/s +Step 24650/44000 | Loss: 2.9224 | LR: 2.09e-04 | 1394.2ms/step | 14.3 samples/s +Step 24660/44000 | Loss: 2.9143 | LR: 2.09e-04 | 1396.2ms/step | 14.3 samples/s +Step 24670/44000 | Loss: 2.9155 | LR: 2.09e-04 | 1396.1ms/step | 14.3 samples/s +Step 24680/44000 | Loss: 2.9113 | LR: 2.08e-04 | 1393.6ms/step | 14.4 samples/s +Step 24690/44000 | Loss: 2.9173 | LR: 2.08e-04 | 1391.2ms/step | 14.4 samples/s +Step 24700/44000 | Loss: 2.9115 | LR: 2.08e-04 | 1394.8ms/step | 14.3 samples/s +Step 24710/44000 | Loss: 2.9148 | LR: 2.08e-04 | 1395.9ms/step | 14.3 samples/s +Step 24720/44000 | Loss: 2.9017 | LR: 2.08e-04 | 1394.2ms/step | 14.3 samples/s +Step 24730/44000 | Loss: 2.9129 | LR: 2.07e-04 | 1400.1ms/step | 14.3 samples/s +Step 24740/44000 | Loss: 2.9612 | LR: 2.07e-04 | 1398.0ms/step | 14.3 samples/s +Step 24750/44000 | Loss: 2.8994 | LR: 2.07e-04 | 1400.0ms/step | 14.3 samples/s +Validation at step 24750: Loss = 2.9334, Perplexity = 18.79 +Step 24760/44000 | Loss: 2.9203 | LR: 2.07e-04 | 3604.1ms/step | 5.5 samples/s +Step 24770/44000 | Loss: 2.9199 | LR: 2.07e-04 | 1400.2ms/step | 14.3 samples/s +Step 24780/44000 | Loss: 2.9431 | LR: 2.07e-04 | 1397.9ms/step | 14.3 samples/s +Step 24790/44000 | Loss: 2.9002 | LR: 2.06e-04 | 1399.9ms/step | 14.3 samples/s +Step 24800/44000 | Loss: 2.9030 | LR: 2.06e-04 | 1398.2ms/step | 14.3 samples/s +Step 24810/44000 | Loss: 2.9352 | LR: 2.06e-04 | 1392.1ms/step | 14.4 samples/s +Step 24820/44000 | Loss: 2.9184 | LR: 2.06e-04 | 1391.8ms/step | 14.4 samples/s +Step 24830/44000 | Loss: 2.8934 | LR: 2.06e-04 | 1392.1ms/step | 14.4 samples/s +Step 24840/44000 | Loss: 2.9110 | LR: 2.05e-04 | 1391.6ms/step | 14.4 samples/s +Step 24850/44000 | Loss: 2.9141 | LR: 2.05e-04 | 1391.4ms/step | 14.4 samples/s +Step 24860/44000 | Loss: 2.9151 | LR: 2.05e-04 | 1390.7ms/step | 14.4 samples/s +Step 24870/44000 | Loss: 2.9048 | LR: 2.05e-04 | 1387.3ms/step | 14.4 samples/s +Step 24880/44000 | Loss: 2.9322 | LR: 2.05e-04 | 1390.3ms/step | 14.4 samples/s +Step 24890/44000 | Loss: 2.9538 | LR: 2.05e-04 | 1392.5ms/step | 14.4 samples/s +Step 24900/44000 | Loss: 2.9097 | LR: 2.04e-04 | 1396.2ms/step | 14.3 samples/s +Step 24910/44000 | Loss: 2.9198 | LR: 2.04e-04 | 1397.9ms/step | 14.3 samples/s +Step 24920/44000 | Loss: 2.9250 | LR: 2.04e-04 | 1396.1ms/step | 14.3 samples/s +Step 24930/44000 | Loss: 2.9200 | LR: 2.04e-04 | 1390.7ms/step | 14.4 samples/s +Step 24940/44000 | Loss: 2.9295 | LR: 2.04e-04 | 1391.8ms/step | 14.4 samples/s +Step 24950/44000 | Loss: 2.9075 | LR: 2.04e-04 | 1390.4ms/step | 14.4 samples/s +Step 24960/44000 | Loss: 2.9555 | LR: 2.03e-04 | 1393.1ms/step | 14.4 samples/s +Step 24970/44000 | Loss: 2.9079 | LR: 2.03e-04 | 1398.0ms/step | 14.3 samples/s +Step 24980/44000 | Loss: 2.9412 | LR: 2.03e-04 | 1400.1ms/step | 14.3 samples/s +Step 24990/44000 | Loss: 2.8922 | LR: 2.03e-04 | 1399.9ms/step | 14.3 samples/s +Step 25000/44000 | Loss: 2.9345 | LR: 2.03e-04 | 1400.1ms/step | 14.3 samples/s +Validation at step 25000: Loss = 2.9311, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-25000 +Step 25010/44000 | Loss: 2.8982 | LR: 2.02e-04 | 3872.9ms/step | 5.2 samples/s +Step 25020/44000 | Loss: 2.9283 | LR: 2.02e-04 | 1391.5ms/step | 14.4 samples/s +Step 25030/44000 | Loss: 2.9081 | LR: 2.02e-04 | 1387.3ms/step | 14.4 samples/s +Step 25040/44000 | Loss: 2.9230 | LR: 2.02e-04 | 1388.2ms/step | 14.4 samples/s +Step 25050/44000 | Loss: 2.9145 | LR: 2.02e-04 | 1391.9ms/step | 14.4 samples/s +Step 25060/44000 | Loss: 2.9107 | LR: 2.02e-04 | 1395.8ms/step | 14.3 samples/s +Step 25070/44000 | Loss: 2.9352 | LR: 2.01e-04 | 1389.7ms/step | 14.4 samples/s +Step 25080/44000 | Loss: 2.8989 | LR: 2.01e-04 | 1387.0ms/step | 14.4 samples/s +Step 25090/44000 | Loss: 2.9369 | LR: 2.01e-04 | 1386.7ms/step | 14.4 samples/s +Step 25100/44000 | Loss: 2.8969 | LR: 2.01e-04 | 1388.9ms/step | 14.4 samples/s +Step 25110/44000 | Loss: 2.9205 | LR: 2.01e-04 | 1392.1ms/step | 14.4 samples/s +Step 25120/44000 | Loss: 2.8950 | LR: 2.00e-04 | 1389.8ms/step | 14.4 samples/s +Step 25130/44000 | Loss: 2.9496 | LR: 2.00e-04 | 1386.6ms/step | 14.4 samples/s +Step 25140/44000 | Loss: 2.9594 | LR: 2.00e-04 | 1390.7ms/step | 14.4 samples/s +Step 25150/44000 | Loss: 2.8819 | LR: 2.00e-04 | 1396.6ms/step | 14.3 samples/s +Step 25160/44000 | Loss: 2.8959 | LR: 2.00e-04 | 1393.2ms/step | 14.4 samples/s +Step 25170/44000 | Loss: 2.9304 | LR: 2.00e-04 | 1391.8ms/step | 14.4 samples/s +Step 25180/44000 | Loss: 2.8704 | LR: 1.99e-04 | 1391.8ms/step | 14.4 samples/s +Step 25190/44000 | Loss: 2.9124 | LR: 1.99e-04 | 1390.2ms/step | 14.4 samples/s +Step 25200/44000 | Loss: 2.9339 | LR: 1.99e-04 | 1391.4ms/step | 14.4 samples/s +Step 25210/44000 | Loss: 2.9290 | LR: 1.99e-04 | 1394.5ms/step | 14.3 samples/s +Step 25220/44000 | Loss: 2.8971 | LR: 1.99e-04 | 1393.6ms/step | 14.4 samples/s +Step 25230/44000 | Loss: 2.9186 | LR: 1.99e-04 | 1396.0ms/step | 14.3 samples/s +Step 25240/44000 | Loss: 2.9001 | LR: 1.98e-04 | 1395.2ms/step | 14.3 samples/s +Step 25250/44000 | Loss: 2.9320 | LR: 1.98e-04 | 1399.2ms/step | 14.3 samples/s +Validation at step 25250: Loss = 2.9294, Perplexity = 18.72 +Step 25260/44000 | Loss: 2.8941 | LR: 1.98e-04 | 3608.0ms/step | 5.5 samples/s +Step 25270/44000 | Loss: 2.9247 | LR: 1.98e-04 | 1395.9ms/step | 14.3 samples/s +Step 25280/44000 | Loss: 2.8906 | LR: 1.98e-04 | 1393.4ms/step | 14.4 samples/s +Step 25290/44000 | Loss: 2.9112 | LR: 1.97e-04 | 1394.7ms/step | 14.3 samples/s +Step 25300/44000 | Loss: 2.9180 | LR: 1.97e-04 | 1395.7ms/step | 14.3 samples/s +Step 25310/44000 | Loss: 2.8681 | LR: 1.97e-04 | 1394.2ms/step | 14.3 samples/s +Step 25320/44000 | Loss: 2.9084 | LR: 1.97e-04 | 1393.8ms/step | 14.3 samples/s +Step 25330/44000 | Loss: 2.9184 | LR: 1.97e-04 | 1393.4ms/step | 14.4 samples/s +Step 25340/44000 | Loss: 2.9111 | LR: 1.97e-04 | 1394.6ms/step | 14.3 samples/s +Step 25350/44000 | Loss: 2.9034 | LR: 1.96e-04 | 1400.0ms/step | 14.3 samples/s +Step 25360/44000 | Loss: 2.9499 | LR: 1.96e-04 | 1396.2ms/step | 14.3 samples/s +Step 25370/44000 | Loss: 2.8985 | LR: 1.96e-04 | 1391.9ms/step | 14.4 samples/s +Step 25380/44000 | Loss: 2.9391 | LR: 1.96e-04 | 1391.5ms/step | 14.4 samples/s +Step 25390/44000 | Loss: 2.9101 | LR: 1.96e-04 | 1394.6ms/step | 14.3 samples/s +Step 25400/44000 | Loss: 2.9237 | LR: 1.96e-04 | 1395.2ms/step | 14.3 samples/s +Step 25410/44000 | Loss: 2.9533 | LR: 1.95e-04 | 1392.8ms/step | 14.4 samples/s +Step 25420/44000 | Loss: 2.9370 | LR: 1.95e-04 | 1395.7ms/step | 14.3 samples/s +Step 25430/44000 | Loss: 2.9172 | LR: 1.95e-04 | 1396.1ms/step | 14.3 samples/s +Step 25440/44000 | Loss: 2.9154 | LR: 1.95e-04 | 1400.1ms/step | 14.3 samples/s +Step 25450/44000 | Loss: 2.9520 | LR: 1.95e-04 | 1398.1ms/step | 14.3 samples/s +Step 25460/44000 | Loss: 2.9239 | LR: 1.94e-04 | 1398.1ms/step | 14.3 samples/s +Step 25470/44000 | Loss: 2.9035 | LR: 1.94e-04 | 1400.0ms/step | 14.3 samples/s +Step 25480/44000 | Loss: 2.9178 | LR: 1.94e-04 | 1400.0ms/step | 14.3 samples/s +Step 25490/44000 | Loss: 2.9183 | LR: 1.94e-04 | 1400.1ms/step | 14.3 samples/s +Step 25500/44000 | Loss: 2.9093 | LR: 1.94e-04 | 1399.8ms/step | 14.3 samples/s +Validation at step 25500: Loss = 2.9277, Perplexity = 18.68 +Step 25510/44000 | Loss: 2.8969 | LR: 1.94e-04 | 3610.0ms/step | 5.5 samples/s +Step 25520/44000 | Loss: 2.8959 | LR: 1.93e-04 | 1400.2ms/step | 14.3 samples/s +Step 25530/44000 | Loss: 2.8986 | LR: 1.93e-04 | 1399.8ms/step | 14.3 samples/s +Step 25540/44000 | Loss: 2.9058 | LR: 1.93e-04 | 1399.9ms/step | 14.3 samples/s +Step 25550/44000 | Loss: 2.9186 | LR: 1.93e-04 | 1399.9ms/step | 14.3 samples/s +Step 25560/44000 | Loss: 2.9255 | LR: 1.93e-04 | 1400.1ms/step | 14.3 samples/s +Step 25570/44000 | Loss: 2.9077 | LR: 1.92e-04 | 1396.1ms/step | 14.3 samples/s +Step 25580/44000 | Loss: 2.9399 | LR: 1.92e-04 | 1392.2ms/step | 14.4 samples/s +Step 25590/44000 | Loss: 2.9193 | LR: 1.92e-04 | 1393.0ms/step | 14.4 samples/s +Step 25600/44000 | Loss: 2.8874 | LR: 1.92e-04 | 1395.0ms/step | 14.3 samples/s +Step 25610/44000 | Loss: 2.9011 | LR: 1.92e-04 | 1397.9ms/step | 14.3 samples/s +Step 25620/44000 | Loss: 2.8903 | LR: 1.92e-04 | 1400.1ms/step | 14.3 samples/s +Step 25630/44000 | Loss: 2.9223 | LR: 1.91e-04 | 1399.8ms/step | 14.3 samples/s +Step 25640/44000 | Loss: 2.9042 | LR: 1.91e-04 | 1398.1ms/step | 14.3 samples/s +Step 25650/44000 | Loss: 2.9083 | LR: 1.91e-04 | 1400.2ms/step | 14.3 samples/s +Step 25660/44000 | Loss: 2.9314 | LR: 1.91e-04 | 1400.0ms/step | 14.3 samples/s +Step 25670/44000 | Loss: 2.9216 | LR: 1.91e-04 | 1399.9ms/step | 14.3 samples/s +Step 25680/44000 | Loss: 2.8868 | LR: 1.91e-04 | 1400.0ms/step | 14.3 samples/s +Step 25690/44000 | Loss: 2.9072 | LR: 1.90e-04 | 1396.2ms/step | 14.3 samples/s +Step 25700/44000 | Loss: 2.8920 | LR: 1.90e-04 | 1390.1ms/step | 14.4 samples/s +Step 25710/44000 | Loss: 2.9057 | LR: 1.90e-04 | 1390.0ms/step | 14.4 samples/s +Step 25720/44000 | Loss: 2.8999 | LR: 1.90e-04 | 1391.6ms/step | 14.4 samples/s +Step 25730/44000 | Loss: 2.8922 | LR: 1.90e-04 | 1392.3ms/step | 14.4 samples/s +Step 25740/44000 | Loss: 2.9083 | LR: 1.89e-04 | 1392.5ms/step | 14.4 samples/s +Step 25750/44000 | Loss: 2.9084 | LR: 1.89e-04 | 1395.2ms/step | 14.3 samples/s +Validation at step 25750: Loss = 2.9262, Perplexity = 18.66 +Step 25760/44000 | Loss: 2.9065 | LR: 1.89e-04 | 3592.1ms/step | 5.6 samples/s +Step 25770/44000 | Loss: 2.9315 | LR: 1.89e-04 | 1394.0ms/step | 14.3 samples/s +Step 25780/44000 | Loss: 2.9156 | LR: 1.89e-04 | 1392.1ms/step | 14.4 samples/s +Step 25790/44000 | Loss: 2.9517 | LR: 1.89e-04 | 1398.0ms/step | 14.3 samples/s +Step 25800/44000 | Loss: 2.8720 | LR: 1.88e-04 | 1394.2ms/step | 14.3 samples/s +Step 25810/44000 | Loss: 2.9094 | LR: 1.88e-04 | 1390.4ms/step | 14.4 samples/s +Step 25820/44000 | Loss: 2.9097 | LR: 1.88e-04 | 1387.0ms/step | 14.4 samples/s +Step 25830/44000 | Loss: 2.9109 | LR: 1.88e-04 | 1386.8ms/step | 14.4 samples/s +Step 25840/44000 | Loss: 2.9228 | LR: 1.88e-04 | 1389.7ms/step | 14.4 samples/s +Step 25850/44000 | Loss: 2.8875 | LR: 1.88e-04 | 1390.2ms/step | 14.4 samples/s +Step 25860/44000 | Loss: 2.8821 | LR: 1.87e-04 | 1391.9ms/step | 14.4 samples/s +Step 25870/44000 | Loss: 2.8848 | LR: 1.87e-04 | 1395.4ms/step | 14.3 samples/s +Step 25880/44000 | Loss: 2.9313 | LR: 1.87e-04 | 1394.5ms/step | 14.3 samples/s +Step 25890/44000 | Loss: 2.9108 | LR: 1.87e-04 | 1395.8ms/step | 14.3 samples/s +Step 25900/44000 | Loss: 2.9126 | LR: 1.87e-04 | 1398.3ms/step | 14.3 samples/s +Step 25910/44000 | Loss: 2.8995 | LR: 1.87e-04 | 1399.7ms/step | 14.3 samples/s +Step 25920/44000 | Loss: 2.8967 | LR: 1.86e-04 | 1396.6ms/step | 14.3 samples/s +Step 25930/44000 | Loss: 2.9271 | LR: 1.86e-04 | 1393.6ms/step | 14.4 samples/s +Step 25940/44000 | Loss: 2.8876 | LR: 1.86e-04 | 1399.9ms/step | 14.3 samples/s +Step 25950/44000 | Loss: 2.8864 | LR: 1.86e-04 | 1398.0ms/step | 14.3 samples/s +Step 25960/44000 | Loss: 2.9061 | LR: 1.86e-04 | 1399.7ms/step | 14.3 samples/s +Step 25970/44000 | Loss: 2.8833 | LR: 1.85e-04 | 1388.7ms/step | 14.4 samples/s +Step 25980/44000 | Loss: 2.9266 | LR: 1.85e-04 | 1396.0ms/step | 14.3 samples/s +Step 25990/44000 | Loss: 2.9204 | LR: 1.85e-04 | 1397.7ms/step | 14.3 samples/s +Step 26000/44000 | Loss: 2.8970 | LR: 1.85e-04 | 1398.3ms/step | 14.3 samples/s +Validation at step 26000: Loss = 2.9248, Perplexity = 18.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-26000 +Step 26010/44000 | Loss: 2.9145 | LR: 1.85e-04 | 3892.9ms/step | 5.1 samples/s +Step 26020/44000 | Loss: 2.9139 | LR: 1.85e-04 | 1393.9ms/step | 14.3 samples/s +Step 26030/44000 | Loss: 2.8984 | LR: 1.84e-04 | 1395.2ms/step | 14.3 samples/s +Step 26040/44000 | Loss: 2.8859 | LR: 1.84e-04 | 1394.1ms/step | 14.3 samples/s +Step 26050/44000 | Loss: 2.9214 | LR: 1.84e-04 | 1397.7ms/step | 14.3 samples/s +Step 26060/44000 | Loss: 2.8710 | LR: 1.84e-04 | 1399.7ms/step | 14.3 samples/s +Step 26070/44000 | Loss: 2.9020 | LR: 1.84e-04 | 1395.2ms/step | 14.3 samples/s +Step 26080/44000 | Loss: 2.8878 | LR: 1.84e-04 | 1394.8ms/step | 14.3 samples/s +Step 26090/44000 | Loss: 2.9005 | LR: 1.83e-04 | 1394.5ms/step | 14.3 samples/s +Step 26100/44000 | Loss: 2.9035 | LR: 1.83e-04 | 1395.6ms/step | 14.3 samples/s +Step 26110/44000 | Loss: 2.9209 | LR: 1.83e-04 | 1398.3ms/step | 14.3 samples/s +Step 26120/44000 | Loss: 2.9016 | LR: 1.83e-04 | 1397.7ms/step | 14.3 samples/s +Step 26130/44000 | Loss: 2.9225 | LR: 1.83e-04 | 1394.4ms/step | 14.3 samples/s +Step 26140/44000 | Loss: 2.9261 | LR: 1.82e-04 | 1391.9ms/step | 14.4 samples/s +Step 26150/44000 | Loss: 2.9180 | LR: 1.82e-04 | 1394.8ms/step | 14.3 samples/s +Step 26160/44000 | Loss: 2.9012 | LR: 1.82e-04 | 1393.0ms/step | 14.4 samples/s +Step 26170/44000 | Loss: 2.9035 | LR: 1.82e-04 | 1394.3ms/step | 14.3 samples/s +Step 26180/44000 | Loss: 2.9180 | LR: 1.82e-04 | 1398.2ms/step | 14.3 samples/s +Step 26190/44000 | Loss: 2.8924 | LR: 1.82e-04 | 1393.9ms/step | 14.3 samples/s +Step 26200/44000 | Loss: 2.8713 | LR: 1.81e-04 | 1395.7ms/step | 14.3 samples/s +Step 26210/44000 | Loss: 2.9364 | LR: 1.81e-04 | 1392.4ms/step | 14.4 samples/s +Step 26220/44000 | Loss: 2.9106 | LR: 1.81e-04 | 1392.1ms/step | 14.4 samples/s +Step 26230/44000 | Loss: 2.9189 | LR: 1.81e-04 | 1393.1ms/step | 14.4 samples/s +Step 26240/44000 | Loss: 2.9116 | LR: 1.81e-04 | 1388.7ms/step | 14.4 samples/s +Step 26250/44000 | Loss: 2.9193 | LR: 1.81e-04 | 1389.0ms/step | 14.4 samples/s +Validation at step 26250: Loss = 2.9233, Perplexity = 18.60 +Step 26260/44000 | Loss: 2.9009 | LR: 1.80e-04 | 3593.1ms/step | 5.6 samples/s +Step 26270/44000 | Loss: 2.8985 | LR: 1.80e-04 | 1393.4ms/step | 14.4 samples/s +Step 26280/44000 | Loss: 2.9205 | LR: 1.80e-04 | 1394.3ms/step | 14.3 samples/s +Step 26290/44000 | Loss: 2.9336 | LR: 1.80e-04 | 1400.3ms/step | 14.3 samples/s +Step 26300/44000 | Loss: 2.9143 | LR: 1.80e-04 | 1398.1ms/step | 14.3 samples/s +Step 26310/44000 | Loss: 2.8988 | LR: 1.80e-04 | 1399.8ms/step | 14.3 samples/s +Step 26320/44000 | Loss: 2.9177 | LR: 1.79e-04 | 1394.6ms/step | 14.3 samples/s +Step 26330/44000 | Loss: 2.9500 | LR: 1.79e-04 | 1397.7ms/step | 14.3 samples/s +Step 26340/44000 | Loss: 2.9039 | LR: 1.79e-04 | 1397.9ms/step | 14.3 samples/s +Step 26350/44000 | Loss: 2.9201 | LR: 1.79e-04 | 1400.0ms/step | 14.3 samples/s +Step 26360/44000 | Loss: 2.8959 | LR: 1.79e-04 | 1400.3ms/step | 14.3 samples/s +Step 26370/44000 | Loss: 2.9208 | LR: 1.78e-04 | 1399.6ms/step | 14.3 samples/s +Step 26380/44000 | Loss: 2.9244 | LR: 1.78e-04 | 1400.3ms/step | 14.3 samples/s +Step 26390/44000 | Loss: 2.9244 | LR: 1.78e-04 | 1398.2ms/step | 14.3 samples/s +Step 26400/44000 | Loss: 2.9240 | LR: 1.78e-04 | 1395.6ms/step | 14.3 samples/s +Step 26410/44000 | Loss: 2.8906 | LR: 1.78e-04 | 1395.4ms/step | 14.3 samples/s +Step 26420/44000 | Loss: 2.8862 | LR: 1.78e-04 | 1392.1ms/step | 14.4 samples/s +Step 26430/44000 | Loss: 2.9273 | LR: 1.77e-04 | 1389.7ms/step | 14.4 samples/s +Step 26440/44000 | Loss: 2.8842 | LR: 1.77e-04 | 1390.2ms/step | 14.4 samples/s +Step 26450/44000 | Loss: 2.9076 | LR: 1.77e-04 | 1390.7ms/step | 14.4 samples/s +Step 26460/44000 | Loss: 2.9118 | LR: 1.77e-04 | 1394.0ms/step | 14.3 samples/s +Step 26470/44000 | Loss: 2.9036 | LR: 1.77e-04 | 1395.7ms/step | 14.3 samples/s +Step 26480/44000 | Loss: 2.9022 | LR: 1.77e-04 | 1394.3ms/step | 14.3 samples/s +Step 26490/44000 | Loss: 2.9380 | LR: 1.76e-04 | 1392.9ms/step | 14.4 samples/s +Step 26500/44000 | Loss: 2.9209 | LR: 1.76e-04 | 1395.1ms/step | 14.3 samples/s +Validation at step 26500: Loss = 2.9218, Perplexity = 18.57 +Step 26510/44000 | Loss: 2.9308 | LR: 1.76e-04 | 3602.2ms/step | 5.6 samples/s +Step 26520/44000 | Loss: 2.9071 | LR: 1.76e-04 | 1397.5ms/step | 14.3 samples/s +Step 26530/44000 | Loss: 2.9102 | LR: 1.76e-04 | 1397.7ms/step | 14.3 samples/s +Step 26540/44000 | Loss: 2.9176 | LR: 1.76e-04 | 1398.9ms/step | 14.3 samples/s +Step 26550/44000 | Loss: 2.9237 | LR: 1.75e-04 | 1395.6ms/step | 14.3 samples/s +Step 26560/44000 | Loss: 2.9559 | LR: 1.75e-04 | 1390.9ms/step | 14.4 samples/s +Step 26570/44000 | Loss: 2.8808 | LR: 1.75e-04 | 1391.6ms/step | 14.4 samples/s +Step 26580/44000 | Loss: 2.9338 | LR: 1.75e-04 | 1391.8ms/step | 14.4 samples/s +Step 26590/44000 | Loss: 2.9350 | LR: 1.75e-04 | 1397.8ms/step | 14.3 samples/s +Step 26600/44000 | Loss: 2.9075 | LR: 1.74e-04 | 1396.1ms/step | 14.3 samples/s +Step 26610/44000 | Loss: 2.9144 | LR: 1.74e-04 | 1395.6ms/step | 14.3 samples/s +Step 26620/44000 | Loss: 2.8944 | LR: 1.74e-04 | 1394.4ms/step | 14.3 samples/s +Step 26630/44000 | Loss: 2.9018 | LR: 1.74e-04 | 1397.8ms/step | 14.3 samples/s +Step 26640/44000 | Loss: 2.8969 | LR: 1.74e-04 | 1400.4ms/step | 14.3 samples/s +Step 26650/44000 | Loss: 2.9291 | LR: 1.74e-04 | 1399.9ms/step | 14.3 samples/s +Step 26660/44000 | Loss: 2.9173 | LR: 1.73e-04 | 1398.6ms/step | 14.3 samples/s +Step 26670/44000 | Loss: 2.9044 | LR: 1.73e-04 | 1395.0ms/step | 14.3 samples/s +Step 26680/44000 | Loss: 2.9200 | LR: 1.73e-04 | 1394.2ms/step | 14.3 samples/s +Step 26690/44000 | Loss: 2.9271 | LR: 1.73e-04 | 1395.4ms/step | 14.3 samples/s +Step 26700/44000 | Loss: 2.9221 | LR: 1.73e-04 | 1393.8ms/step | 14.3 samples/s +Step 26710/44000 | Loss: 2.8920 | LR: 1.73e-04 | 1398.8ms/step | 14.3 samples/s +Step 26720/44000 | Loss: 2.8998 | LR: 1.72e-04 | 1400.1ms/step | 14.3 samples/s +Step 26730/44000 | Loss: 2.8893 | LR: 1.72e-04 | 1398.1ms/step | 14.3 samples/s +Step 26740/44000 | Loss: 2.9113 | LR: 1.72e-04 | 1400.2ms/step | 14.3 samples/s +Step 26750/44000 | Loss: 2.9479 | LR: 1.72e-04 | 1399.8ms/step | 14.3 samples/s +Validation at step 26750: Loss = 2.9206, Perplexity = 18.55 +Step 26760/44000 | Loss: 2.9012 | LR: 1.72e-04 | 3626.1ms/step | 5.5 samples/s +Step 26770/44000 | Loss: 2.8781 | LR: 1.72e-04 | 1400.1ms/step | 14.3 samples/s +Step 26780/44000 | Loss: 2.8917 | LR: 1.71e-04 | 1400.1ms/step | 14.3 samples/s +Step 26790/44000 | Loss: 2.9041 | LR: 1.71e-04 | 1399.9ms/step | 14.3 samples/s +Step 26800/44000 | Loss: 2.9140 | LR: 1.71e-04 | 1400.0ms/step | 14.3 samples/s +Step 26810/44000 | Loss: 2.9145 | LR: 1.71e-04 | 1400.0ms/step | 14.3 samples/s +Step 26820/44000 | Loss: 2.9318 | LR: 1.71e-04 | 1399.9ms/step | 14.3 samples/s +Step 26830/44000 | Loss: 2.9066 | LR: 1.71e-04 | 1397.6ms/step | 14.3 samples/s +Step 26840/44000 | Loss: 2.9288 | LR: 1.70e-04 | 1396.2ms/step | 14.3 samples/s +Step 26850/44000 | Loss: 2.9058 | LR: 1.70e-04 | 1395.6ms/step | 14.3 samples/s +Step 26860/44000 | Loss: 2.8907 | LR: 1.70e-04 | 1396.8ms/step | 14.3 samples/s +Step 26870/44000 | Loss: 2.9175 | LR: 1.70e-04 | 1391.6ms/step | 14.4 samples/s +Step 26880/44000 | Loss: 2.8839 | LR: 1.70e-04 | 1392.0ms/step | 14.4 samples/s +Step 26890/44000 | Loss: 2.9282 | LR: 1.69e-04 | 1396.2ms/step | 14.3 samples/s +Step 26900/44000 | Loss: 2.8780 | LR: 1.69e-04 | 1399.5ms/step | 14.3 samples/s +Step 26910/44000 | Loss: 2.9213 | LR: 1.69e-04 | 1398.5ms/step | 14.3 samples/s +Step 26920/44000 | Loss: 2.8959 | LR: 1.69e-04 | 1397.9ms/step | 14.3 samples/s +Step 26930/44000 | Loss: 2.8979 | LR: 1.69e-04 | 1395.7ms/step | 14.3 samples/s +Step 26940/44000 | Loss: 2.9106 | LR: 1.69e-04 | 1392.7ms/step | 14.4 samples/s +Step 26950/44000 | Loss: 2.9137 | LR: 1.68e-04 | 1388.8ms/step | 14.4 samples/s +Step 26960/44000 | Loss: 2.8968 | LR: 1.68e-04 | 1389.1ms/step | 14.4 samples/s +Step 26970/44000 | Loss: 2.8994 | LR: 1.68e-04 | 1392.3ms/step | 14.4 samples/s +Step 26980/44000 | Loss: 2.9331 | LR: 1.68e-04 | 1389.3ms/step | 14.4 samples/s +Step 26990/44000 | Loss: 2.8933 | LR: 1.68e-04 | 1389.5ms/step | 14.4 samples/s +Step 27000/44000 | Loss: 2.9091 | LR: 1.68e-04 | 1389.4ms/step | 14.4 samples/s +Validation at step 27000: Loss = 2.9189, Perplexity = 18.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-27000 +Step 27010/44000 | Loss: 2.9235 | LR: 1.67e-04 | 3895.5ms/step | 5.1 samples/s +Step 27020/44000 | Loss: 2.9041 | LR: 1.67e-04 | 1391.4ms/step | 14.4 samples/s +Step 27030/44000 | Loss: 2.9231 | LR: 1.67e-04 | 1394.6ms/step | 14.3 samples/s +Step 27040/44000 | Loss: 2.9103 | LR: 1.67e-04 | 1395.7ms/step | 14.3 samples/s +Step 27050/44000 | Loss: 2.9198 | LR: 1.67e-04 | 1396.1ms/step | 14.3 samples/s +Step 27060/44000 | Loss: 2.8789 | LR: 1.67e-04 | 1397.5ms/step | 14.3 samples/s +Step 27070/44000 | Loss: 2.9148 | LR: 1.66e-04 | 1390.7ms/step | 14.4 samples/s +Step 27080/44000 | Loss: 2.9162 | LR: 1.66e-04 | 1389.9ms/step | 14.4 samples/s +Step 27090/44000 | Loss: 2.9004 | LR: 1.66e-04 | 1393.5ms/step | 14.4 samples/s +Step 27100/44000 | Loss: 2.8901 | LR: 1.66e-04 | 1396.5ms/step | 14.3 samples/s +Step 27110/44000 | Loss: 2.8843 | LR: 1.66e-04 | 1397.9ms/step | 14.3 samples/s +Step 27120/44000 | Loss: 2.8913 | LR: 1.66e-04 | 1396.0ms/step | 14.3 samples/s +Step 27130/44000 | Loss: 2.8839 | LR: 1.65e-04 | 1396.2ms/step | 14.3 samples/s +Step 27140/44000 | Loss: 2.8987 | LR: 1.65e-04 | 1397.7ms/step | 14.3 samples/s +Step 27150/44000 | Loss: 2.9174 | LR: 1.65e-04 | 1400.2ms/step | 14.3 samples/s +Step 27160/44000 | Loss: 2.8819 | LR: 1.65e-04 | 1397.4ms/step | 14.3 samples/s +Step 27170/44000 | Loss: 2.9104 | LR: 1.65e-04 | 1398.4ms/step | 14.3 samples/s +Step 27180/44000 | Loss: 2.9169 | LR: 1.65e-04 | 1400.0ms/step | 14.3 samples/s +Step 27190/44000 | Loss: 2.9152 | LR: 1.64e-04 | 1394.2ms/step | 14.3 samples/s +Step 27200/44000 | Loss: 2.9194 | LR: 1.64e-04 | 1395.8ms/step | 14.3 samples/s +Step 27210/44000 | Loss: 2.9048 | LR: 1.64e-04 | 1400.2ms/step | 14.3 samples/s +Step 27220/44000 | Loss: 2.8873 | LR: 1.64e-04 | 1399.7ms/step | 14.3 samples/s +Step 27230/44000 | Loss: 2.8812 | LR: 1.64e-04 | 1396.4ms/step | 14.3 samples/s +Step 27240/44000 | Loss: 2.9145 | LR: 1.63e-04 | 1393.7ms/step | 14.4 samples/s +Step 27250/44000 | Loss: 2.8836 | LR: 1.63e-04 | 1392.0ms/step | 14.4 samples/s +Validation at step 27250: Loss = 2.9180, Perplexity = 18.50 +Step 27260/44000 | Loss: 2.9237 | LR: 1.63e-04 | 3600.3ms/step | 5.6 samples/s +Step 27270/44000 | Loss: 2.9340 | LR: 1.63e-04 | 1399.9ms/step | 14.3 samples/s +Step 27280/44000 | Loss: 2.9246 | LR: 1.63e-04 | 1398.0ms/step | 14.3 samples/s +Step 27290/44000 | Loss: 2.8937 | LR: 1.63e-04 | 1397.9ms/step | 14.3 samples/s +Step 27300/44000 | Loss: 2.8806 | LR: 1.62e-04 | 1398.2ms/step | 14.3 samples/s +Step 27310/44000 | Loss: 2.9130 | LR: 1.62e-04 | 1393.9ms/step | 14.3 samples/s +Step 27320/44000 | Loss: 2.9203 | LR: 1.62e-04 | 1391.9ms/step | 14.4 samples/s +Step 27330/44000 | Loss: 2.9232 | LR: 1.62e-04 | 1393.9ms/step | 14.3 samples/s +Step 27340/44000 | Loss: 2.8789 | LR: 1.62e-04 | 1395.7ms/step | 14.3 samples/s +Step 27350/44000 | Loss: 2.8767 | LR: 1.62e-04 | 1396.3ms/step | 14.3 samples/s +Step 27360/44000 | Loss: 2.8841 | LR: 1.61e-04 | 1394.3ms/step | 14.3 samples/s +Step 27370/44000 | Loss: 2.8919 | LR: 1.61e-04 | 1392.5ms/step | 14.4 samples/s +Step 27380/44000 | Loss: 2.9349 | LR: 1.61e-04 | 1392.7ms/step | 14.4 samples/s +Step 27390/44000 | Loss: 2.8930 | LR: 1.61e-04 | 1392.5ms/step | 14.4 samples/s +Step 27400/44000 | Loss: 2.9209 | LR: 1.61e-04 | 1394.2ms/step | 14.3 samples/s +Step 27410/44000 | Loss: 2.9024 | LR: 1.61e-04 | 1400.0ms/step | 14.3 samples/s +Step 27420/44000 | Loss: 2.8972 | LR: 1.60e-04 | 1400.1ms/step | 14.3 samples/s +Step 27430/44000 | Loss: 2.9191 | LR: 1.60e-04 | 1395.6ms/step | 14.3 samples/s +Step 27440/44000 | Loss: 2.9091 | LR: 1.60e-04 | 1390.5ms/step | 14.4 samples/s +Step 27450/44000 | Loss: 2.8764 | LR: 1.60e-04 | 1390.6ms/step | 14.4 samples/s +Step 27460/44000 | Loss: 2.8966 | LR: 1.60e-04 | 1391.2ms/step | 14.4 samples/s +Step 27470/44000 | Loss: 2.9285 | LR: 1.60e-04 | 1391.6ms/step | 14.4 samples/s +Step 27480/44000 | Loss: 2.8995 | LR: 1.59e-04 | 1392.5ms/step | 14.4 samples/s +Step 27490/44000 | Loss: 2.9188 | LR: 1.59e-04 | 1399.4ms/step | 14.3 samples/s +Step 27500/44000 | Loss: 2.9074 | LR: 1.59e-04 | 1394.5ms/step | 14.3 samples/s +Validation at step 27500: Loss = 2.9165, Perplexity = 18.48 +Step 27510/44000 | Loss: 2.9412 | LR: 1.59e-04 | 3596.0ms/step | 5.6 samples/s +Step 27520/44000 | Loss: 2.9002 | LR: 1.59e-04 | 1400.2ms/step | 14.3 samples/s +Step 27530/44000 | Loss: 2.9143 | LR: 1.59e-04 | 1397.8ms/step | 14.3 samples/s +Step 27540/44000 | Loss: 2.8718 | LR: 1.58e-04 | 1391.5ms/step | 14.4 samples/s +Step 27550/44000 | Loss: 2.8724 | LR: 1.58e-04 | 1390.0ms/step | 14.4 samples/s +Step 27560/44000 | Loss: 2.8877 | LR: 1.58e-04 | 1390.3ms/step | 14.4 samples/s +Step 27570/44000 | Loss: 2.8779 | LR: 1.58e-04 | 1389.4ms/step | 14.4 samples/s +Step 27580/44000 | Loss: 2.9285 | LR: 1.58e-04 | 1388.2ms/step | 14.4 samples/s +Step 27590/44000 | Loss: 2.8948 | LR: 1.58e-04 | 1388.7ms/step | 14.4 samples/s +Step 27600/44000 | Loss: 2.9087 | LR: 1.57e-04 | 1395.9ms/step | 14.3 samples/s +Step 27610/44000 | Loss: 2.8776 | LR: 1.57e-04 | 1397.9ms/step | 14.3 samples/s +Step 27620/44000 | Loss: 2.9102 | LR: 1.57e-04 | 1398.2ms/step | 14.3 samples/s +Step 27630/44000 | Loss: 2.8763 | LR: 1.57e-04 | 1399.5ms/step | 14.3 samples/s +Step 27640/44000 | Loss: 2.9382 | LR: 1.57e-04 | 1389.7ms/step | 14.4 samples/s +Step 27650/44000 | Loss: 2.9580 | LR: 1.57e-04 | 1389.1ms/step | 14.4 samples/s +Step 27660/44000 | Loss: 2.9066 | LR: 1.56e-04 | 1387.7ms/step | 14.4 samples/s +Step 27670/44000 | Loss: 2.8970 | LR: 1.56e-04 | 1389.2ms/step | 14.4 samples/s +Step 27680/44000 | Loss: 2.9114 | LR: 1.56e-04 | 1387.7ms/step | 14.4 samples/s +Step 27690/44000 | Loss: 2.9010 | LR: 1.56e-04 | 1391.3ms/step | 14.4 samples/s +Step 27700/44000 | Loss: 2.8925 | LR: 1.56e-04 | 1395.6ms/step | 14.3 samples/s +Step 27710/44000 | Loss: 2.9121 | LR: 1.56e-04 | 1398.4ms/step | 14.3 samples/s +Step 27720/44000 | Loss: 2.8965 | LR: 1.55e-04 | 1398.8ms/step | 14.3 samples/s +Step 27730/44000 | Loss: 2.8980 | LR: 1.55e-04 | 1389.0ms/step | 14.4 samples/s +Step 27740/44000 | Loss: 2.8959 | LR: 1.55e-04 | 1387.9ms/step | 14.4 samples/s +Step 27750/44000 | Loss: 2.8958 | LR: 1.55e-04 | 1387.8ms/step | 14.4 samples/s +Validation at step 27750: Loss = 2.9154, Perplexity = 18.46 +Step 27760/44000 | Loss: 2.9012 | LR: 1.55e-04 | 3592.5ms/step | 5.6 samples/s +Step 27770/44000 | Loss: 2.8967 | LR: 1.55e-04 | 1393.8ms/step | 14.3 samples/s +Step 27780/44000 | Loss: 2.8962 | LR: 1.54e-04 | 1394.0ms/step | 14.3 samples/s +Step 27790/44000 | Loss: 2.8585 | LR: 1.54e-04 | 1393.6ms/step | 14.4 samples/s +Step 27800/44000 | Loss: 2.8905 | LR: 1.54e-04 | 1389.6ms/step | 14.4 samples/s +Step 27810/44000 | Loss: 2.8622 | LR: 1.54e-04 | 1388.6ms/step | 14.4 samples/s +Step 27820/44000 | Loss: 2.8871 | LR: 1.54e-04 | 1388.4ms/step | 14.4 samples/s +Step 27830/44000 | Loss: 2.9046 | LR: 1.54e-04 | 1388.1ms/step | 14.4 samples/s +Step 27840/44000 | Loss: 2.9178 | LR: 1.53e-04 | 1389.9ms/step | 14.4 samples/s +Step 27850/44000 | Loss: 2.8856 | LR: 1.53e-04 | 1387.9ms/step | 14.4 samples/s +Step 27860/44000 | Loss: 2.8940 | LR: 1.53e-04 | 1388.4ms/step | 14.4 samples/s +Step 27870/44000 | Loss: 2.9252 | LR: 1.53e-04 | 1388.8ms/step | 14.4 samples/s +Step 27880/44000 | Loss: 2.8966 | LR: 1.53e-04 | 1390.8ms/step | 14.4 samples/s +Step 27890/44000 | Loss: 2.8754 | LR: 1.53e-04 | 1392.1ms/step | 14.4 samples/s +Step 27900/44000 | Loss: 2.8812 | LR: 1.52e-04 | 1400.1ms/step | 14.3 samples/s +Step 27910/44000 | Loss: 2.8949 | LR: 1.52e-04 | 1399.8ms/step | 14.3 samples/s +Step 27920/44000 | Loss: 2.9336 | LR: 1.52e-04 | 1400.2ms/step | 14.3 samples/s +Step 27930/44000 | Loss: 2.9093 | LR: 1.52e-04 | 1397.7ms/step | 14.3 samples/s +Step 27940/44000 | Loss: 2.9004 | LR: 1.52e-04 | 1400.2ms/step | 14.3 samples/s +Step 27950/44000 | Loss: 2.9205 | LR: 1.52e-04 | 1400.0ms/step | 14.3 samples/s +Step 27960/44000 | Loss: 2.9035 | LR: 1.51e-04 | 1397.8ms/step | 14.3 samples/s +Step 27970/44000 | Loss: 2.8968 | LR: 1.51e-04 | 1396.1ms/step | 14.3 samples/s +Step 27980/44000 | Loss: 2.9033 | LR: 1.51e-04 | 1400.2ms/step | 14.3 samples/s +Step 27990/44000 | Loss: 2.8942 | LR: 1.51e-04 | 1399.9ms/step | 14.3 samples/s +Step 28000/44000 | Loss: 2.8970 | LR: 1.51e-04 | 1400.0ms/step | 14.3 samples/s +Validation at step 28000: Loss = 2.9140, Perplexity = 18.43 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-28000 +Step 28010/44000 | Loss: 2.8974 | LR: 1.51e-04 | 3932.1ms/step | 5.1 samples/s +Step 28020/44000 | Loss: 2.9351 | LR: 1.50e-04 | 1398.0ms/step | 14.3 samples/s +Step 28030/44000 | Loss: 2.8926 | LR: 1.50e-04 | 1398.0ms/step | 14.3 samples/s +Step 28040/44000 | Loss: 2.9130 | LR: 1.50e-04 | 1399.7ms/step | 14.3 samples/s +Step 28050/44000 | Loss: 2.9189 | LR: 1.50e-04 | 1399.5ms/step | 14.3 samples/s +Step 28060/44000 | Loss: 2.9232 | LR: 1.50e-04 | 1398.6ms/step | 14.3 samples/s +Step 28070/44000 | Loss: 2.8943 | LR: 1.50e-04 | 1397.9ms/step | 14.3 samples/s +Step 28080/44000 | Loss: 2.8799 | LR: 1.49e-04 | 1394.3ms/step | 14.3 samples/s +Step 28090/44000 | Loss: 2.9047 | LR: 1.49e-04 | 1391.9ms/step | 14.4 samples/s +Step 28100/44000 | Loss: 2.8972 | LR: 1.49e-04 | 1393.6ms/step | 14.4 samples/s +Step 28110/44000 | Loss: 2.8769 | LR: 1.49e-04 | 1392.5ms/step | 14.4 samples/s +Step 28120/44000 | Loss: 2.9076 | LR: 1.49e-04 | 1397.9ms/step | 14.3 samples/s +Step 28130/44000 | Loss: 2.9052 | LR: 1.49e-04 | 1394.1ms/step | 14.3 samples/s +Step 28140/44000 | Loss: 2.9019 | LR: 1.48e-04 | 1393.7ms/step | 14.4 samples/s +Step 28150/44000 | Loss: 2.8943 | LR: 1.48e-04 | 1396.2ms/step | 14.3 samples/s +Step 28160/44000 | Loss: 2.9082 | LR: 1.48e-04 | 1396.1ms/step | 14.3 samples/s +Step 28170/44000 | Loss: 2.8817 | LR: 1.48e-04 | 1400.1ms/step | 14.3 samples/s +Step 28180/44000 | Loss: 2.9199 | LR: 1.48e-04 | 1399.6ms/step | 14.3 samples/s +Step 28190/44000 | Loss: 2.9111 | LR: 1.48e-04 | 1396.1ms/step | 14.3 samples/s +Step 28200/44000 | Loss: 2.9197 | LR: 1.47e-04 | 1398.2ms/step | 14.3 samples/s +Step 28210/44000 | Loss: 2.8673 | LR: 1.47e-04 | 1398.1ms/step | 14.3 samples/s +Step 28220/44000 | Loss: 2.8850 | LR: 1.47e-04 | 1395.6ms/step | 14.3 samples/s +Step 28230/44000 | Loss: 2.9060 | LR: 1.47e-04 | 1396.2ms/step | 14.3 samples/s +Step 28240/44000 | Loss: 2.8686 | LR: 1.47e-04 | 1396.1ms/step | 14.3 samples/s +Step 28250/44000 | Loss: 2.8809 | LR: 1.47e-04 | 1400.2ms/step | 14.3 samples/s +Validation at step 28250: Loss = 2.9133, Perplexity = 18.42 +Step 28260/44000 | Loss: 2.9021 | LR: 1.46e-04 | 3594.1ms/step | 5.6 samples/s +Step 28270/44000 | Loss: 2.8857 | LR: 1.46e-04 | 1395.8ms/step | 14.3 samples/s +Step 28280/44000 | Loss: 2.9273 | LR: 1.46e-04 | 1396.1ms/step | 14.3 samples/s +Step 28290/44000 | Loss: 2.8567 | LR: 1.46e-04 | 1392.0ms/step | 14.4 samples/s +Step 28300/44000 | Loss: 2.8968 | LR: 1.46e-04 | 1395.8ms/step | 14.3 samples/s +Step 28310/44000 | Loss: 2.8731 | LR: 1.46e-04 | 1391.9ms/step | 14.4 samples/s +Step 28320/44000 | Loss: 2.9352 | LR: 1.45e-04 | 1389.9ms/step | 14.4 samples/s +Step 28330/44000 | Loss: 2.8976 | LR: 1.45e-04 | 1390.6ms/step | 14.4 samples/s +Step 28340/44000 | Loss: 2.9092 | LR: 1.45e-04 | 1388.4ms/step | 14.4 samples/s +Step 28350/44000 | Loss: 2.8877 | LR: 1.45e-04 | 1391.6ms/step | 14.4 samples/s +Step 28360/44000 | Loss: 2.8947 | LR: 1.45e-04 | 1394.1ms/step | 14.3 samples/s +Step 28370/44000 | Loss: 2.8726 | LR: 1.45e-04 | 1397.9ms/step | 14.3 samples/s +Step 28380/44000 | Loss: 2.8797 | LR: 1.44e-04 | 1400.2ms/step | 14.3 samples/s +Step 28390/44000 | Loss: 2.8955 | LR: 1.44e-04 | 1399.8ms/step | 14.3 samples/s +Step 28400/44000 | Loss: 2.8953 | LR: 1.44e-04 | 1400.0ms/step | 14.3 samples/s +Step 28410/44000 | Loss: 2.9210 | LR: 1.44e-04 | 1398.4ms/step | 14.3 samples/s +Step 28420/44000 | Loss: 2.9421 | LR: 1.44e-04 | 1398.6ms/step | 14.3 samples/s +Step 28430/44000 | Loss: 2.8663 | LR: 1.44e-04 | 1394.9ms/step | 14.3 samples/s +Step 28440/44000 | Loss: 2.9064 | LR: 1.43e-04 | 1395.8ms/step | 14.3 samples/s +Step 28450/44000 | Loss: 2.9033 | LR: 1.43e-04 | 1396.3ms/step | 14.3 samples/s +Step 28460/44000 | Loss: 2.9122 | LR: 1.43e-04 | 1391.9ms/step | 14.4 samples/s +Step 28470/44000 | Loss: 2.9107 | LR: 1.43e-04 | 1395.7ms/step | 14.3 samples/s +Step 28480/44000 | Loss: 2.8886 | LR: 1.43e-04 | 1393.1ms/step | 14.4 samples/s +Step 28490/44000 | Loss: 2.8970 | LR: 1.43e-04 | 1394.7ms/step | 14.3 samples/s +Step 28500/44000 | Loss: 2.8983 | LR: 1.42e-04 | 1398.5ms/step | 14.3 samples/s +Validation at step 28500: Loss = 2.9119, Perplexity = 18.39 +Step 28510/44000 | Loss: 2.9224 | LR: 1.42e-04 | 3607.9ms/step | 5.5 samples/s +Step 28520/44000 | Loss: 2.9013 | LR: 1.42e-04 | 1400.0ms/step | 14.3 samples/s +Step 28530/44000 | Loss: 2.8905 | LR: 1.42e-04 | 1400.0ms/step | 14.3 samples/s +Step 28540/44000 | Loss: 2.9161 | LR: 1.42e-04 | 1400.0ms/step | 14.3 samples/s +Step 28550/44000 | Loss: 2.9340 | LR: 1.42e-04 | 1394.0ms/step | 14.3 samples/s +Step 28560/44000 | Loss: 2.8836 | LR: 1.41e-04 | 1396.0ms/step | 14.3 samples/s +Step 28570/44000 | Loss: 2.8767 | LR: 1.41e-04 | 1400.1ms/step | 14.3 samples/s +Step 28580/44000 | Loss: 2.9087 | LR: 1.41e-04 | 1399.7ms/step | 14.3 samples/s +Step 28590/44000 | Loss: 2.9253 | LR: 1.41e-04 | 1394.6ms/step | 14.3 samples/s +Step 28600/44000 | Loss: 2.9251 | LR: 1.41e-04 | 1395.7ms/step | 14.3 samples/s +Step 28610/44000 | Loss: 2.8897 | LR: 1.41e-04 | 1396.2ms/step | 14.3 samples/s +Step 28620/44000 | Loss: 2.8747 | LR: 1.40e-04 | 1389.7ms/step | 14.4 samples/s +Step 28630/44000 | Loss: 2.8854 | LR: 1.40e-04 | 1390.0ms/step | 14.4 samples/s +Step 28640/44000 | Loss: 2.8663 | LR: 1.40e-04 | 1391.3ms/step | 14.4 samples/s +Step 28650/44000 | Loss: 2.8903 | LR: 1.40e-04 | 1388.5ms/step | 14.4 samples/s +Step 28660/44000 | Loss: 2.8862 | LR: 1.40e-04 | 1392.0ms/step | 14.4 samples/s +Step 28670/44000 | Loss: 2.8569 | LR: 1.40e-04 | 1388.5ms/step | 14.4 samples/s +Step 28680/44000 | Loss: 2.9072 | LR: 1.39e-04 | 1390.8ms/step | 14.4 samples/s +Step 28690/44000 | Loss: 2.9017 | LR: 1.39e-04 | 1389.0ms/step | 14.4 samples/s +Step 28700/44000 | Loss: 2.9065 | LR: 1.39e-04 | 1393.7ms/step | 14.4 samples/s +Step 28710/44000 | Loss: 2.9117 | LR: 1.39e-04 | 1397.9ms/step | 14.3 samples/s +Step 28720/44000 | Loss: 2.8858 | LR: 1.39e-04 | 1394.4ms/step | 14.3 samples/s +Step 28730/44000 | Loss: 2.8890 | LR: 1.39e-04 | 1397.6ms/step | 14.3 samples/s +Step 28740/44000 | Loss: 2.9125 | LR: 1.38e-04 | 1392.1ms/step | 14.4 samples/s +Step 28750/44000 | Loss: 2.8752 | LR: 1.38e-04 | 1391.1ms/step | 14.4 samples/s +Validation at step 28750: Loss = 2.9110, Perplexity = 18.37 +Step 28760/44000 | Loss: 2.9018 | LR: 1.38e-04 | 3595.1ms/step | 5.6 samples/s +Step 28770/44000 | Loss: 2.8703 | LR: 1.38e-04 | 1400.0ms/step | 14.3 samples/s +Step 28780/44000 | Loss: 2.9032 | LR: 1.38e-04 | 1400.0ms/step | 14.3 samples/s +Step 28790/44000 | Loss: 2.9117 | LR: 1.38e-04 | 1393.8ms/step | 14.3 samples/s +Step 28800/44000 | Loss: 2.9156 | LR: 1.38e-04 | 1389.8ms/step | 14.4 samples/s +Step 28810/44000 | Loss: 2.9016 | LR: 1.37e-04 | 1395.8ms/step | 14.3 samples/s +Step 28820/44000 | Loss: 2.8758 | LR: 1.37e-04 | 1396.5ms/step | 14.3 samples/s +Step 28830/44000 | Loss: 2.9072 | LR: 1.37e-04 | 1397.6ms/step | 14.3 samples/s +Step 28840/44000 | Loss: 2.9017 | LR: 1.37e-04 | 1394.6ms/step | 14.3 samples/s +Step 28850/44000 | Loss: 2.8794 | LR: 1.37e-04 | 1399.8ms/step | 14.3 samples/s +Step 28860/44000 | Loss: 2.9107 | LR: 1.37e-04 | 1398.3ms/step | 14.3 samples/s +Step 28870/44000 | Loss: 2.9165 | LR: 1.36e-04 | 1399.8ms/step | 14.3 samples/s +Step 28880/44000 | Loss: 2.8895 | LR: 1.36e-04 | 1397.9ms/step | 14.3 samples/s +Step 28890/44000 | Loss: 2.9010 | LR: 1.36e-04 | 1398.3ms/step | 14.3 samples/s +Step 28900/44000 | Loss: 2.8965 | LR: 1.36e-04 | 1400.0ms/step | 14.3 samples/s +Step 28910/44000 | Loss: 2.8921 | LR: 1.36e-04 | 1400.0ms/step | 14.3 samples/s +Step 28920/44000 | Loss: 2.9057 | LR: 1.36e-04 | 1399.7ms/step | 14.3 samples/s +Step 28930/44000 | Loss: 2.8979 | LR: 1.35e-04 | 1396.2ms/step | 14.3 samples/s +Step 28940/44000 | Loss: 2.8917 | LR: 1.35e-04 | 1392.2ms/step | 14.4 samples/s +Step 28950/44000 | Loss: 2.9351 | LR: 1.35e-04 | 1390.8ms/step | 14.4 samples/s +Step 28960/44000 | Loss: 2.8947 | LR: 1.35e-04 | 1388.7ms/step | 14.4 samples/s +Step 28970/44000 | Loss: 2.9184 | LR: 1.35e-04 | 1392.2ms/step | 14.4 samples/s +Step 28980/44000 | Loss: 2.9022 | LR: 1.35e-04 | 1390.3ms/step | 14.4 samples/s +Step 28990/44000 | Loss: 2.8920 | LR: 1.34e-04 | 1391.7ms/step | 14.4 samples/s +Step 29000/44000 | Loss: 2.8893 | LR: 1.34e-04 | 1392.3ms/step | 14.4 samples/s +Validation at step 29000: Loss = 2.9097, Perplexity = 18.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-29000 +Step 29010/44000 | Loss: 2.8852 | LR: 1.34e-04 | 3895.9ms/step | 5.1 samples/s +Step 29020/44000 | Loss: 2.8962 | LR: 1.34e-04 | 1389.9ms/step | 14.4 samples/s +Step 29030/44000 | Loss: 2.8845 | LR: 1.34e-04 | 1398.1ms/step | 14.3 samples/s +Step 29040/44000 | Loss: 2.8710 | LR: 1.34e-04 | 1392.2ms/step | 14.4 samples/s +Step 29050/44000 | Loss: 2.8772 | LR: 1.33e-04 | 1391.3ms/step | 14.4 samples/s +Step 29060/44000 | Loss: 2.9065 | LR: 1.33e-04 | 1396.7ms/step | 14.3 samples/s +Step 29070/44000 | Loss: 2.9014 | LR: 1.33e-04 | 1395.7ms/step | 14.3 samples/s +Step 29080/44000 | Loss: 2.8698 | LR: 1.33e-04 | 1398.2ms/step | 14.3 samples/s +Step 29090/44000 | Loss: 2.9031 | LR: 1.33e-04 | 1400.1ms/step | 14.3 samples/s +Step 29100/44000 | Loss: 2.9154 | LR: 1.33e-04 | 1398.0ms/step | 14.3 samples/s +Step 29110/44000 | Loss: 2.9105 | LR: 1.33e-04 | 1392.1ms/step | 14.4 samples/s +Step 29120/44000 | Loss: 2.8799 | LR: 1.32e-04 | 1393.8ms/step | 14.3 samples/s +Step 29130/44000 | Loss: 2.9063 | LR: 1.32e-04 | 1387.6ms/step | 14.4 samples/s +Step 29140/44000 | Loss: 2.8985 | LR: 1.32e-04 | 1387.6ms/step | 14.4 samples/s +Step 29150/44000 | Loss: 2.9169 | LR: 1.32e-04 | 1388.8ms/step | 14.4 samples/s +Step 29160/44000 | Loss: 2.8925 | LR: 1.32e-04 | 1393.6ms/step | 14.4 samples/s +Step 29170/44000 | Loss: 2.8713 | LR: 1.32e-04 | 1398.3ms/step | 14.3 samples/s +Step 29180/44000 | Loss: 2.8724 | LR: 1.31e-04 | 1395.1ms/step | 14.3 samples/s +Step 29190/44000 | Loss: 2.8870 | LR: 1.31e-04 | 1391.4ms/step | 14.4 samples/s +Step 29200/44000 | Loss: 2.8935 | LR: 1.31e-04 | 1391.4ms/step | 14.4 samples/s +Step 29210/44000 | Loss: 2.8479 | LR: 1.31e-04 | 1396.3ms/step | 14.3 samples/s +Step 29220/44000 | Loss: 2.8913 | LR: 1.31e-04 | 1395.9ms/step | 14.3 samples/s +Step 29230/44000 | Loss: 2.8808 | LR: 1.31e-04 | 1400.1ms/step | 14.3 samples/s +Step 29240/44000 | Loss: 2.9075 | LR: 1.30e-04 | 1400.1ms/step | 14.3 samples/s +Step 29250/44000 | Loss: 2.8886 | LR: 1.30e-04 | 1398.0ms/step | 14.3 samples/s +Validation at step 29250: Loss = 2.9090, Perplexity = 18.34 +Step 29260/44000 | Loss: 2.8756 | LR: 1.30e-04 | 3630.0ms/step | 5.5 samples/s +Step 29270/44000 | Loss: 2.8935 | LR: 1.30e-04 | 1400.0ms/step | 14.3 samples/s +Step 29280/44000 | Loss: 2.8845 | LR: 1.30e-04 | 1400.1ms/step | 14.3 samples/s +Step 29290/44000 | Loss: 2.9005 | LR: 1.30e-04 | 1399.9ms/step | 14.3 samples/s +Step 29300/44000 | Loss: 2.9041 | LR: 1.29e-04 | 1399.9ms/step | 14.3 samples/s +Step 29310/44000 | Loss: 2.9089 | LR: 1.29e-04 | 1399.8ms/step | 14.3 samples/s +Step 29320/44000 | Loss: 2.9046 | LR: 1.29e-04 | 1400.2ms/step | 14.3 samples/s +Step 29330/44000 | Loss: 2.9104 | LR: 1.29e-04 | 1400.0ms/step | 14.3 samples/s +Step 29340/44000 | Loss: 2.8655 | LR: 1.29e-04 | 1400.1ms/step | 14.3 samples/s +Step 29350/44000 | Loss: 2.9094 | LR: 1.29e-04 | 1399.9ms/step | 14.3 samples/s +Step 29360/44000 | Loss: 2.9130 | LR: 1.29e-04 | 1396.7ms/step | 14.3 samples/s +Step 29370/44000 | Loss: 2.8849 | LR: 1.28e-04 | 1392.9ms/step | 14.4 samples/s +Step 29380/44000 | Loss: 2.8934 | LR: 1.28e-04 | 1394.2ms/step | 14.3 samples/s +Step 29390/44000 | Loss: 2.9114 | LR: 1.28e-04 | 1394.0ms/step | 14.3 samples/s +Step 29400/44000 | Loss: 2.8955 | LR: 1.28e-04 | 1392.2ms/step | 14.4 samples/s +Step 29410/44000 | Loss: 2.8860 | LR: 1.28e-04 | 1395.8ms/step | 14.3 samples/s +Step 29420/44000 | Loss: 2.9265 | LR: 1.28e-04 | 1394.1ms/step | 14.3 samples/s +Step 29430/44000 | Loss: 2.8997 | LR: 1.27e-04 | 1396.2ms/step | 14.3 samples/s +Step 29440/44000 | Loss: 2.8995 | LR: 1.27e-04 | 1397.9ms/step | 14.3 samples/s +Step 29450/44000 | Loss: 2.9189 | LR: 1.27e-04 | 1400.1ms/step | 14.3 samples/s +Step 29460/44000 | Loss: 2.9052 | LR: 1.27e-04 | 1399.3ms/step | 14.3 samples/s +Step 29470/44000 | Loss: 2.9008 | LR: 1.27e-04 | 1391.9ms/step | 14.4 samples/s +Step 29480/44000 | Loss: 2.8977 | LR: 1.27e-04 | 1394.4ms/step | 14.3 samples/s +Step 29490/44000 | Loss: 2.8879 | LR: 1.26e-04 | 1400.3ms/step | 14.3 samples/s +Step 29500/44000 | Loss: 2.8824 | LR: 1.26e-04 | 1397.9ms/step | 14.3 samples/s +Validation at step 29500: Loss = 2.9081, Perplexity = 18.32 +Step 29510/44000 | Loss: 2.8889 | LR: 1.26e-04 | 3604.1ms/step | 5.5 samples/s +Step 29520/44000 | Loss: 2.9010 | LR: 1.26e-04 | 1395.9ms/step | 14.3 samples/s +Step 29530/44000 | Loss: 2.8806 | LR: 1.26e-04 | 1397.7ms/step | 14.3 samples/s +Step 29540/44000 | Loss: 2.8761 | LR: 1.26e-04 | 1393.9ms/step | 14.3 samples/s +Step 29550/44000 | Loss: 2.8842 | LR: 1.26e-04 | 1393.7ms/step | 14.4 samples/s +Step 29560/44000 | Loss: 2.8832 | LR: 1.25e-04 | 1394.9ms/step | 14.3 samples/s +Step 29570/44000 | Loss: 2.8929 | LR: 1.25e-04 | 1399.5ms/step | 14.3 samples/s +Step 29580/44000 | Loss: 2.8824 | LR: 1.25e-04 | 1394.4ms/step | 14.3 samples/s +Step 29590/44000 | Loss: 2.9142 | LR: 1.25e-04 | 1397.3ms/step | 14.3 samples/s +Step 29600/44000 | Loss: 2.9022 | LR: 1.25e-04 | 1396.8ms/step | 14.3 samples/s +Step 29610/44000 | Loss: 2.8922 | LR: 1.25e-04 | 1399.9ms/step | 14.3 samples/s +Step 29620/44000 | Loss: 2.8757 | LR: 1.24e-04 | 1400.1ms/step | 14.3 samples/s +Step 29630/44000 | Loss: 2.9179 | LR: 1.24e-04 | 1400.0ms/step | 14.3 samples/s +Step 29640/44000 | Loss: 2.8878 | LR: 1.24e-04 | 1394.9ms/step | 14.3 samples/s +Step 29650/44000 | Loss: 2.8802 | LR: 1.24e-04 | 1388.9ms/step | 14.4 samples/s +Step 29660/44000 | Loss: 2.9360 | LR: 1.24e-04 | 1392.2ms/step | 14.4 samples/s +Step 29670/44000 | Loss: 2.8788 | LR: 1.24e-04 | 1399.9ms/step | 14.3 samples/s +Step 29680/44000 | Loss: 2.8905 | LR: 1.23e-04 | 1397.8ms/step | 14.3 samples/s +Step 29690/44000 | Loss: 2.8989 | LR: 1.23e-04 | 1396.1ms/step | 14.3 samples/s +Step 29700/44000 | Loss: 2.9016 | LR: 1.23e-04 | 1398.4ms/step | 14.3 samples/s +Step 29710/44000 | Loss: 2.9129 | LR: 1.23e-04 | 1397.8ms/step | 14.3 samples/s +Step 29720/44000 | Loss: 2.8532 | LR: 1.23e-04 | 1400.2ms/step | 14.3 samples/s +Step 29730/44000 | Loss: 2.9024 | LR: 1.23e-04 | 1399.8ms/step | 14.3 samples/s +Step 29740/44000 | Loss: 2.8670 | LR: 1.23e-04 | 1397.3ms/step | 14.3 samples/s +Step 29750/44000 | Loss: 2.8833 | LR: 1.22e-04 | 1389.4ms/step | 14.4 samples/s +Validation at step 29750: Loss = 2.9070, Perplexity = 18.30 +Step 29760/44000 | Loss: 2.9142 | LR: 1.22e-04 | 3589.5ms/step | 5.6 samples/s +Step 29770/44000 | Loss: 2.8844 | LR: 1.22e-04 | 1391.6ms/step | 14.4 samples/s +Step 29780/44000 | Loss: 2.8937 | LR: 1.22e-04 | 1396.2ms/step | 14.3 samples/s +Step 29790/44000 | Loss: 2.8623 | LR: 1.22e-04 | 1397.9ms/step | 14.3 samples/s +Step 29800/44000 | Loss: 2.8704 | LR: 1.22e-04 | 1398.1ms/step | 14.3 samples/s +Step 29810/44000 | Loss: 2.8787 | LR: 1.21e-04 | 1400.1ms/step | 14.3 samples/s +Step 29820/44000 | Loss: 2.9231 | LR: 1.21e-04 | 1395.8ms/step | 14.3 samples/s +Step 29830/44000 | Loss: 2.9108 | LR: 1.21e-04 | 1392.1ms/step | 14.4 samples/s +Step 29840/44000 | Loss: 2.8589 | LR: 1.21e-04 | 1387.9ms/step | 14.4 samples/s +Step 29850/44000 | Loss: 2.8819 | LR: 1.21e-04 | 1391.6ms/step | 14.4 samples/s +Step 29860/44000 | Loss: 2.8977 | LR: 1.21e-04 | 1392.8ms/step | 14.4 samples/s +Step 29870/44000 | Loss: 2.8672 | LR: 1.21e-04 | 1392.5ms/step | 14.4 samples/s +Step 29880/44000 | Loss: 2.8856 | LR: 1.20e-04 | 1394.3ms/step | 14.3 samples/s +Step 29890/44000 | Loss: 2.8907 | LR: 1.20e-04 | 1394.4ms/step | 14.3 samples/s +Step 29900/44000 | Loss: 2.9309 | LR: 1.20e-04 | 1398.6ms/step | 14.3 samples/s +Step 29910/44000 | Loss: 2.8870 | LR: 1.20e-04 | 1397.9ms/step | 14.3 samples/s +Step 29920/44000 | Loss: 2.8770 | LR: 1.20e-04 | 1395.0ms/step | 14.3 samples/s +Step 29930/44000 | Loss: 2.8840 | LR: 1.20e-04 | 1389.9ms/step | 14.4 samples/s +Step 29940/44000 | Loss: 2.8798 | LR: 1.19e-04 | 1389.1ms/step | 14.4 samples/s +Step 29950/44000 | Loss: 2.8890 | LR: 1.19e-04 | 1393.5ms/step | 14.4 samples/s +Step 29960/44000 | Loss: 2.8718 | LR: 1.19e-04 | 1390.4ms/step | 14.4 samples/s +Step 29970/44000 | Loss: 2.8837 | LR: 1.19e-04 | 1392.0ms/step | 14.4 samples/s +Step 29980/44000 | Loss: 2.8829 | LR: 1.19e-04 | 1398.1ms/step | 14.3 samples/s +Step 29990/44000 | Loss: 2.8972 | LR: 1.19e-04 | 1398.1ms/step | 14.3 samples/s +Step 30000/44000 | Loss: 2.8733 | LR: 1.19e-04 | 1398.0ms/step | 14.3 samples/s +Validation at step 30000: Loss = 2.9061, Perplexity = 18.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-30000 +Step 30010/44000 | Loss: 2.9033 | LR: 1.18e-04 | 3904.6ms/step | 5.1 samples/s +Step 30020/44000 | Loss: 2.9144 | LR: 1.18e-04 | 1395.4ms/step | 14.3 samples/s +Step 30030/44000 | Loss: 2.8819 | LR: 1.18e-04 | 1391.4ms/step | 14.4 samples/s +Step 30040/44000 | Loss: 2.9001 | LR: 1.18e-04 | 1392.9ms/step | 14.4 samples/s +Step 30050/44000 | Loss: 2.8735 | LR: 1.18e-04 | 1394.1ms/step | 14.3 samples/s +Step 30060/44000 | Loss: 2.8990 | LR: 1.18e-04 | 1391.3ms/step | 14.4 samples/s +Step 30070/44000 | Loss: 2.8515 | LR: 1.17e-04 | 1390.4ms/step | 14.4 samples/s +Step 30080/44000 | Loss: 2.9265 | LR: 1.17e-04 | 1392.6ms/step | 14.4 samples/s +Step 30090/44000 | Loss: 2.9007 | LR: 1.17e-04 | 1391.7ms/step | 14.4 samples/s +Step 30100/44000 | Loss: 2.9002 | LR: 1.17e-04 | 1393.1ms/step | 14.4 samples/s +Step 30110/44000 | Loss: 2.8885 | LR: 1.17e-04 | 1396.7ms/step | 14.3 samples/s +Step 30120/44000 | Loss: 2.8757 | LR: 1.17e-04 | 1400.1ms/step | 14.3 samples/s +Step 30130/44000 | Loss: 2.9038 | LR: 1.17e-04 | 1394.1ms/step | 14.3 samples/s +Step 30140/44000 | Loss: 2.8951 | LR: 1.16e-04 | 1400.0ms/step | 14.3 samples/s +Step 30150/44000 | Loss: 2.8772 | LR: 1.16e-04 | 1391.3ms/step | 14.4 samples/s +Step 30160/44000 | Loss: 2.8636 | LR: 1.16e-04 | 1388.9ms/step | 14.4 samples/s +Step 30170/44000 | Loss: 2.8666 | LR: 1.16e-04 | 1389.6ms/step | 14.4 samples/s +Step 30180/44000 | Loss: 2.9009 | LR: 1.16e-04 | 1394.1ms/step | 14.3 samples/s +Step 30190/44000 | Loss: 2.8786 | LR: 1.16e-04 | 1395.5ms/step | 14.3 samples/s +Step 30200/44000 | Loss: 2.8873 | LR: 1.15e-04 | 1394.3ms/step | 14.3 samples/s +Step 30210/44000 | Loss: 2.8886 | LR: 1.15e-04 | 1394.2ms/step | 14.3 samples/s +Step 30220/44000 | Loss: 2.8637 | LR: 1.15e-04 | 1393.4ms/step | 14.4 samples/s +Step 30230/44000 | Loss: 2.9079 | LR: 1.15e-04 | 1392.0ms/step | 14.4 samples/s +Step 30240/44000 | Loss: 2.8911 | LR: 1.15e-04 | 1396.6ms/step | 14.3 samples/s +Step 30250/44000 | Loss: 2.8773 | LR: 1.15e-04 | 1398.1ms/step | 14.3 samples/s +Validation at step 30250: Loss = 2.9051, Perplexity = 18.27 +Step 30260/44000 | Loss: 2.8821 | LR: 1.15e-04 | 3600.0ms/step | 5.6 samples/s +Step 30270/44000 | Loss: 2.8714 | LR: 1.14e-04 | 1390.1ms/step | 14.4 samples/s +Step 30280/44000 | Loss: 2.9156 | LR: 1.14e-04 | 1391.1ms/step | 14.4 samples/s +Step 30290/44000 | Loss: 2.8718 | LR: 1.14e-04 | 1392.6ms/step | 14.4 samples/s +Step 30300/44000 | Loss: 2.8563 | LR: 1.14e-04 | 1393.1ms/step | 14.4 samples/s +Step 30310/44000 | Loss: 2.8672 | LR: 1.14e-04 | 1393.0ms/step | 14.4 samples/s +Step 30320/44000 | Loss: 2.8665 | LR: 1.14e-04 | 1390.4ms/step | 14.4 samples/s +Step 30330/44000 | Loss: 2.8826 | LR: 1.13e-04 | 1390.6ms/step | 14.4 samples/s +Step 30340/44000 | Loss: 2.8899 | LR: 1.13e-04 | 1392.7ms/step | 14.4 samples/s +Step 30350/44000 | Loss: 2.8531 | LR: 1.13e-04 | 1396.1ms/step | 14.3 samples/s +Step 30360/44000 | Loss: 2.8901 | LR: 1.13e-04 | 1400.1ms/step | 14.3 samples/s +Step 30370/44000 | Loss: 2.9024 | LR: 1.13e-04 | 1397.7ms/step | 14.3 samples/s +Step 30380/44000 | Loss: 2.9107 | LR: 1.13e-04 | 1396.5ms/step | 14.3 samples/s +Step 30390/44000 | Loss: 2.8827 | LR: 1.13e-04 | 1393.9ms/step | 14.3 samples/s +Step 30400/44000 | Loss: 2.8840 | LR: 1.12e-04 | 1393.8ms/step | 14.3 samples/s +Step 30410/44000 | Loss: 2.8921 | LR: 1.12e-04 | 1400.4ms/step | 14.3 samples/s +Step 30420/44000 | Loss: 2.8857 | LR: 1.12e-04 | 1399.5ms/step | 14.3 samples/s +Step 30430/44000 | Loss: 2.8895 | LR: 1.12e-04 | 1397.1ms/step | 14.3 samples/s +Step 30440/44000 | Loss: 2.8897 | LR: 1.12e-04 | 1393.1ms/step | 14.4 samples/s +Step 30450/44000 | Loss: 2.8612 | LR: 1.12e-04 | 1390.8ms/step | 14.4 samples/s +Step 30460/44000 | Loss: 2.8906 | LR: 1.11e-04 | 1393.4ms/step | 14.4 samples/s +Step 30470/44000 | Loss: 2.9021 | LR: 1.11e-04 | 1398.1ms/step | 14.3 samples/s +Step 30480/44000 | Loss: 2.8919 | LR: 1.11e-04 | 1399.8ms/step | 14.3 samples/s +Step 30490/44000 | Loss: 2.8850 | LR: 1.11e-04 | 1400.1ms/step | 14.3 samples/s +Step 30500/44000 | Loss: 2.8810 | LR: 1.11e-04 | 1397.9ms/step | 14.3 samples/s +Validation at step 30500: Loss = 2.9043, Perplexity = 18.25 +Step 30510/44000 | Loss: 2.8792 | LR: 1.11e-04 | 3604.2ms/step | 5.5 samples/s +Step 30520/44000 | Loss: 2.8854 | LR: 1.11e-04 | 1400.1ms/step | 14.3 samples/s +Step 30530/44000 | Loss: 2.9244 | LR: 1.10e-04 | 1400.2ms/step | 14.3 samples/s +Step 30540/44000 | Loss: 2.9215 | LR: 1.10e-04 | 1399.7ms/step | 14.3 samples/s +Step 30550/44000 | Loss: 2.8937 | LR: 1.10e-04 | 1400.1ms/step | 14.3 samples/s +Step 30560/44000 | Loss: 2.8939 | LR: 1.10e-04 | 1400.0ms/step | 14.3 samples/s +Step 30570/44000 | Loss: 2.9046 | LR: 1.10e-04 | 1391.8ms/step | 14.4 samples/s +Step 30580/44000 | Loss: 2.9014 | LR: 1.10e-04 | 1389.7ms/step | 14.4 samples/s +Step 30590/44000 | Loss: 2.8897 | LR: 1.10e-04 | 1390.5ms/step | 14.4 samples/s +Step 30600/44000 | Loss: 2.8858 | LR: 1.09e-04 | 1394.0ms/step | 14.3 samples/s +Step 30610/44000 | Loss: 2.8884 | LR: 1.09e-04 | 1397.8ms/step | 14.3 samples/s +Step 30620/44000 | Loss: 2.8991 | LR: 1.09e-04 | 1399.3ms/step | 14.3 samples/s +Step 30630/44000 | Loss: 2.8697 | LR: 1.09e-04 | 1398.8ms/step | 14.3 samples/s +Step 30640/44000 | Loss: 2.9006 | LR: 1.09e-04 | 1400.1ms/step | 14.3 samples/s +Step 30650/44000 | Loss: 2.9085 | LR: 1.09e-04 | 1400.2ms/step | 14.3 samples/s +Step 30660/44000 | Loss: 2.9265 | LR: 1.08e-04 | 1397.9ms/step | 14.3 samples/s +Step 30670/44000 | Loss: 2.9154 | LR: 1.08e-04 | 1396.2ms/step | 14.3 samples/s +Step 30680/44000 | Loss: 2.8954 | LR: 1.08e-04 | 1399.3ms/step | 14.3 samples/s +Step 30690/44000 | Loss: 2.8809 | LR: 1.08e-04 | 1388.5ms/step | 14.4 samples/s +Step 30700/44000 | Loss: 2.9032 | LR: 1.08e-04 | 1392.0ms/step | 14.4 samples/s +Step 30710/44000 | Loss: 2.8838 | LR: 1.08e-04 | 1394.2ms/step | 14.3 samples/s +Step 30720/44000 | Loss: 2.9006 | LR: 1.08e-04 | 1390.1ms/step | 14.4 samples/s +Step 30730/44000 | Loss: 2.8856 | LR: 1.07e-04 | 1387.9ms/step | 14.4 samples/s +Step 30740/44000 | Loss: 2.9026 | LR: 1.07e-04 | 1391.2ms/step | 14.4 samples/s +Step 30750/44000 | Loss: 2.8859 | LR: 1.07e-04 | 1392.1ms/step | 14.4 samples/s +Validation at step 30750: Loss = 2.9036, Perplexity = 18.24 +Step 30760/44000 | Loss: 2.8896 | LR: 1.07e-04 | 3600.7ms/step | 5.6 samples/s +Step 30770/44000 | Loss: 2.9310 | LR: 1.07e-04 | 1400.2ms/step | 14.3 samples/s +Step 30780/44000 | Loss: 2.8823 | LR: 1.07e-04 | 1399.9ms/step | 14.3 samples/s +Step 30790/44000 | Loss: 2.8877 | LR: 1.07e-04 | 1396.1ms/step | 14.3 samples/s +Step 30800/44000 | Loss: 2.8972 | LR: 1.06e-04 | 1392.0ms/step | 14.4 samples/s +Step 30810/44000 | Loss: 2.8637 | LR: 1.06e-04 | 1387.2ms/step | 14.4 samples/s +Step 30820/44000 | Loss: 2.8962 | LR: 1.06e-04 | 1387.6ms/step | 14.4 samples/s +Step 30830/44000 | Loss: 2.9334 | LR: 1.06e-04 | 1391.1ms/step | 14.4 samples/s +Step 30840/44000 | Loss: 2.9046 | LR: 1.06e-04 | 1394.0ms/step | 14.3 samples/s +Step 30850/44000 | Loss: 2.8755 | LR: 1.06e-04 | 1398.1ms/step | 14.3 samples/s +Step 30860/44000 | Loss: 2.9050 | LR: 1.05e-04 | 1393.6ms/step | 14.4 samples/s +Step 30870/44000 | Loss: 2.8636 | LR: 1.05e-04 | 1392.6ms/step | 14.4 samples/s +Step 30880/44000 | Loss: 2.8822 | LR: 1.05e-04 | 1397.4ms/step | 14.3 samples/s +Step 30890/44000 | Loss: 2.8364 | LR: 1.05e-04 | 1392.3ms/step | 14.4 samples/s +Step 30900/44000 | Loss: 2.8678 | LR: 1.05e-04 | 1392.2ms/step | 14.4 samples/s +Step 30910/44000 | Loss: 2.8783 | LR: 1.05e-04 | 1396.1ms/step | 14.3 samples/s +Step 30920/44000 | Loss: 2.8945 | LR: 1.05e-04 | 1397.7ms/step | 14.3 samples/s +Step 30930/44000 | Loss: 2.8790 | LR: 1.04e-04 | 1400.0ms/step | 14.3 samples/s +Step 30940/44000 | Loss: 2.8662 | LR: 1.04e-04 | 1400.3ms/step | 14.3 samples/s +Step 30950/44000 | Loss: 2.8898 | LR: 1.04e-04 | 1399.9ms/step | 14.3 samples/s +Step 30960/44000 | Loss: 2.8835 | LR: 1.04e-04 | 1400.2ms/step | 14.3 samples/s +Step 30970/44000 | Loss: 2.9233 | LR: 1.04e-04 | 1399.7ms/step | 14.3 samples/s +Step 30980/44000 | Loss: 2.8690 | LR: 1.04e-04 | 1400.3ms/step | 14.3 samples/s +Step 30990/44000 | Loss: 2.8735 | LR: 1.04e-04 | 1399.8ms/step | 14.3 samples/s +Step 31000/44000 | Loss: 2.8627 | LR: 1.03e-04 | 1390.6ms/step | 14.4 samples/s +Validation at step 31000: Loss = 2.9028, Perplexity = 18.22 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-31000 +Step 31010/44000 | Loss: 2.8726 | LR: 1.03e-04 | 3886.9ms/step | 5.1 samples/s +Step 31020/44000 | Loss: 2.9013 | LR: 1.03e-04 | 1392.2ms/step | 14.4 samples/s +Step 31030/44000 | Loss: 2.8542 | LR: 1.03e-04 | 1396.3ms/step | 14.3 samples/s +Step 31040/44000 | Loss: 2.8849 | LR: 1.03e-04 | 1392.4ms/step | 14.4 samples/s +Step 31050/44000 | Loss: 2.8961 | LR: 1.03e-04 | 1393.2ms/step | 14.4 samples/s +Step 31060/44000 | Loss: 2.9033 | LR: 1.03e-04 | 1392.2ms/step | 14.4 samples/s +Step 31070/44000 | Loss: 2.8885 | LR: 1.02e-04 | 1390.1ms/step | 14.4 samples/s +Step 31080/44000 | Loss: 2.8822 | LR: 1.02e-04 | 1393.8ms/step | 14.3 samples/s +Step 31090/44000 | Loss: 2.8800 | LR: 1.02e-04 | 1391.7ms/step | 14.4 samples/s +Step 31100/44000 | Loss: 2.8707 | LR: 1.02e-04 | 1390.4ms/step | 14.4 samples/s +Step 31110/44000 | Loss: 2.8735 | LR: 1.02e-04 | 1393.6ms/step | 14.4 samples/s +Step 31120/44000 | Loss: 2.8919 | LR: 1.02e-04 | 1394.7ms/step | 14.3 samples/s +Step 31130/44000 | Loss: 2.9215 | LR: 1.02e-04 | 1398.0ms/step | 14.3 samples/s +Step 31140/44000 | Loss: 2.8743 | LR: 1.01e-04 | 1394.4ms/step | 14.3 samples/s +Step 31150/44000 | Loss: 2.9189 | LR: 1.01e-04 | 1389.7ms/step | 14.4 samples/s +Step 31160/44000 | Loss: 2.8906 | LR: 1.01e-04 | 1390.9ms/step | 14.4 samples/s +Step 31170/44000 | Loss: 2.8842 | LR: 1.01e-04 | 1393.0ms/step | 14.4 samples/s +Step 31180/44000 | Loss: 2.8930 | LR: 1.01e-04 | 1395.7ms/step | 14.3 samples/s +Step 31190/44000 | Loss: 2.8650 | LR: 1.01e-04 | 1391.1ms/step | 14.4 samples/s +Step 31200/44000 | Loss: 2.8768 | LR: 1.00e-04 | 1393.3ms/step | 14.4 samples/s +Step 31210/44000 | Loss: 2.8973 | LR: 1.00e-04 | 1393.3ms/step | 14.4 samples/s +Step 31220/44000 | Loss: 2.8847 | LR: 1.00e-04 | 1398.8ms/step | 14.3 samples/s +Step 31230/44000 | Loss: 2.9187 | LR: 1.00e-04 | 1400.0ms/step | 14.3 samples/s +Step 31240/44000 | Loss: 2.8855 | LR: 9.99e-05 | 1399.4ms/step | 14.3 samples/s +Step 31250/44000 | Loss: 2.8688 | LR: 9.98e-05 | 1390.6ms/step | 14.4 samples/s +Validation at step 31250: Loss = 2.9023, Perplexity = 18.22 +Step 31260/44000 | Loss: 2.9002 | LR: 9.96e-05 | 3598.1ms/step | 5.6 samples/s +Step 31270/44000 | Loss: 2.9139 | LR: 9.95e-05 | 1395.9ms/step | 14.3 samples/s +Step 31280/44000 | Loss: 2.8724 | LR: 9.93e-05 | 1396.2ms/step | 14.3 samples/s +Step 31290/44000 | Loss: 2.9013 | LR: 9.92e-05 | 1397.1ms/step | 14.3 samples/s +Step 31300/44000 | Loss: 2.8985 | LR: 9.90e-05 | 1390.8ms/step | 14.4 samples/s +Step 31310/44000 | Loss: 2.8874 | LR: 9.89e-05 | 1393.6ms/step | 14.4 samples/s +Step 31320/44000 | Loss: 2.8844 | LR: 9.88e-05 | 1390.2ms/step | 14.4 samples/s +Step 31330/44000 | Loss: 2.9129 | LR: 9.86e-05 | 1391.8ms/step | 14.4 samples/s +Step 31340/44000 | Loss: 2.8788 | LR: 9.85e-05 | 1392.9ms/step | 14.4 samples/s +Step 31350/44000 | Loss: 2.8583 | LR: 9.83e-05 | 1395.3ms/step | 14.3 samples/s +Step 31360/44000 | Loss: 2.8680 | LR: 9.82e-05 | 1396.0ms/step | 14.3 samples/s +Step 31370/44000 | Loss: 2.9007 | LR: 9.80e-05 | 1395.9ms/step | 14.3 samples/s +Step 31380/44000 | Loss: 2.9132 | LR: 9.79e-05 | 1394.2ms/step | 14.3 samples/s +Step 31390/44000 | Loss: 2.8516 | LR: 9.77e-05 | 1397.5ms/step | 14.3 samples/s +Step 31400/44000 | Loss: 2.8858 | LR: 9.76e-05 | 1393.0ms/step | 14.4 samples/s +Step 31410/44000 | Loss: 2.8576 | LR: 9.75e-05 | 1389.9ms/step | 14.4 samples/s +Step 31420/44000 | Loss: 2.8762 | LR: 9.73e-05 | 1395.7ms/step | 14.3 samples/s +Step 31430/44000 | Loss: 2.8983 | LR: 9.72e-05 | 1395.4ms/step | 14.3 samples/s +Step 31440/44000 | Loss: 2.8830 | LR: 9.70e-05 | 1393.9ms/step | 14.3 samples/s +Step 31450/44000 | Loss: 2.8949 | LR: 9.69e-05 | 1392.9ms/step | 14.4 samples/s +Step 31460/44000 | Loss: 2.8745 | LR: 9.67e-05 | 1393.8ms/step | 14.3 samples/s +Step 31470/44000 | Loss: 2.9011 | LR: 9.66e-05 | 1390.8ms/step | 14.4 samples/s +Step 31480/44000 | Loss: 2.8650 | LR: 9.65e-05 | 1387.5ms/step | 14.4 samples/s +Step 31490/44000 | Loss: 2.8741 | LR: 9.63e-05 | 1393.8ms/step | 14.3 samples/s +Step 31500/44000 | Loss: 2.8875 | LR: 9.62e-05 | 1395.9ms/step | 14.3 samples/s +Validation at step 31500: Loss = 2.9014, Perplexity = 18.20 +Step 31510/44000 | Loss: 2.8703 | LR: 9.60e-05 | 3604.1ms/step | 5.5 samples/s +Step 31520/44000 | Loss: 2.8799 | LR: 9.59e-05 | 1400.0ms/step | 14.3 samples/s +Step 31530/44000 | Loss: 2.8939 | LR: 9.57e-05 | 1397.9ms/step | 14.3 samples/s +Step 31540/44000 | Loss: 2.8743 | LR: 9.56e-05 | 1396.2ms/step | 14.3 samples/s +Step 31550/44000 | Loss: 2.8985 | LR: 9.54e-05 | 1396.0ms/step | 14.3 samples/s +Step 31560/44000 | Loss: 2.8848 | LR: 9.53e-05 | 1396.1ms/step | 14.3 samples/s +Step 31570/44000 | Loss: 2.8747 | LR: 9.52e-05 | 1399.6ms/step | 14.3 samples/s +Step 31580/44000 | Loss: 2.8996 | LR: 9.50e-05 | 1400.2ms/step | 14.3 samples/s +Step 31590/44000 | Loss: 2.9013 | LR: 9.49e-05 | 1400.0ms/step | 14.3 samples/s +Step 31600/44000 | Loss: 2.8828 | LR: 9.47e-05 | 1392.0ms/step | 14.4 samples/s +Step 31610/44000 | Loss: 2.8724 | LR: 9.46e-05 | 1387.8ms/step | 14.4 samples/s +Step 31620/44000 | Loss: 2.8952 | LR: 9.45e-05 | 1389.9ms/step | 14.4 samples/s +Step 31630/44000 | Loss: 2.8758 | LR: 9.43e-05 | 1393.1ms/step | 14.4 samples/s +Step 31640/44000 | Loss: 2.8531 | LR: 9.42e-05 | 1392.4ms/step | 14.4 samples/s +Step 31650/44000 | Loss: 2.8578 | LR: 9.40e-05 | 1387.2ms/step | 14.4 samples/s +Step 31660/44000 | Loss: 2.8742 | LR: 9.39e-05 | 1388.0ms/step | 14.4 samples/s +Step 31670/44000 | Loss: 2.8665 | LR: 9.37e-05 | 1393.9ms/step | 14.3 samples/s +Step 31680/44000 | Loss: 2.8801 | LR: 9.36e-05 | 1390.1ms/step | 14.4 samples/s +Step 31690/44000 | Loss: 2.8875 | LR: 9.35e-05 | 1391.3ms/step | 14.4 samples/s +Step 31700/44000 | Loss: 2.9034 | LR: 9.33e-05 | 1390.7ms/step | 14.4 samples/s +Step 31710/44000 | Loss: 2.8791 | LR: 9.32e-05 | 1397.2ms/step | 14.3 samples/s +Step 31720/44000 | Loss: 2.8792 | LR: 9.30e-05 | 1398.3ms/step | 14.3 samples/s +Step 31730/44000 | Loss: 2.8542 | LR: 9.29e-05 | 1400.4ms/step | 14.3 samples/s +Step 31740/44000 | Loss: 2.8876 | LR: 9.28e-05 | 1397.9ms/step | 14.3 samples/s +Step 31750/44000 | Loss: 2.8962 | LR: 9.26e-05 | 1400.1ms/step | 14.3 samples/s +Validation at step 31750: Loss = 2.9007, Perplexity = 18.19 +Step 31760/44000 | Loss: 2.8701 | LR: 9.25e-05 | 3618.0ms/step | 5.5 samples/s +Step 31770/44000 | Loss: 2.8960 | LR: 9.23e-05 | 1400.0ms/step | 14.3 samples/s +Step 31780/44000 | Loss: 2.8859 | LR: 9.22e-05 | 1400.1ms/step | 14.3 samples/s +Step 31790/44000 | Loss: 2.8888 | LR: 9.20e-05 | 1400.0ms/step | 14.3 samples/s +Step 31800/44000 | Loss: 2.8582 | LR: 9.19e-05 | 1400.0ms/step | 14.3 samples/s +Step 31810/44000 | Loss: 2.8586 | LR: 9.18e-05 | 1400.0ms/step | 14.3 samples/s +Step 31820/44000 | Loss: 2.8757 | LR: 9.16e-05 | 1399.8ms/step | 14.3 samples/s +Step 31830/44000 | Loss: 2.8808 | LR: 9.15e-05 | 1397.8ms/step | 14.3 samples/s +Step 31840/44000 | Loss: 2.8936 | LR: 9.13e-05 | 1394.2ms/step | 14.3 samples/s +Step 31850/44000 | Loss: 2.8958 | LR: 9.12e-05 | 1392.9ms/step | 14.4 samples/s +Step 31860/44000 | Loss: 2.9041 | LR: 9.11e-05 | 1392.7ms/step | 14.4 samples/s +Step 31870/44000 | Loss: 2.8811 | LR: 9.09e-05 | 1392.5ms/step | 14.4 samples/s +Step 31880/44000 | Loss: 2.9147 | LR: 9.08e-05 | 1387.0ms/step | 14.4 samples/s +Step 31890/44000 | Loss: 2.8551 | LR: 9.06e-05 | 1386.5ms/step | 14.4 samples/s +Step 31900/44000 | Loss: 2.8751 | LR: 9.05e-05 | 1388.4ms/step | 14.4 samples/s +Step 31910/44000 | Loss: 2.9072 | LR: 9.04e-05 | 1388.1ms/step | 14.4 samples/s +Step 31920/44000 | Loss: 2.8331 | LR: 9.02e-05 | 1387.8ms/step | 14.4 samples/s +Step 31930/44000 | Loss: 2.8893 | LR: 9.01e-05 | 1388.4ms/step | 14.4 samples/s +Step 31940/44000 | Loss: 2.8985 | LR: 8.99e-05 | 1389.4ms/step | 14.4 samples/s +Step 31950/44000 | Loss: 2.8873 | LR: 8.98e-05 | 1389.5ms/step | 14.4 samples/s +Step 31960/44000 | Loss: 2.9016 | LR: 8.97e-05 | 1390.7ms/step | 14.4 samples/s +Step 31970/44000 | Loss: 2.8693 | LR: 8.95e-05 | 1386.9ms/step | 14.4 samples/s +Step 31980/44000 | Loss: 2.8828 | LR: 8.94e-05 | 1393.0ms/step | 14.4 samples/s +Step 31990/44000 | Loss: 2.8499 | LR: 8.92e-05 | 1396.4ms/step | 14.3 samples/s +Step 32000/44000 | Loss: 2.8724 | LR: 8.91e-05 | 1398.1ms/step | 14.3 samples/s +Validation at step 32000: Loss = 2.9001, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-32000 +Step 32010/44000 | Loss: 2.8949 | LR: 8.90e-05 | 3900.0ms/step | 5.1 samples/s +Step 32020/44000 | Loss: 2.8839 | LR: 8.88e-05 | 1399.7ms/step | 14.3 samples/s +Step 32030/44000 | Loss: 2.8916 | LR: 8.87e-05 | 1400.3ms/step | 14.3 samples/s +Step 32040/44000 | Loss: 2.8941 | LR: 8.86e-05 | 1400.1ms/step | 14.3 samples/s +Step 32050/44000 | Loss: 2.8680 | LR: 8.84e-05 | 1397.7ms/step | 14.3 samples/s +Step 32060/44000 | Loss: 2.8532 | LR: 8.83e-05 | 1394.7ms/step | 14.3 samples/s +Step 32070/44000 | Loss: 2.8794 | LR: 8.81e-05 | 1397.3ms/step | 14.3 samples/s +Step 32080/44000 | Loss: 2.8524 | LR: 8.80e-05 | 1400.4ms/step | 14.3 samples/s +Step 32090/44000 | Loss: 2.8866 | LR: 8.79e-05 | 1397.7ms/step | 14.3 samples/s +Step 32100/44000 | Loss: 2.8896 | LR: 8.77e-05 | 1392.2ms/step | 14.4 samples/s +Step 32110/44000 | Loss: 2.8712 | LR: 8.76e-05 | 1390.1ms/step | 14.4 samples/s +Step 32120/44000 | Loss: 2.8758 | LR: 8.74e-05 | 1393.8ms/step | 14.3 samples/s +Step 32130/44000 | Loss: 2.8700 | LR: 8.73e-05 | 1394.1ms/step | 14.3 samples/s +Step 32140/44000 | Loss: 2.8658 | LR: 8.72e-05 | 1393.8ms/step | 14.3 samples/s +Step 32150/44000 | Loss: 2.8842 | LR: 8.70e-05 | 1391.4ms/step | 14.4 samples/s +Step 32160/44000 | Loss: 2.8556 | LR: 8.69e-05 | 1391.0ms/step | 14.4 samples/s +Step 32170/44000 | Loss: 2.8792 | LR: 8.68e-05 | 1391.7ms/step | 14.4 samples/s +Step 32180/44000 | Loss: 2.9117 | LR: 8.66e-05 | 1392.7ms/step | 14.4 samples/s +Step 32190/44000 | Loss: 2.8854 | LR: 8.65e-05 | 1395.9ms/step | 14.3 samples/s +Step 32200/44000 | Loss: 2.8968 | LR: 8.63e-05 | 1395.9ms/step | 14.3 samples/s +Step 32210/44000 | Loss: 2.9028 | LR: 8.62e-05 | 1399.8ms/step | 14.3 samples/s +Step 32220/44000 | Loss: 2.8587 | LR: 8.61e-05 | 1400.2ms/step | 14.3 samples/s +Step 32230/44000 | Loss: 2.8867 | LR: 8.59e-05 | 1397.9ms/step | 14.3 samples/s +Step 32240/44000 | Loss: 2.8743 | LR: 8.58e-05 | 1392.0ms/step | 14.4 samples/s +Step 32250/44000 | Loss: 2.8965 | LR: 8.57e-05 | 1388.7ms/step | 14.4 samples/s +Validation at step 32250: Loss = 2.8997, Perplexity = 18.17 +Step 32260/44000 | Loss: 2.9087 | LR: 8.55e-05 | 3597.6ms/step | 5.6 samples/s +Step 32270/44000 | Loss: 2.8822 | LR: 8.54e-05 | 1395.7ms/step | 14.3 samples/s +Step 32280/44000 | Loss: 2.9052 | LR: 8.53e-05 | 1391.9ms/step | 14.4 samples/s +Step 32290/44000 | Loss: 2.8722 | LR: 8.51e-05 | 1393.0ms/step | 14.4 samples/s +Step 32300/44000 | Loss: 2.8673 | LR: 8.50e-05 | 1392.2ms/step | 14.4 samples/s +Step 32310/44000 | Loss: 2.8909 | LR: 8.48e-05 | 1393.8ms/step | 14.3 samples/s +Step 32320/44000 | Loss: 2.9110 | LR: 8.47e-05 | 1391.0ms/step | 14.4 samples/s +Step 32330/44000 | Loss: 2.8781 | LR: 8.46e-05 | 1395.4ms/step | 14.3 samples/s +Step 32340/44000 | Loss: 2.8770 | LR: 8.44e-05 | 1392.6ms/step | 14.4 samples/s +Step 32350/44000 | Loss: 2.9005 | LR: 8.43e-05 | 1392.1ms/step | 14.4 samples/s +Step 32360/44000 | Loss: 2.8783 | LR: 8.42e-05 | 1394.0ms/step | 14.3 samples/s +Step 32370/44000 | Loss: 2.8761 | LR: 8.40e-05 | 1390.0ms/step | 14.4 samples/s +Step 32380/44000 | Loss: 2.8784 | LR: 8.39e-05 | 1392.1ms/step | 14.4 samples/s +Step 32390/44000 | Loss: 2.8791 | LR: 8.38e-05 | 1392.0ms/step | 14.4 samples/s +Step 32400/44000 | Loss: 2.8868 | LR: 8.36e-05 | 1388.1ms/step | 14.4 samples/s +Step 32410/44000 | Loss: 2.8829 | LR: 8.35e-05 | 1391.7ms/step | 14.4 samples/s +Step 32420/44000 | Loss: 2.8788 | LR: 8.33e-05 | 1398.0ms/step | 14.3 samples/s +Step 32430/44000 | Loss: 2.9259 | LR: 8.32e-05 | 1397.9ms/step | 14.3 samples/s +Step 32440/44000 | Loss: 2.8926 | LR: 8.31e-05 | 1396.2ms/step | 14.3 samples/s +Step 32450/44000 | Loss: 2.8850 | LR: 8.29e-05 | 1400.2ms/step | 14.3 samples/s +Step 32460/44000 | Loss: 2.9064 | LR: 8.28e-05 | 1400.0ms/step | 14.3 samples/s +Step 32470/44000 | Loss: 2.8618 | LR: 8.27e-05 | 1400.1ms/step | 14.3 samples/s +Step 32480/44000 | Loss: 2.8960 | LR: 8.25e-05 | 1397.4ms/step | 14.3 samples/s +Step 32490/44000 | Loss: 2.8715 | LR: 8.24e-05 | 1392.5ms/step | 14.4 samples/s +Step 32500/44000 | Loss: 2.8902 | LR: 8.23e-05 | 1394.1ms/step | 14.3 samples/s +Validation at step 32500: Loss = 2.8992, Perplexity = 18.16 +Step 32510/44000 | Loss: 2.8567 | LR: 8.21e-05 | 3601.8ms/step | 5.6 samples/s +Step 32520/44000 | Loss: 2.8833 | LR: 8.20e-05 | 1397.9ms/step | 14.3 samples/s +Step 32530/44000 | Loss: 2.9125 | LR: 8.19e-05 | 1392.0ms/step | 14.4 samples/s +Step 32540/44000 | Loss: 2.8696 | LR: 8.17e-05 | 1392.2ms/step | 14.4 samples/s +Step 32550/44000 | Loss: 2.8385 | LR: 8.16e-05 | 1392.0ms/step | 14.4 samples/s +Step 32560/44000 | Loss: 2.9045 | LR: 8.15e-05 | 1394.0ms/step | 14.3 samples/s +Step 32570/44000 | Loss: 2.8855 | LR: 8.13e-05 | 1395.9ms/step | 14.3 samples/s +Step 32580/44000 | Loss: 2.8899 | LR: 8.12e-05 | 1396.3ms/step | 14.3 samples/s +Step 32590/44000 | Loss: 2.9134 | LR: 8.11e-05 | 1399.5ms/step | 14.3 samples/s +Step 32600/44000 | Loss: 2.8890 | LR: 8.09e-05 | 1393.9ms/step | 14.3 samples/s +Step 32610/44000 | Loss: 2.8915 | LR: 8.08e-05 | 1392.6ms/step | 14.4 samples/s +Step 32620/44000 | Loss: 2.8999 | LR: 8.07e-05 | 1392.2ms/step | 14.4 samples/s +Step 32630/44000 | Loss: 2.8606 | LR: 8.05e-05 | 1393.9ms/step | 14.3 samples/s +Step 32640/44000 | Loss: 2.9212 | LR: 8.04e-05 | 1395.0ms/step | 14.3 samples/s +Step 32650/44000 | Loss: 2.8992 | LR: 8.03e-05 | 1390.8ms/step | 14.4 samples/s +Step 32660/44000 | Loss: 2.8869 | LR: 8.01e-05 | 1393.5ms/step | 14.4 samples/s +Step 32670/44000 | Loss: 2.9228 | LR: 8.00e-05 | 1394.3ms/step | 14.3 samples/s +Step 32680/44000 | Loss: 2.8804 | LR: 7.99e-05 | 1393.8ms/step | 14.3 samples/s +Step 32690/44000 | Loss: 2.8912 | LR: 7.97e-05 | 1395.1ms/step | 14.3 samples/s +Step 32700/44000 | Loss: 2.8719 | LR: 7.96e-05 | 1393.2ms/step | 14.4 samples/s +Step 32710/44000 | Loss: 2.8997 | LR: 7.95e-05 | 1396.1ms/step | 14.3 samples/s +Step 32720/44000 | Loss: 2.8989 | LR: 7.93e-05 | 1400.1ms/step | 14.3 samples/s +Step 32730/44000 | Loss: 2.8773 | LR: 7.92e-05 | 1398.1ms/step | 14.3 samples/s +Step 32740/44000 | Loss: 2.8654 | LR: 7.91e-05 | 1400.1ms/step | 14.3 samples/s +Step 32750/44000 | Loss: 2.8580 | LR: 7.89e-05 | 1397.8ms/step | 14.3 samples/s +Validation at step 32750: Loss = 2.8987, Perplexity = 18.15 +Step 32760/44000 | Loss: 2.8929 | LR: 7.88e-05 | 3592.0ms/step | 5.6 samples/s +Step 32770/44000 | Loss: 2.8892 | LR: 7.87e-05 | 1393.7ms/step | 14.3 samples/s +Step 32780/44000 | Loss: 2.9134 | LR: 7.85e-05 | 1398.1ms/step | 14.3 samples/s +Step 32790/44000 | Loss: 2.9194 | LR: 7.84e-05 | 1396.4ms/step | 14.3 samples/s +Step 32800/44000 | Loss: 2.8742 | LR: 7.83e-05 | 1396.1ms/step | 14.3 samples/s +Step 32810/44000 | Loss: 2.8846 | LR: 7.81e-05 | 1400.1ms/step | 14.3 samples/s +Step 32820/44000 | Loss: 2.9016 | LR: 7.80e-05 | 1400.1ms/step | 14.3 samples/s +Step 32830/44000 | Loss: 2.8982 | LR: 7.79e-05 | 1397.8ms/step | 14.3 samples/s +Step 32840/44000 | Loss: 2.8500 | LR: 7.77e-05 | 1397.7ms/step | 14.3 samples/s +Step 32850/44000 | Loss: 2.8790 | LR: 7.76e-05 | 1393.8ms/step | 14.3 samples/s +Step 32860/44000 | Loss: 2.8798 | LR: 7.75e-05 | 1394.4ms/step | 14.3 samples/s +Step 32870/44000 | Loss: 2.8969 | LR: 7.73e-05 | 1394.4ms/step | 14.3 samples/s +Step 32880/44000 | Loss: 2.8571 | LR: 7.72e-05 | 1399.8ms/step | 14.3 samples/s +Step 32890/44000 | Loss: 2.8948 | LR: 7.71e-05 | 1397.9ms/step | 14.3 samples/s +Step 32900/44000 | Loss: 2.8887 | LR: 7.70e-05 | 1393.9ms/step | 14.3 samples/s +Step 32910/44000 | Loss: 2.8794 | LR: 7.68e-05 | 1390.1ms/step | 14.4 samples/s +Step 32920/44000 | Loss: 2.9057 | LR: 7.67e-05 | 1391.7ms/step | 14.4 samples/s +Step 32930/44000 | Loss: 2.8772 | LR: 7.66e-05 | 1392.7ms/step | 14.4 samples/s +Step 32940/44000 | Loss: 2.8618 | LR: 7.64e-05 | 1397.8ms/step | 14.3 samples/s +Step 32950/44000 | Loss: 2.8699 | LR: 7.63e-05 | 1396.2ms/step | 14.3 samples/s +Step 32960/44000 | Loss: 2.8765 | LR: 7.62e-05 | 1392.2ms/step | 14.4 samples/s +Step 32970/44000 | Loss: 2.8805 | LR: 7.60e-05 | 1390.3ms/step | 14.4 samples/s +Step 32980/44000 | Loss: 2.8834 | LR: 7.59e-05 | 1397.5ms/step | 14.3 samples/s +Step 32990/44000 | Loss: 2.8874 | LR: 7.58e-05 | 1399.7ms/step | 14.3 samples/s +Step 33000/44000 | Loss: 2.9090 | LR: 7.56e-05 | 1400.4ms/step | 14.3 samples/s +Validation at step 33000: Loss = 2.8983, Perplexity = 18.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-33000 +Step 33010/44000 | Loss: 2.8959 | LR: 7.55e-05 | 3909.8ms/step | 5.1 samples/s +Step 33020/44000 | Loss: 2.8611 | LR: 7.54e-05 | 1400.4ms/step | 14.3 samples/s +Step 33030/44000 | Loss: 2.9041 | LR: 7.53e-05 | 1399.8ms/step | 14.3 samples/s +Step 33040/44000 | Loss: 2.8889 | LR: 7.51e-05 | 1400.0ms/step | 14.3 samples/s +Step 33050/44000 | Loss: 2.8976 | LR: 7.50e-05 | 1400.0ms/step | 14.3 samples/s +Step 33060/44000 | Loss: 2.8666 | LR: 7.49e-05 | 1400.1ms/step | 14.3 samples/s +Step 33070/44000 | Loss: 2.8977 | LR: 7.47e-05 | 1400.0ms/step | 14.3 samples/s +Step 33080/44000 | Loss: 2.8996 | LR: 7.46e-05 | 1397.4ms/step | 14.3 samples/s +Step 33090/44000 | Loss: 2.8879 | LR: 7.45e-05 | 1390.5ms/step | 14.4 samples/s +Step 33100/44000 | Loss: 2.8604 | LR: 7.44e-05 | 1390.2ms/step | 14.4 samples/s +Step 33110/44000 | Loss: 2.8374 | LR: 7.42e-05 | 1397.9ms/step | 14.3 samples/s +Step 33120/44000 | Loss: 2.8703 | LR: 7.41e-05 | 1397.9ms/step | 14.3 samples/s +Step 33130/44000 | Loss: 2.8880 | LR: 7.40e-05 | 1396.2ms/step | 14.3 samples/s +Step 33140/44000 | Loss: 2.9041 | LR: 7.38e-05 | 1395.7ms/step | 14.3 samples/s +Step 33150/44000 | Loss: 2.8955 | LR: 7.37e-05 | 1393.8ms/step | 14.3 samples/s +Step 33160/44000 | Loss: 2.8714 | LR: 7.36e-05 | 1392.1ms/step | 14.4 samples/s +Step 33170/44000 | Loss: 2.8779 | LR: 7.34e-05 | 1390.7ms/step | 14.4 samples/s +Step 33180/44000 | Loss: 2.9039 | LR: 7.33e-05 | 1397.5ms/step | 14.3 samples/s +Step 33190/44000 | Loss: 2.9028 | LR: 7.32e-05 | 1392.2ms/step | 14.4 samples/s +Step 33200/44000 | Loss: 2.8907 | LR: 7.31e-05 | 1395.3ms/step | 14.3 samples/s +Step 33210/44000 | Loss: 2.8788 | LR: 7.29e-05 | 1392.8ms/step | 14.4 samples/s +Step 33220/44000 | Loss: 2.8957 | LR: 7.28e-05 | 1400.1ms/step | 14.3 samples/s +Step 33230/44000 | Loss: 2.8802 | LR: 7.27e-05 | 1400.0ms/step | 14.3 samples/s +Step 33240/44000 | Loss: 2.8737 | LR: 7.26e-05 | 1399.9ms/step | 14.3 samples/s +Step 33250/44000 | Loss: 2.8832 | LR: 7.24e-05 | 1400.2ms/step | 14.3 samples/s +Validation at step 33250: Loss = 2.8979, Perplexity = 18.14 +Step 33260/44000 | Loss: 2.8952 | LR: 7.23e-05 | 3603.9ms/step | 5.5 samples/s +Step 33270/44000 | Loss: 2.8756 | LR: 7.22e-05 | 1398.2ms/step | 14.3 samples/s +Step 33280/44000 | Loss: 2.8954 | LR: 7.20e-05 | 1393.9ms/step | 14.3 samples/s +Step 33290/44000 | Loss: 2.8442 | LR: 7.19e-05 | 1391.6ms/step | 14.4 samples/s +Step 33300/44000 | Loss: 2.8857 | LR: 7.18e-05 | 1388.6ms/step | 14.4 samples/s +Step 33310/44000 | Loss: 2.9008 | LR: 7.17e-05 | 1388.0ms/step | 14.4 samples/s +Step 33320/44000 | Loss: 2.8822 | LR: 7.15e-05 | 1389.6ms/step | 14.4 samples/s +Step 33330/44000 | Loss: 2.8724 | LR: 7.14e-05 | 1387.8ms/step | 14.4 samples/s +Step 33340/44000 | Loss: 2.8741 | LR: 7.13e-05 | 1392.4ms/step | 14.4 samples/s +Step 33350/44000 | Loss: 2.8850 | LR: 7.11e-05 | 1395.8ms/step | 14.3 samples/s +Step 33360/44000 | Loss: 2.8960 | LR: 7.10e-05 | 1398.4ms/step | 14.3 samples/s +Step 33370/44000 | Loss: 2.8576 | LR: 7.09e-05 | 1395.7ms/step | 14.3 samples/s +Step 33380/44000 | Loss: 2.8728 | LR: 7.08e-05 | 1397.7ms/step | 14.3 samples/s +Step 33390/44000 | Loss: 2.9030 | LR: 7.06e-05 | 1396.1ms/step | 14.3 samples/s +Step 33400/44000 | Loss: 2.8509 | LR: 7.05e-05 | 1396.4ms/step | 14.3 samples/s +Step 33410/44000 | Loss: 2.8966 | LR: 7.04e-05 | 1399.7ms/step | 14.3 samples/s +Step 33420/44000 | Loss: 2.8742 | LR: 7.03e-05 | 1398.1ms/step | 14.3 samples/s +Step 33430/44000 | Loss: 2.8727 | LR: 7.01e-05 | 1399.8ms/step | 14.3 samples/s +Step 33440/44000 | Loss: 2.9004 | LR: 7.00e-05 | 1397.9ms/step | 14.3 samples/s +Step 33450/44000 | Loss: 2.8951 | LR: 6.99e-05 | 1394.6ms/step | 14.3 samples/s +Step 33460/44000 | Loss: 2.8893 | LR: 6.98e-05 | 1393.7ms/step | 14.4 samples/s +Step 33470/44000 | Loss: 2.8793 | LR: 6.96e-05 | 1397.4ms/step | 14.3 samples/s +Step 33480/44000 | Loss: 2.8663 | LR: 6.95e-05 | 1396.6ms/step | 14.3 samples/s +Step 33490/44000 | Loss: 2.8970 | LR: 6.94e-05 | 1400.1ms/step | 14.3 samples/s +Step 33500/44000 | Loss: 2.8554 | LR: 6.93e-05 | 1400.1ms/step | 14.3 samples/s +Validation at step 33500: Loss = 2.8974, Perplexity = 18.13 +Step 33510/44000 | Loss: 2.8629 | LR: 6.91e-05 | 3603.9ms/step | 5.5 samples/s +Step 33520/44000 | Loss: 2.9003 | LR: 6.90e-05 | 1392.1ms/step | 14.4 samples/s +Step 33530/44000 | Loss: 2.8626 | LR: 6.89e-05 | 1398.1ms/step | 14.3 samples/s +Step 33540/44000 | Loss: 2.8715 | LR: 6.88e-05 | 1396.1ms/step | 14.3 samples/s +Step 33550/44000 | Loss: 2.8790 | LR: 6.86e-05 | 1391.0ms/step | 14.4 samples/s +Step 33560/44000 | Loss: 2.8587 | LR: 6.85e-05 | 1394.1ms/step | 14.3 samples/s +Step 33570/44000 | Loss: 2.8534 | LR: 6.84e-05 | 1392.2ms/step | 14.4 samples/s +Step 33580/44000 | Loss: 2.9014 | LR: 6.83e-05 | 1390.4ms/step | 14.4 samples/s +Step 33590/44000 | Loss: 2.8975 | LR: 6.81e-05 | 1392.2ms/step | 14.4 samples/s +Step 33600/44000 | Loss: 2.9036 | LR: 6.80e-05 | 1393.6ms/step | 14.4 samples/s +Step 33610/44000 | Loss: 2.8590 | LR: 6.79e-05 | 1392.2ms/step | 14.4 samples/s +Step 33620/44000 | Loss: 2.8739 | LR: 6.78e-05 | 1396.0ms/step | 14.3 samples/s +Step 33630/44000 | Loss: 2.8629 | LR: 6.76e-05 | 1394.0ms/step | 14.3 samples/s +Step 33640/44000 | Loss: 2.8756 | LR: 6.75e-05 | 1393.9ms/step | 14.3 samples/s +Step 33650/44000 | Loss: 2.8848 | LR: 6.74e-05 | 1394.4ms/step | 14.3 samples/s +Step 33660/44000 | Loss: 2.8895 | LR: 6.73e-05 | 1393.7ms/step | 14.3 samples/s +Step 33670/44000 | Loss: 2.8977 | LR: 6.71e-05 | 1395.5ms/step | 14.3 samples/s +Step 33680/44000 | Loss: 2.8828 | LR: 6.70e-05 | 1392.3ms/step | 14.4 samples/s +Step 33690/44000 | Loss: 2.8800 | LR: 6.69e-05 | 1390.0ms/step | 14.4 samples/s +Step 33700/44000 | Loss: 2.8958 | LR: 6.68e-05 | 1393.3ms/step | 14.4 samples/s +Step 33710/44000 | Loss: 2.9102 | LR: 6.66e-05 | 1392.1ms/step | 14.4 samples/s +Step 33720/44000 | Loss: 2.8877 | LR: 6.65e-05 | 1394.9ms/step | 14.3 samples/s +Step 33730/44000 | Loss: 2.8717 | LR: 6.64e-05 | 1395.8ms/step | 14.3 samples/s +Step 33740/44000 | Loss: 2.8846 | LR: 6.63e-05 | 1397.9ms/step | 14.3 samples/s +Step 33750/44000 | Loss: 2.8916 | LR: 6.61e-05 | 1398.3ms/step | 14.3 samples/s +Validation at step 33750: Loss = 2.8970, Perplexity = 18.12 +Step 33760/44000 | Loss: 2.8556 | LR: 6.60e-05 | 3596.4ms/step | 5.6 samples/s +Step 33770/44000 | Loss: 2.9047 | LR: 6.59e-05 | 1391.5ms/step | 14.4 samples/s +Step 33780/44000 | Loss: 2.9045 | LR: 6.58e-05 | 1398.1ms/step | 14.3 samples/s +Step 33790/44000 | Loss: 2.8605 | LR: 6.57e-05 | 1390.1ms/step | 14.4 samples/s +Step 33800/44000 | Loss: 2.8936 | LR: 6.55e-05 | 1393.6ms/step | 14.4 samples/s +Step 33810/44000 | Loss: 2.9115 | LR: 6.54e-05 | 1391.3ms/step | 14.4 samples/s +Step 33820/44000 | Loss: 2.8832 | LR: 6.53e-05 | 1392.9ms/step | 14.4 samples/s +Step 33830/44000 | Loss: 2.9033 | LR: 6.52e-05 | 1396.1ms/step | 14.3 samples/s +Step 33840/44000 | Loss: 2.8845 | LR: 6.50e-05 | 1394.5ms/step | 14.3 samples/s +Step 33850/44000 | Loss: 2.8951 | LR: 6.49e-05 | 1391.6ms/step | 14.4 samples/s +Step 33860/44000 | Loss: 2.8793 | LR: 6.48e-05 | 1391.9ms/step | 14.4 samples/s +Step 33870/44000 | Loss: 2.8877 | LR: 6.47e-05 | 1394.1ms/step | 14.3 samples/s +Step 33880/44000 | Loss: 2.9178 | LR: 6.46e-05 | 1395.1ms/step | 14.3 samples/s +Step 33890/44000 | Loss: 2.8764 | LR: 6.44e-05 | 1390.3ms/step | 14.4 samples/s +Step 33900/44000 | Loss: 2.9014 | LR: 6.43e-05 | 1390.3ms/step | 14.4 samples/s +Step 33910/44000 | Loss: 2.8920 | LR: 6.42e-05 | 1395.3ms/step | 14.3 samples/s +Step 33920/44000 | Loss: 2.8833 | LR: 6.41e-05 | 1388.3ms/step | 14.4 samples/s +Step 33930/44000 | Loss: 2.8423 | LR: 6.40e-05 | 1390.7ms/step | 14.4 samples/s +Step 33940/44000 | Loss: 2.8783 | LR: 6.38e-05 | 1394.0ms/step | 14.3 samples/s +Step 33950/44000 | Loss: 2.8752 | LR: 6.37e-05 | 1390.8ms/step | 14.4 samples/s +Step 33960/44000 | Loss: 2.8661 | LR: 6.36e-05 | 1392.9ms/step | 14.4 samples/s +Step 33970/44000 | Loss: 2.8727 | LR: 6.35e-05 | 1390.1ms/step | 14.4 samples/s +Step 33980/44000 | Loss: 2.9057 | LR: 6.33e-05 | 1389.4ms/step | 14.4 samples/s +Step 33990/44000 | Loss: 2.8990 | LR: 6.32e-05 | 1392.4ms/step | 14.4 samples/s +Step 34000/44000 | Loss: 2.9027 | LR: 6.31e-05 | 1394.2ms/step | 14.3 samples/s +Validation at step 34000: Loss = 2.8967, Perplexity = 18.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-34000 +Step 34010/44000 | Loss: 2.8801 | LR: 6.30e-05 | 3897.9ms/step | 5.1 samples/s +Step 34020/44000 | Loss: 2.8745 | LR: 6.29e-05 | 1396.4ms/step | 14.3 samples/s +Step 34030/44000 | Loss: 2.9242 | LR: 6.27e-05 | 1395.9ms/step | 14.3 samples/s +Step 34040/44000 | Loss: 2.8753 | LR: 6.26e-05 | 1395.2ms/step | 14.3 samples/s +Step 34050/44000 | Loss: 2.8746 | LR: 6.25e-05 | 1396.8ms/step | 14.3 samples/s +Step 34060/44000 | Loss: 2.8833 | LR: 6.24e-05 | 1391.9ms/step | 14.4 samples/s +Step 34070/44000 | Loss: 2.9061 | LR: 6.23e-05 | 1394.5ms/step | 14.3 samples/s +Step 34080/44000 | Loss: 2.8880 | LR: 6.21e-05 | 1389.4ms/step | 14.4 samples/s +Step 34090/44000 | Loss: 2.8715 | LR: 6.20e-05 | 1388.7ms/step | 14.4 samples/s +Step 34100/44000 | Loss: 2.8859 | LR: 6.19e-05 | 1391.3ms/step | 14.4 samples/s +Step 34110/44000 | Loss: 2.8522 | LR: 6.18e-05 | 1396.3ms/step | 14.3 samples/s +Step 34120/44000 | Loss: 2.8790 | LR: 6.17e-05 | 1397.4ms/step | 14.3 samples/s +Step 34130/44000 | Loss: 2.8831 | LR: 6.15e-05 | 1392.4ms/step | 14.4 samples/s +Step 34140/44000 | Loss: 2.8925 | LR: 6.14e-05 | 1390.2ms/step | 14.4 samples/s +Step 34150/44000 | Loss: 2.8857 | LR: 6.13e-05 | 1391.8ms/step | 14.4 samples/s +Step 34160/44000 | Loss: 2.9145 | LR: 6.12e-05 | 1388.3ms/step | 14.4 samples/s +Step 34170/44000 | Loss: 2.8922 | LR: 6.11e-05 | 1386.7ms/step | 14.4 samples/s +Step 34180/44000 | Loss: 2.8684 | LR: 6.09e-05 | 1391.3ms/step | 14.4 samples/s +Step 34190/44000 | Loss: 2.8926 | LR: 6.08e-05 | 1393.7ms/step | 14.4 samples/s +Step 34200/44000 | Loss: 2.9103 | LR: 6.07e-05 | 1394.3ms/step | 14.3 samples/s +Step 34210/44000 | Loss: 2.8897 | LR: 6.06e-05 | 1397.9ms/step | 14.3 samples/s +Step 34220/44000 | Loss: 2.8825 | LR: 6.05e-05 | 1392.2ms/step | 14.4 samples/s +Step 34230/44000 | Loss: 2.8984 | LR: 6.04e-05 | 1400.1ms/step | 14.3 samples/s +Step 34240/44000 | Loss: 2.9157 | LR: 6.02e-05 | 1399.8ms/step | 14.3 samples/s +Step 34250/44000 | Loss: 2.8936 | LR: 6.01e-05 | 1400.1ms/step | 14.3 samples/s +Validation at step 34250: Loss = 2.8964, Perplexity = 18.11 +Step 34260/44000 | Loss: 2.8909 | LR: 6.00e-05 | 3614.2ms/step | 5.5 samples/s +Step 34270/44000 | Loss: 2.8653 | LR: 5.99e-05 | 1399.7ms/step | 14.3 samples/s +Step 34280/44000 | Loss: 2.9078 | LR: 5.98e-05 | 1398.2ms/step | 14.3 samples/s +Step 34290/44000 | Loss: 2.8685 | LR: 5.96e-05 | 1398.1ms/step | 14.3 samples/s +Step 34300/44000 | Loss: 2.8663 | LR: 5.95e-05 | 1400.0ms/step | 14.3 samples/s +Step 34310/44000 | Loss: 2.8815 | LR: 5.94e-05 | 1399.8ms/step | 14.3 samples/s +Step 34320/44000 | Loss: 2.9299 | LR: 5.93e-05 | 1400.0ms/step | 14.3 samples/s +Step 34330/44000 | Loss: 2.8526 | LR: 5.92e-05 | 1400.1ms/step | 14.3 samples/s +Step 34340/44000 | Loss: 2.8547 | LR: 5.91e-05 | 1397.4ms/step | 14.3 samples/s +Step 34350/44000 | Loss: 2.8762 | LR: 5.89e-05 | 1390.4ms/step | 14.4 samples/s +Step 34360/44000 | Loss: 2.8888 | LR: 5.88e-05 | 1393.1ms/step | 14.4 samples/s +Step 34370/44000 | Loss: 2.8757 | LR: 5.87e-05 | 1386.1ms/step | 14.4 samples/s +Step 34380/44000 | Loss: 2.8757 | LR: 5.86e-05 | 1390.7ms/step | 14.4 samples/s +Step 34390/44000 | Loss: 2.8870 | LR: 5.85e-05 | 1390.3ms/step | 14.4 samples/s +Step 34400/44000 | Loss: 2.8868 | LR: 5.84e-05 | 1390.5ms/step | 14.4 samples/s +Step 34410/44000 | Loss: 2.9044 | LR: 5.82e-05 | 1393.8ms/step | 14.3 samples/s +Step 34420/44000 | Loss: 2.8828 | LR: 5.81e-05 | 1395.9ms/step | 14.3 samples/s +Step 34430/44000 | Loss: 2.8798 | LR: 5.80e-05 | 1395.5ms/step | 14.3 samples/s +Step 34440/44000 | Loss: 2.8917 | LR: 5.79e-05 | 1391.9ms/step | 14.4 samples/s +Step 34450/44000 | Loss: 2.8697 | LR: 5.78e-05 | 1393.9ms/step | 14.3 samples/s +Step 34460/44000 | Loss: 2.9007 | LR: 5.77e-05 | 1392.6ms/step | 14.4 samples/s +Step 34470/44000 | Loss: 2.8829 | LR: 5.75e-05 | 1395.8ms/step | 14.3 samples/s +Step 34480/44000 | Loss: 2.8895 | LR: 5.74e-05 | 1398.0ms/step | 14.3 samples/s +Step 34490/44000 | Loss: 2.8523 | LR: 5.73e-05 | 1399.9ms/step | 14.3 samples/s +Step 34500/44000 | Loss: 2.8989 | LR: 5.72e-05 | 1394.0ms/step | 14.3 samples/s +Validation at step 34500: Loss = 2.8961, Perplexity = 18.10 +Step 34510/44000 | Loss: 2.8807 | LR: 5.71e-05 | 3595.7ms/step | 5.6 samples/s +Step 34520/44000 | Loss: 2.8756 | LR: 5.70e-05 | 1392.5ms/step | 14.4 samples/s +Step 34530/44000 | Loss: 2.8653 | LR: 5.69e-05 | 1392.0ms/step | 14.4 samples/s +Step 34540/44000 | Loss: 2.8854 | LR: 5.67e-05 | 1394.1ms/step | 14.3 samples/s +Step 34550/44000 | Loss: 2.9043 | LR: 5.66e-05 | 1397.9ms/step | 14.3 samples/s +Step 34560/44000 | Loss: 2.8527 | LR: 5.65e-05 | 1396.1ms/step | 14.3 samples/s +Step 34570/44000 | Loss: 2.8705 | LR: 5.64e-05 | 1395.9ms/step | 14.3 samples/s +Step 34580/44000 | Loss: 2.8633 | LR: 5.63e-05 | 1396.1ms/step | 14.3 samples/s +Step 34590/44000 | Loss: 2.8758 | LR: 5.62e-05 | 1395.8ms/step | 14.3 samples/s +Step 34600/44000 | Loss: 2.8865 | LR: 5.60e-05 | 1392.1ms/step | 14.4 samples/s +Step 34610/44000 | Loss: 2.8885 | LR: 5.59e-05 | 1396.3ms/step | 14.3 samples/s +Step 34620/44000 | Loss: 2.8836 | LR: 5.58e-05 | 1400.1ms/step | 14.3 samples/s +Step 34630/44000 | Loss: 2.8707 | LR: 5.57e-05 | 1399.6ms/step | 14.3 samples/s +Step 34640/44000 | Loss: 2.8799 | LR: 5.56e-05 | 1400.2ms/step | 14.3 samples/s +Step 34650/44000 | Loss: 2.8937 | LR: 5.55e-05 | 1398.0ms/step | 14.3 samples/s +Step 34660/44000 | Loss: 2.9005 | LR: 5.54e-05 | 1396.9ms/step | 14.3 samples/s +Step 34670/44000 | Loss: 2.8515 | LR: 5.52e-05 | 1392.1ms/step | 14.4 samples/s +Step 34680/44000 | Loss: 2.8839 | LR: 5.51e-05 | 1396.6ms/step | 14.3 samples/s +Step 34690/44000 | Loss: 2.8675 | LR: 5.50e-05 | 1395.2ms/step | 14.3 samples/s +Step 34700/44000 | Loss: 2.8584 | LR: 5.49e-05 | 1394.5ms/step | 14.3 samples/s +Step 34710/44000 | Loss: 2.8949 | LR: 5.48e-05 | 1394.2ms/step | 14.3 samples/s +Step 34720/44000 | Loss: 2.8691 | LR: 5.47e-05 | 1392.4ms/step | 14.4 samples/s +Step 34730/44000 | Loss: 2.8960 | LR: 5.46e-05 | 1396.0ms/step | 14.3 samples/s +Step 34740/44000 | Loss: 2.8692 | LR: 5.45e-05 | 1395.8ms/step | 14.3 samples/s +Step 34750/44000 | Loss: 2.8783 | LR: 5.43e-05 | 1398.5ms/step | 14.3 samples/s +Validation at step 34750: Loss = 2.8958, Perplexity = 18.10 +Step 34760/44000 | Loss: 2.8644 | LR: 5.42e-05 | 3594.2ms/step | 5.6 samples/s +Step 34770/44000 | Loss: 2.8719 | LR: 5.41e-05 | 1388.4ms/step | 14.4 samples/s +Step 34780/44000 | Loss: 2.9099 | LR: 5.40e-05 | 1389.4ms/step | 14.4 samples/s +Step 34790/44000 | Loss: 2.9059 | LR: 5.39e-05 | 1392.0ms/step | 14.4 samples/s +Step 34800/44000 | Loss: 2.8855 | LR: 5.38e-05 | 1393.4ms/step | 14.4 samples/s +Step 34810/44000 | Loss: 2.9027 | LR: 5.37e-05 | 1391.6ms/step | 14.4 samples/s +Step 34820/44000 | Loss: 2.8819 | LR: 5.36e-05 | 1391.1ms/step | 14.4 samples/s +Step 34830/44000 | Loss: 2.8808 | LR: 5.34e-05 | 1394.1ms/step | 14.3 samples/s +Step 34840/44000 | Loss: 2.8900 | LR: 5.33e-05 | 1393.8ms/step | 14.3 samples/s +Step 34850/44000 | Loss: 2.8991 | LR: 5.32e-05 | 1395.4ms/step | 14.3 samples/s +Step 34860/44000 | Loss: 2.8909 | LR: 5.31e-05 | 1394.6ms/step | 14.3 samples/s +Step 34870/44000 | Loss: 2.8718 | LR: 5.30e-05 | 1389.6ms/step | 14.4 samples/s +Step 34880/44000 | Loss: 2.8790 | LR: 5.29e-05 | 1390.5ms/step | 14.4 samples/s +Step 34890/44000 | Loss: 2.8741 | LR: 5.28e-05 | 1389.9ms/step | 14.4 samples/s +Step 34900/44000 | Loss: 2.8921 | LR: 5.27e-05 | 1390.3ms/step | 14.4 samples/s +Step 34910/44000 | Loss: 2.8895 | LR: 5.25e-05 | 1397.5ms/step | 14.3 samples/s +Step 34920/44000 | Loss: 2.8749 | LR: 5.24e-05 | 1393.7ms/step | 14.4 samples/s +Step 34930/44000 | Loss: 2.8632 | LR: 5.23e-05 | 1394.8ms/step | 14.3 samples/s +Step 34940/44000 | Loss: 2.8363 | LR: 5.22e-05 | 1399.9ms/step | 14.3 samples/s +Step 34950/44000 | Loss: 2.8945 | LR: 5.21e-05 | 1397.8ms/step | 14.3 samples/s +Step 34960/44000 | Loss: 2.8855 | LR: 5.20e-05 | 1394.2ms/step | 14.3 samples/s +Step 34970/44000 | Loss: 2.8650 | LR: 5.19e-05 | 1394.0ms/step | 14.3 samples/s +Step 34980/44000 | Loss: 2.8875 | LR: 5.18e-05 | 1387.9ms/step | 14.4 samples/s +Step 34990/44000 | Loss: 2.8923 | LR: 5.17e-05 | 1393.9ms/step | 14.3 samples/s +Step 35000/44000 | Loss: 2.8600 | LR: 5.15e-05 | 1389.7ms/step | 14.4 samples/s +Validation at step 35000: Loss = 2.8955, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-35000 +Step 35010/44000 | Loss: 2.8840 | LR: 5.14e-05 | 3893.8ms/step | 5.1 samples/s +Step 35020/44000 | Loss: 2.8526 | LR: 5.13e-05 | 1391.9ms/step | 14.4 samples/s +Step 35030/44000 | Loss: 2.8612 | LR: 5.12e-05 | 1392.9ms/step | 14.4 samples/s +Step 35040/44000 | Loss: 2.8717 | LR: 5.11e-05 | 1393.2ms/step | 14.4 samples/s +Step 35050/44000 | Loss: 2.8630 | LR: 5.10e-05 | 1392.8ms/step | 14.4 samples/s +Step 35060/44000 | Loss: 2.8742 | LR: 5.09e-05 | 1395.1ms/step | 14.3 samples/s +Step 35070/44000 | Loss: 2.8422 | LR: 5.08e-05 | 1393.0ms/step | 14.4 samples/s +Step 35080/44000 | Loss: 2.8778 | LR: 5.07e-05 | 1395.8ms/step | 14.3 samples/s +Step 35090/44000 | Loss: 2.8660 | LR: 5.06e-05 | 1393.9ms/step | 14.3 samples/s +Step 35100/44000 | Loss: 2.9079 | LR: 5.04e-05 | 1398.1ms/step | 14.3 samples/s +Step 35110/44000 | Loss: 2.8705 | LR: 5.03e-05 | 1398.1ms/step | 14.3 samples/s +Step 35120/44000 | Loss: 2.8871 | LR: 5.02e-05 | 1397.7ms/step | 14.3 samples/s +Step 35130/44000 | Loss: 2.8840 | LR: 5.01e-05 | 1400.0ms/step | 14.3 samples/s +Step 35140/44000 | Loss: 2.8528 | LR: 5.00e-05 | 1391.9ms/step | 14.4 samples/s +Step 35150/44000 | Loss: 2.8924 | LR: 4.99e-05 | 1389.7ms/step | 14.4 samples/s +Step 35160/44000 | Loss: 2.9133 | LR: 4.98e-05 | 1388.4ms/step | 14.4 samples/s +Step 35170/44000 | Loss: 2.8905 | LR: 4.97e-05 | 1388.3ms/step | 14.4 samples/s +Step 35180/44000 | Loss: 2.8975 | LR: 4.96e-05 | 1390.1ms/step | 14.4 samples/s +Step 35190/44000 | Loss: 2.8642 | LR: 4.95e-05 | 1394.1ms/step | 14.3 samples/s +Step 35200/44000 | Loss: 2.8948 | LR: 4.94e-05 | 1395.7ms/step | 14.3 samples/s +Step 35210/44000 | Loss: 2.8932 | LR: 4.93e-05 | 1394.1ms/step | 14.3 samples/s +Step 35220/44000 | Loss: 2.8702 | LR: 4.91e-05 | 1397.2ms/step | 14.3 samples/s +Step 35230/44000 | Loss: 2.8776 | LR: 4.90e-05 | 1392.7ms/step | 14.4 samples/s +Step 35240/44000 | Loss: 2.8671 | LR: 4.89e-05 | 1389.8ms/step | 14.4 samples/s +Step 35250/44000 | Loss: 2.9141 | LR: 4.88e-05 | 1390.0ms/step | 14.4 samples/s +Validation at step 35250: Loss = 2.8953, Perplexity = 18.09 +Step 35260/44000 | Loss: 2.8711 | LR: 4.87e-05 | 3595.8ms/step | 5.6 samples/s +Step 35270/44000 | Loss: 2.9026 | LR: 4.86e-05 | 1393.2ms/step | 14.4 samples/s +Step 35280/44000 | Loss: 2.8588 | LR: 4.85e-05 | 1395.3ms/step | 14.3 samples/s +Step 35290/44000 | Loss: 2.8671 | LR: 4.84e-05 | 1398.1ms/step | 14.3 samples/s +Step 35300/44000 | Loss: 2.8693 | LR: 4.83e-05 | 1397.9ms/step | 14.3 samples/s +Step 35310/44000 | Loss: 2.8892 | LR: 4.82e-05 | 1391.9ms/step | 14.4 samples/s +Step 35320/44000 | Loss: 2.8518 | LR: 4.81e-05 | 1389.8ms/step | 14.4 samples/s +Step 35330/44000 | Loss: 2.8866 | LR: 4.80e-05 | 1390.5ms/step | 14.4 samples/s +Step 35340/44000 | Loss: 2.8819 | LR: 4.79e-05 | 1395.9ms/step | 14.3 samples/s +Step 35350/44000 | Loss: 2.9096 | LR: 4.77e-05 | 1393.4ms/step | 14.4 samples/s +Step 35360/44000 | Loss: 2.8648 | LR: 4.76e-05 | 1391.5ms/step | 14.4 samples/s +Step 35370/44000 | Loss: 2.8757 | LR: 4.75e-05 | 1393.2ms/step | 14.4 samples/s +Step 35380/44000 | Loss: 2.8630 | LR: 4.74e-05 | 1390.0ms/step | 14.4 samples/s +Step 35390/44000 | Loss: 2.8704 | LR: 4.73e-05 | 1393.8ms/step | 14.3 samples/s +Step 35400/44000 | Loss: 2.8581 | LR: 4.72e-05 | 1395.8ms/step | 14.3 samples/s +Step 35410/44000 | Loss: 2.8922 | LR: 4.71e-05 | 1396.4ms/step | 14.3 samples/s +Step 35420/44000 | Loss: 2.8617 | LR: 4.70e-05 | 1397.9ms/step | 14.3 samples/s +Step 35430/44000 | Loss: 2.9219 | LR: 4.69e-05 | 1397.9ms/step | 14.3 samples/s +Step 35440/44000 | Loss: 2.9016 | LR: 4.68e-05 | 1392.4ms/step | 14.4 samples/s +Step 35450/44000 | Loss: 2.8842 | LR: 4.67e-05 | 1399.9ms/step | 14.3 samples/s +Step 35460/44000 | Loss: 2.9038 | LR: 4.66e-05 | 1399.9ms/step | 14.3 samples/s +Step 35470/44000 | Loss: 2.8761 | LR: 4.65e-05 | 1394.0ms/step | 14.3 samples/s +Step 35480/44000 | Loss: 2.8860 | LR: 4.64e-05 | 1395.8ms/step | 14.3 samples/s +Step 35490/44000 | Loss: 2.8899 | LR: 4.63e-05 | 1399.9ms/step | 14.3 samples/s +Step 35500/44000 | Loss: 2.8626 | LR: 4.62e-05 | 1400.4ms/step | 14.3 samples/s +Validation at step 35500: Loss = 2.8951, Perplexity = 18.09 +Step 35510/44000 | Loss: 2.8751 | LR: 4.61e-05 | 3619.9ms/step | 5.5 samples/s +Step 35520/44000 | Loss: 2.8832 | LR: 4.59e-05 | 1400.1ms/step | 14.3 samples/s +Step 35530/44000 | Loss: 2.8828 | LR: 4.58e-05 | 1400.2ms/step | 14.3 samples/s +Step 35540/44000 | Loss: 2.9015 | LR: 4.57e-05 | 1399.8ms/step | 14.3 samples/s +Step 35550/44000 | Loss: 2.9030 | LR: 4.56e-05 | 1400.1ms/step | 14.3 samples/s +Step 35560/44000 | Loss: 2.8928 | LR: 4.55e-05 | 1400.0ms/step | 14.3 samples/s +Step 35570/44000 | Loss: 2.9030 | LR: 4.54e-05 | 1400.0ms/step | 14.3 samples/s +Step 35580/44000 | Loss: 2.8853 | LR: 4.53e-05 | 1400.1ms/step | 14.3 samples/s +Step 35590/44000 | Loss: 2.8952 | LR: 4.52e-05 | 1399.9ms/step | 14.3 samples/s +Step 35600/44000 | Loss: 2.8588 | LR: 4.51e-05 | 1399.6ms/step | 14.3 samples/s +Step 35610/44000 | Loss: 2.8991 | LR: 4.50e-05 | 1394.4ms/step | 14.3 samples/s +Step 35620/44000 | Loss: 2.8755 | LR: 4.49e-05 | 1391.4ms/step | 14.4 samples/s +Step 35630/44000 | Loss: 2.8867 | LR: 4.48e-05 | 1390.7ms/step | 14.4 samples/s +Step 35640/44000 | Loss: 2.8640 | LR: 4.47e-05 | 1389.9ms/step | 14.4 samples/s +Step 35650/44000 | Loss: 2.8719 | LR: 4.46e-05 | 1392.6ms/step | 14.4 samples/s +Step 35660/44000 | Loss: 2.8871 | LR: 4.45e-05 | 1393.0ms/step | 14.4 samples/s +Step 35670/44000 | Loss: 2.8765 | LR: 4.44e-05 | 1391.7ms/step | 14.4 samples/s +Step 35680/44000 | Loss: 2.8790 | LR: 4.43e-05 | 1392.5ms/step | 14.4 samples/s +Step 35690/44000 | Loss: 2.8916 | LR: 4.42e-05 | 1388.8ms/step | 14.4 samples/s +Step 35700/44000 | Loss: 2.8683 | LR: 4.41e-05 | 1386.6ms/step | 14.4 samples/s +Step 35710/44000 | Loss: 2.8832 | LR: 4.40e-05 | 1386.6ms/step | 14.4 samples/s +Step 35720/44000 | Loss: 2.8654 | LR: 4.39e-05 | 1389.3ms/step | 14.4 samples/s +Step 35730/44000 | Loss: 2.8575 | LR: 4.38e-05 | 1391.0ms/step | 14.4 samples/s +Step 35740/44000 | Loss: 2.8863 | LR: 4.37e-05 | 1392.0ms/step | 14.4 samples/s +Step 35750/44000 | Loss: 2.8681 | LR: 4.36e-05 | 1392.8ms/step | 14.4 samples/s +Validation at step 35750: Loss = 2.8949, Perplexity = 18.08 +Step 35760/44000 | Loss: 2.8980 | LR: 4.35e-05 | 3601.1ms/step | 5.6 samples/s +Step 35770/44000 | Loss: 2.8721 | LR: 4.34e-05 | 1400.1ms/step | 14.3 samples/s +Step 35780/44000 | Loss: 2.8623 | LR: 4.33e-05 | 1400.1ms/step | 14.3 samples/s +Step 35790/44000 | Loss: 2.8899 | LR: 4.32e-05 | 1392.7ms/step | 14.4 samples/s +Step 35800/44000 | Loss: 2.8977 | LR: 4.31e-05 | 1388.1ms/step | 14.4 samples/s +Step 35810/44000 | Loss: 2.8917 | LR: 4.29e-05 | 1391.4ms/step | 14.4 samples/s +Step 35820/44000 | Loss: 2.8952 | LR: 4.28e-05 | 1395.5ms/step | 14.3 samples/s +Step 35830/44000 | Loss: 2.9139 | LR: 4.27e-05 | 1393.9ms/step | 14.3 samples/s +Step 35840/44000 | Loss: 2.8739 | LR: 4.26e-05 | 1398.5ms/step | 14.3 samples/s +Step 35850/44000 | Loss: 2.8828 | LR: 4.25e-05 | 1388.6ms/step | 14.4 samples/s +Step 35860/44000 | Loss: 2.9068 | LR: 4.24e-05 | 1391.2ms/step | 14.4 samples/s +Step 35870/44000 | Loss: 2.8823 | LR: 4.23e-05 | 1396.0ms/step | 14.3 samples/s +Step 35880/44000 | Loss: 2.9001 | LR: 4.22e-05 | 1394.2ms/step | 14.3 samples/s +Step 35890/44000 | Loss: 2.8644 | LR: 4.21e-05 | 1393.9ms/step | 14.3 samples/s +Step 35900/44000 | Loss: 2.8853 | LR: 4.20e-05 | 1396.1ms/step | 14.3 samples/s +Step 35910/44000 | Loss: 2.8687 | LR: 4.19e-05 | 1398.2ms/step | 14.3 samples/s +Step 35920/44000 | Loss: 2.8838 | LR: 4.18e-05 | 1397.9ms/step | 14.3 samples/s +Step 35930/44000 | Loss: 2.8702 | LR: 4.17e-05 | 1395.9ms/step | 14.3 samples/s +Step 35940/44000 | Loss: 2.8727 | LR: 4.16e-05 | 1394.0ms/step | 14.3 samples/s +Step 35950/44000 | Loss: 2.8589 | LR: 4.15e-05 | 1393.8ms/step | 14.3 samples/s +Step 35960/44000 | Loss: 2.8622 | LR: 4.14e-05 | 1400.1ms/step | 14.3 samples/s +Step 35970/44000 | Loss: 2.8543 | LR: 4.13e-05 | 1396.1ms/step | 14.3 samples/s +Step 35980/44000 | Loss: 2.9077 | LR: 4.12e-05 | 1400.0ms/step | 14.3 samples/s +Step 35990/44000 | Loss: 2.8776 | LR: 4.11e-05 | 1399.7ms/step | 14.3 samples/s +Step 36000/44000 | Loss: 2.8784 | LR: 4.10e-05 | 1396.2ms/step | 14.3 samples/s +Validation at step 36000: Loss = 2.8948, Perplexity = 18.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-36000 +Step 36010/44000 | Loss: 2.8943 | LR: 4.09e-05 | 3888.0ms/step | 5.1 samples/s +Step 36020/44000 | Loss: 2.8715 | LR: 4.08e-05 | 1396.2ms/step | 14.3 samples/s +Step 36030/44000 | Loss: 2.8616 | LR: 4.07e-05 | 1395.3ms/step | 14.3 samples/s +Step 36040/44000 | Loss: 2.8681 | LR: 4.06e-05 | 1392.6ms/step | 14.4 samples/s +Step 36050/44000 | Loss: 2.8644 | LR: 4.05e-05 | 1398.2ms/step | 14.3 samples/s +Step 36060/44000 | Loss: 2.8939 | LR: 4.04e-05 | 1395.5ms/step | 14.3 samples/s +Step 36070/44000 | Loss: 2.8838 | LR: 4.03e-05 | 1394.4ms/step | 14.3 samples/s +Step 36080/44000 | Loss: 2.8990 | LR: 4.02e-05 | 1396.0ms/step | 14.3 samples/s +Step 36090/44000 | Loss: 2.9050 | LR: 4.01e-05 | 1392.0ms/step | 14.4 samples/s +Step 36100/44000 | Loss: 2.8851 | LR: 4.00e-05 | 1390.0ms/step | 14.4 samples/s +Step 36110/44000 | Loss: 2.8988 | LR: 3.99e-05 | 1389.4ms/step | 14.4 samples/s +Step 36120/44000 | Loss: 2.8852 | LR: 3.98e-05 | 1391.7ms/step | 14.4 samples/s +Step 36130/44000 | Loss: 2.8883 | LR: 3.97e-05 | 1390.8ms/step | 14.4 samples/s +Step 36140/44000 | Loss: 2.8793 | LR: 3.97e-05 | 1394.1ms/step | 14.3 samples/s +Step 36150/44000 | Loss: 2.8902 | LR: 3.96e-05 | 1395.1ms/step | 14.3 samples/s +Step 36160/44000 | Loss: 2.8827 | LR: 3.95e-05 | 1395.0ms/step | 14.3 samples/s +Step 36170/44000 | Loss: 2.8412 | LR: 3.94e-05 | 1395.8ms/step | 14.3 samples/s +Step 36180/44000 | Loss: 2.8812 | LR: 3.93e-05 | 1396.0ms/step | 14.3 samples/s +Step 36190/44000 | Loss: 2.8786 | LR: 3.92e-05 | 1398.2ms/step | 14.3 samples/s +Step 36200/44000 | Loss: 2.8718 | LR: 3.91e-05 | 1395.9ms/step | 14.3 samples/s +Step 36210/44000 | Loss: 2.8803 | LR: 3.90e-05 | 1400.2ms/step | 14.3 samples/s +Step 36220/44000 | Loss: 2.8746 | LR: 3.89e-05 | 1400.1ms/step | 14.3 samples/s +Step 36230/44000 | Loss: 2.8758 | LR: 3.88e-05 | 1399.8ms/step | 14.3 samples/s +Step 36240/44000 | Loss: 2.8861 | LR: 3.87e-05 | 1400.0ms/step | 14.3 samples/s +Step 36250/44000 | Loss: 2.9016 | LR: 3.86e-05 | 1400.0ms/step | 14.3 samples/s +Validation at step 36250: Loss = 2.8946, Perplexity = 18.08 +Step 36260/44000 | Loss: 2.8922 | LR: 3.85e-05 | 3607.9ms/step | 5.5 samples/s +Step 36270/44000 | Loss: 2.9027 | LR: 3.84e-05 | 1393.3ms/step | 14.4 samples/s +Step 36280/44000 | Loss: 2.8739 | LR: 3.83e-05 | 1386.8ms/step | 14.4 samples/s +Step 36290/44000 | Loss: 2.8887 | LR: 3.82e-05 | 1389.0ms/step | 14.4 samples/s +Step 36300/44000 | Loss: 2.8732 | LR: 3.81e-05 | 1389.2ms/step | 14.4 samples/s +Step 36310/44000 | Loss: 2.8751 | LR: 3.80e-05 | 1389.7ms/step | 14.4 samples/s +Step 36320/44000 | Loss: 2.8498 | LR: 3.79e-05 | 1394.1ms/step | 14.3 samples/s +Step 36330/44000 | Loss: 2.8676 | LR: 3.78e-05 | 1392.0ms/step | 14.4 samples/s +Step 36340/44000 | Loss: 2.8867 | LR: 3.77e-05 | 1388.3ms/step | 14.4 samples/s +Step 36350/44000 | Loss: 2.8557 | LR: 3.76e-05 | 1392.0ms/step | 14.4 samples/s +Step 36360/44000 | Loss: 2.8647 | LR: 3.75e-05 | 1395.9ms/step | 14.3 samples/s +Step 36370/44000 | Loss: 2.8748 | LR: 3.74e-05 | 1394.3ms/step | 14.3 samples/s +Step 36380/44000 | Loss: 2.8744 | LR: 3.73e-05 | 1393.3ms/step | 14.4 samples/s +Step 36390/44000 | Loss: 2.8974 | LR: 3.72e-05 | 1388.5ms/step | 14.4 samples/s +Step 36400/44000 | Loss: 2.8730 | LR: 3.71e-05 | 1395.9ms/step | 14.3 samples/s +Step 36410/44000 | Loss: 2.8837 | LR: 3.70e-05 | 1391.2ms/step | 14.4 samples/s +Step 36420/44000 | Loss: 2.8519 | LR: 3.69e-05 | 1394.4ms/step | 14.3 samples/s +Step 36430/44000 | Loss: 2.8604 | LR: 3.69e-05 | 1396.2ms/step | 14.3 samples/s +Step 36440/44000 | Loss: 2.8690 | LR: 3.68e-05 | 1392.2ms/step | 14.4 samples/s +Step 36450/44000 | Loss: 2.9103 | LR: 3.67e-05 | 1393.6ms/step | 14.4 samples/s +Step 36460/44000 | Loss: 2.8843 | LR: 3.66e-05 | 1388.4ms/step | 14.4 samples/s +Step 36470/44000 | Loss: 2.8584 | LR: 3.65e-05 | 1389.0ms/step | 14.4 samples/s +Step 36480/44000 | Loss: 2.8819 | LR: 3.64e-05 | 1388.5ms/step | 14.4 samples/s +Step 36490/44000 | Loss: 2.9046 | LR: 3.63e-05 | 1387.6ms/step | 14.4 samples/s +Step 36500/44000 | Loss: 2.8538 | LR: 3.62e-05 | 1388.8ms/step | 14.4 samples/s +Validation at step 36500: Loss = 2.8944, Perplexity = 18.07 +Step 36510/44000 | Loss: 2.8931 | LR: 3.61e-05 | 3590.2ms/step | 5.6 samples/s +Step 36520/44000 | Loss: 2.8672 | LR: 3.60e-05 | 1394.2ms/step | 14.3 samples/s +Step 36530/44000 | Loss: 2.8762 | LR: 3.59e-05 | 1392.8ms/step | 14.4 samples/s +Step 36540/44000 | Loss: 2.8621 | LR: 3.58e-05 | 1389.2ms/step | 14.4 samples/s +Step 36550/44000 | Loss: 2.8717 | LR: 3.57e-05 | 1389.0ms/step | 14.4 samples/s +Step 36560/44000 | Loss: 2.8916 | LR: 3.56e-05 | 1392.5ms/step | 14.4 samples/s +Step 36570/44000 | Loss: 2.8530 | LR: 3.55e-05 | 1395.4ms/step | 14.3 samples/s +Step 36580/44000 | Loss: 2.8700 | LR: 3.54e-05 | 1392.6ms/step | 14.4 samples/s +Step 36590/44000 | Loss: 2.8812 | LR: 3.53e-05 | 1390.2ms/step | 14.4 samples/s +Step 36600/44000 | Loss: 2.8524 | LR: 3.53e-05 | 1396.2ms/step | 14.3 samples/s +Step 36610/44000 | Loss: 2.8746 | LR: 3.52e-05 | 1391.8ms/step | 14.4 samples/s +Step 36620/44000 | Loss: 2.8659 | LR: 3.51e-05 | 1392.5ms/step | 14.4 samples/s +Step 36630/44000 | Loss: 2.8808 | LR: 3.50e-05 | 1397.8ms/step | 14.3 samples/s +Step 36640/44000 | Loss: 2.8791 | LR: 3.49e-05 | 1393.8ms/step | 14.3 samples/s +Step 36650/44000 | Loss: 2.8747 | LR: 3.48e-05 | 1392.1ms/step | 14.4 samples/s +Step 36660/44000 | Loss: 2.8669 | LR: 3.47e-05 | 1392.4ms/step | 14.4 samples/s +Step 36670/44000 | Loss: 2.9064 | LR: 3.46e-05 | 1396.9ms/step | 14.3 samples/s +Step 36680/44000 | Loss: 2.8939 | LR: 3.45e-05 | 1398.5ms/step | 14.3 samples/s +Step 36690/44000 | Loss: 2.8712 | LR: 3.44e-05 | 1392.2ms/step | 14.4 samples/s +Step 36700/44000 | Loss: 2.8716 | LR: 3.43e-05 | 1388.9ms/step | 14.4 samples/s +Step 36710/44000 | Loss: 2.8642 | LR: 3.42e-05 | 1387.2ms/step | 14.4 samples/s +Step 36720/44000 | Loss: 2.8773 | LR: 3.41e-05 | 1389.9ms/step | 14.4 samples/s +Step 36730/44000 | Loss: 2.8680 | LR: 3.41e-05 | 1398.2ms/step | 14.3 samples/s +Step 36740/44000 | Loss: 2.8917 | LR: 3.40e-05 | 1400.1ms/step | 14.3 samples/s +Step 36750/44000 | Loss: 2.8628 | LR: 3.39e-05 | 1400.0ms/step | 14.3 samples/s +Validation at step 36750: Loss = 2.8943, Perplexity = 18.07 +Step 36760/44000 | Loss: 2.8687 | LR: 3.38e-05 | 3622.0ms/step | 5.5 samples/s +Step 36770/44000 | Loss: 2.8813 | LR: 3.37e-05 | 1400.0ms/step | 14.3 samples/s +Step 36780/44000 | Loss: 2.8900 | LR: 3.36e-05 | 1400.0ms/step | 14.3 samples/s +Step 36790/44000 | Loss: 2.8868 | LR: 3.35e-05 | 1400.0ms/step | 14.3 samples/s +Step 36800/44000 | Loss: 2.8787 | LR: 3.34e-05 | 1399.9ms/step | 14.3 samples/s +Step 36810/44000 | Loss: 2.8734 | LR: 3.33e-05 | 1400.3ms/step | 14.3 samples/s +Step 36820/44000 | Loss: 2.8732 | LR: 3.32e-05 | 1399.8ms/step | 14.3 samples/s +Step 36830/44000 | Loss: 2.8745 | LR: 3.31e-05 | 1397.9ms/step | 14.3 samples/s +Step 36840/44000 | Loss: 2.8797 | LR: 3.31e-05 | 1398.1ms/step | 14.3 samples/s +Step 36850/44000 | Loss: 2.8801 | LR: 3.30e-05 | 1395.3ms/step | 14.3 samples/s +Step 36860/44000 | Loss: 2.8612 | LR: 3.29e-05 | 1391.7ms/step | 14.4 samples/s +Step 36870/44000 | Loss: 2.8825 | LR: 3.28e-05 | 1392.9ms/step | 14.4 samples/s +Step 36880/44000 | Loss: 2.8544 | LR: 3.27e-05 | 1396.3ms/step | 14.3 samples/s +Step 36890/44000 | Loss: 2.8517 | LR: 3.26e-05 | 1395.9ms/step | 14.3 samples/s +Step 36900/44000 | Loss: 2.8560 | LR: 3.25e-05 | 1400.1ms/step | 14.3 samples/s +Step 36910/44000 | Loss: 2.8728 | LR: 3.24e-05 | 1399.8ms/step | 14.3 samples/s +Step 36920/44000 | Loss: 2.8722 | LR: 3.23e-05 | 1400.0ms/step | 14.3 samples/s +Step 36930/44000 | Loss: 2.8496 | LR: 3.22e-05 | 1396.3ms/step | 14.3 samples/s +Step 36940/44000 | Loss: 2.8903 | LR: 3.22e-05 | 1391.9ms/step | 14.4 samples/s +Step 36950/44000 | Loss: 2.8852 | LR: 3.21e-05 | 1388.8ms/step | 14.4 samples/s +Step 36960/44000 | Loss: 2.8668 | LR: 3.20e-05 | 1389.0ms/step | 14.4 samples/s +Step 36970/44000 | Loss: 2.8501 | LR: 3.19e-05 | 1391.5ms/step | 14.4 samples/s +Step 36980/44000 | Loss: 2.8721 | LR: 3.18e-05 | 1388.2ms/step | 14.4 samples/s +Step 36990/44000 | Loss: 2.8829 | LR: 3.17e-05 | 1389.9ms/step | 14.4 samples/s +Step 37000/44000 | Loss: 2.8929 | LR: 3.16e-05 | 1390.0ms/step | 14.4 samples/s +Validation at step 37000: Loss = 2.8943, Perplexity = 18.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-37000 +Step 37010/44000 | Loss: 2.8915 | LR: 3.15e-05 | 3892.5ms/step | 5.1 samples/s +Step 37020/44000 | Loss: 2.9278 | LR: 3.15e-05 | 1392.4ms/step | 14.4 samples/s +Step 37030/44000 | Loss: 2.8947 | LR: 3.14e-05 | 1389.5ms/step | 14.4 samples/s +Step 37040/44000 | Loss: 2.8916 | LR: 3.13e-05 | 1392.5ms/step | 14.4 samples/s +Step 37050/44000 | Loss: 2.8829 | LR: 3.12e-05 | 1391.3ms/step | 14.4 samples/s +Step 37060/44000 | Loss: 2.8507 | LR: 3.11e-05 | 1397.4ms/step | 14.3 samples/s +Step 37070/44000 | Loss: 2.8812 | LR: 3.10e-05 | 1394.8ms/step | 14.3 samples/s +Step 37080/44000 | Loss: 2.9116 | LR: 3.09e-05 | 1392.4ms/step | 14.4 samples/s +Step 37090/44000 | Loss: 2.8906 | LR: 3.08e-05 | 1393.8ms/step | 14.3 samples/s +Step 37100/44000 | Loss: 2.8739 | LR: 3.07e-05 | 1391.6ms/step | 14.4 samples/s +Step 37110/44000 | Loss: 2.8675 | LR: 3.07e-05 | 1391.7ms/step | 14.4 samples/s +Step 37120/44000 | Loss: 2.8901 | LR: 3.06e-05 | 1389.8ms/step | 14.4 samples/s +Step 37130/44000 | Loss: 2.8828 | LR: 3.05e-05 | 1390.8ms/step | 14.4 samples/s +Step 37140/44000 | Loss: 2.8754 | LR: 3.04e-05 | 1398.3ms/step | 14.3 samples/s +Step 37150/44000 | Loss: 2.8377 | LR: 3.03e-05 | 1395.8ms/step | 14.3 samples/s +Step 37160/44000 | Loss: 2.8882 | LR: 3.02e-05 | 1398.1ms/step | 14.3 samples/s +Step 37170/44000 | Loss: 2.9205 | LR: 3.01e-05 | 1399.7ms/step | 14.3 samples/s +Step 37180/44000 | Loss: 2.8722 | LR: 3.01e-05 | 1394.1ms/step | 14.3 samples/s +Step 37190/44000 | Loss: 2.8757 | LR: 3.00e-05 | 1391.7ms/step | 14.4 samples/s +Step 37200/44000 | Loss: 2.8807 | LR: 2.99e-05 | 1391.6ms/step | 14.4 samples/s +Step 37210/44000 | Loss: 2.8952 | LR: 2.98e-05 | 1391.8ms/step | 14.4 samples/s +Step 37220/44000 | Loss: 2.8762 | LR: 2.97e-05 | 1393.0ms/step | 14.4 samples/s +Step 37230/44000 | Loss: 2.9043 | LR: 2.96e-05 | 1391.0ms/step | 14.4 samples/s +Step 37240/44000 | Loss: 2.8811 | LR: 2.95e-05 | 1390.9ms/step | 14.4 samples/s +Step 37250/44000 | Loss: 2.8719 | LR: 2.95e-05 | 1395.8ms/step | 14.3 samples/s +Validation at step 37250: Loss = 2.8941, Perplexity = 18.07 +Step 37260/44000 | Loss: 2.8855 | LR: 2.94e-05 | 3598.3ms/step | 5.6 samples/s +Step 37270/44000 | Loss: 2.8973 | LR: 2.93e-05 | 1399.2ms/step | 14.3 samples/s +Step 37280/44000 | Loss: 2.8780 | LR: 2.92e-05 | 1399.0ms/step | 14.3 samples/s +Step 37290/44000 | Loss: 2.8875 | LR: 2.91e-05 | 1397.8ms/step | 14.3 samples/s +Step 37300/44000 | Loss: 2.8728 | LR: 2.90e-05 | 1398.2ms/step | 14.3 samples/s +Step 37310/44000 | Loss: 2.8751 | LR: 2.89e-05 | 1398.2ms/step | 14.3 samples/s +Step 37320/44000 | Loss: 2.8797 | LR: 2.89e-05 | 1397.7ms/step | 14.3 samples/s +Step 37330/44000 | Loss: 2.9042 | LR: 2.88e-05 | 1390.9ms/step | 14.4 samples/s +Step 37340/44000 | Loss: 2.8768 | LR: 2.87e-05 | 1389.3ms/step | 14.4 samples/s +Step 37350/44000 | Loss: 2.8966 | LR: 2.86e-05 | 1390.2ms/step | 14.4 samples/s +Step 37360/44000 | Loss: 2.8816 | LR: 2.85e-05 | 1389.6ms/step | 14.4 samples/s +Step 37370/44000 | Loss: 2.8847 | LR: 2.84e-05 | 1391.3ms/step | 14.4 samples/s +Step 37380/44000 | Loss: 2.8653 | LR: 2.84e-05 | 1390.9ms/step | 14.4 samples/s +Step 37390/44000 | Loss: 2.8646 | LR: 2.83e-05 | 1391.8ms/step | 14.4 samples/s +Step 37400/44000 | Loss: 2.8674 | LR: 2.82e-05 | 1390.0ms/step | 14.4 samples/s +Step 37410/44000 | Loss: 2.8634 | LR: 2.81e-05 | 1390.1ms/step | 14.4 samples/s +Step 37420/44000 | Loss: 2.8915 | LR: 2.80e-05 | 1390.2ms/step | 14.4 samples/s +Step 37430/44000 | Loss: 2.8539 | LR: 2.79e-05 | 1393.7ms/step | 14.4 samples/s +Step 37440/44000 | Loss: 2.8660 | LR: 2.78e-05 | 1391.3ms/step | 14.4 samples/s +Step 37450/44000 | Loss: 2.8807 | LR: 2.78e-05 | 1391.2ms/step | 14.4 samples/s +Step 37460/44000 | Loss: 2.8954 | LR: 2.77e-05 | 1390.7ms/step | 14.4 samples/s +Step 37470/44000 | Loss: 2.8780 | LR: 2.76e-05 | 1391.1ms/step | 14.4 samples/s +Step 37480/44000 | Loss: 2.8897 | LR: 2.75e-05 | 1391.9ms/step | 14.4 samples/s +Step 37490/44000 | Loss: 2.8737 | LR: 2.74e-05 | 1391.9ms/step | 14.4 samples/s +Step 37500/44000 | Loss: 2.8790 | LR: 2.74e-05 | 1392.2ms/step | 14.4 samples/s +Validation at step 37500: Loss = 2.8940, Perplexity = 18.07 +Step 37510/44000 | Loss: 2.8988 | LR: 2.73e-05 | 3593.7ms/step | 5.6 samples/s +Step 37520/44000 | Loss: 2.8529 | LR: 2.72e-05 | 1390.4ms/step | 14.4 samples/s +Step 37530/44000 | Loss: 2.9197 | LR: 2.71e-05 | 1398.0ms/step | 14.3 samples/s +Step 37540/44000 | Loss: 2.8877 | LR: 2.70e-05 | 1399.6ms/step | 14.3 samples/s +Step 37550/44000 | Loss: 2.8795 | LR: 2.69e-05 | 1398.3ms/step | 14.3 samples/s +Step 37560/44000 | Loss: 2.8978 | LR: 2.69e-05 | 1398.0ms/step | 14.3 samples/s +Step 37570/44000 | Loss: 2.8773 | LR: 2.68e-05 | 1392.0ms/step | 14.4 samples/s +Step 37580/44000 | Loss: 2.8697 | LR: 2.67e-05 | 1396.1ms/step | 14.3 samples/s +Step 37590/44000 | Loss: 2.8709 | LR: 2.66e-05 | 1393.9ms/step | 14.3 samples/s +Step 37600/44000 | Loss: 2.8761 | LR: 2.65e-05 | 1391.5ms/step | 14.4 samples/s +Step 37610/44000 | Loss: 2.8666 | LR: 2.65e-05 | 1388.6ms/step | 14.4 samples/s +Step 37620/44000 | Loss: 2.8914 | LR: 2.64e-05 | 1393.3ms/step | 14.4 samples/s +Step 37630/44000 | Loss: 2.8449 | LR: 2.63e-05 | 1390.5ms/step | 14.4 samples/s +Step 37640/44000 | Loss: 2.8939 | LR: 2.62e-05 | 1397.8ms/step | 14.3 samples/s +Step 37650/44000 | Loss: 2.8652 | LR: 2.61e-05 | 1391.3ms/step | 14.4 samples/s +Step 37660/44000 | Loss: 2.9039 | LR: 2.60e-05 | 1395.0ms/step | 14.3 samples/s +Step 37670/44000 | Loss: 2.8837 | LR: 2.60e-05 | 1396.0ms/step | 14.3 samples/s +Step 37680/44000 | Loss: 2.8958 | LR: 2.59e-05 | 1394.1ms/step | 14.3 samples/s +Step 37690/44000 | Loss: 2.8725 | LR: 2.58e-05 | 1395.6ms/step | 14.3 samples/s +Step 37700/44000 | Loss: 2.8681 | LR: 2.57e-05 | 1394.6ms/step | 14.3 samples/s +Step 37710/44000 | Loss: 2.8604 | LR: 2.56e-05 | 1399.8ms/step | 14.3 samples/s +Step 37720/44000 | Loss: 2.8948 | LR: 2.56e-05 | 1398.2ms/step | 14.3 samples/s +Step 37730/44000 | Loss: 2.8676 | LR: 2.55e-05 | 1399.9ms/step | 14.3 samples/s +Step 37740/44000 | Loss: 2.8778 | LR: 2.54e-05 | 1393.0ms/step | 14.4 samples/s +Step 37750/44000 | Loss: 2.8615 | LR: 2.53e-05 | 1393.4ms/step | 14.4 samples/s +Validation at step 37750: Loss = 2.8940, Perplexity = 18.06 +Step 37760/44000 | Loss: 2.8836 | LR: 2.52e-05 | 3597.2ms/step | 5.6 samples/s +Step 37770/44000 | Loss: 2.8589 | LR: 2.52e-05 | 1396.1ms/step | 14.3 samples/s +Step 37780/44000 | Loss: 2.8643 | LR: 2.51e-05 | 1396.4ms/step | 14.3 samples/s +Step 37790/44000 | Loss: 2.8996 | LR: 2.50e-05 | 1397.6ms/step | 14.3 samples/s +Step 37800/44000 | Loss: 2.8699 | LR: 2.49e-05 | 1398.2ms/step | 14.3 samples/s +Step 37810/44000 | Loss: 2.8443 | LR: 2.48e-05 | 1396.8ms/step | 14.3 samples/s +Step 37820/44000 | Loss: 2.9005 | LR: 2.48e-05 | 1391.5ms/step | 14.4 samples/s +Step 37830/44000 | Loss: 2.9011 | LR: 2.47e-05 | 1393.6ms/step | 14.4 samples/s +Step 37840/44000 | Loss: 2.8860 | LR: 2.46e-05 | 1392.2ms/step | 14.4 samples/s +Step 37850/44000 | Loss: 2.8651 | LR: 2.45e-05 | 1392.5ms/step | 14.4 samples/s +Step 37860/44000 | Loss: 2.8459 | LR: 2.45e-05 | 1391.5ms/step | 14.4 samples/s +Step 37870/44000 | Loss: 2.8694 | LR: 2.44e-05 | 1396.1ms/step | 14.3 samples/s +Step 37880/44000 | Loss: 2.8665 | LR: 2.43e-05 | 1397.8ms/step | 14.3 samples/s +Step 37890/44000 | Loss: 2.8912 | LR: 2.42e-05 | 1394.3ms/step | 14.3 samples/s +Step 37900/44000 | Loss: 2.8931 | LR: 2.41e-05 | 1394.0ms/step | 14.3 samples/s +Step 37910/44000 | Loss: 2.8781 | LR: 2.41e-05 | 1396.0ms/step | 14.3 samples/s +Step 37920/44000 | Loss: 2.8987 | LR: 2.40e-05 | 1398.2ms/step | 14.3 samples/s +Step 37930/44000 | Loss: 2.8746 | LR: 2.39e-05 | 1397.7ms/step | 14.3 samples/s +Step 37940/44000 | Loss: 2.8860 | LR: 2.38e-05 | 1398.0ms/step | 14.3 samples/s +Step 37950/44000 | Loss: 2.9020 | LR: 2.38e-05 | 1400.3ms/step | 14.3 samples/s +Step 37960/44000 | Loss: 2.9190 | LR: 2.37e-05 | 1398.0ms/step | 14.3 samples/s +Step 37970/44000 | Loss: 2.8759 | LR: 2.36e-05 | 1395.9ms/step | 14.3 samples/s +Step 37980/44000 | Loss: 2.9072 | LR: 2.35e-05 | 1393.9ms/step | 14.3 samples/s +Step 37990/44000 | Loss: 2.8758 | LR: 2.34e-05 | 1400.1ms/step | 14.3 samples/s +Step 38000/44000 | Loss: 2.8794 | LR: 2.34e-05 | 1400.0ms/step | 14.3 samples/s +Validation at step 38000: Loss = 2.8939, Perplexity = 18.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-38000 +Step 38010/44000 | Loss: 2.8315 | LR: 2.33e-05 | 3917.9ms/step | 5.1 samples/s +Step 38020/44000 | Loss: 2.8650 | LR: 2.32e-05 | 1400.2ms/step | 14.3 samples/s +Step 38030/44000 | Loss: 2.8317 | LR: 2.31e-05 | 1400.0ms/step | 14.3 samples/s +Step 38040/44000 | Loss: 2.9031 | LR: 2.31e-05 | 1400.0ms/step | 14.3 samples/s +Step 38050/44000 | Loss: 2.8871 | LR: 2.30e-05 | 1400.0ms/step | 14.3 samples/s +Step 38060/44000 | Loss: 2.8908 | LR: 2.29e-05 | 1400.2ms/step | 14.3 samples/s +Step 38070/44000 | Loss: 2.8640 | LR: 2.28e-05 | 1399.8ms/step | 14.3 samples/s +Step 38080/44000 | Loss: 2.8793 | LR: 2.28e-05 | 1400.0ms/step | 14.3 samples/s +Step 38090/44000 | Loss: 2.8903 | LR: 2.27e-05 | 1397.8ms/step | 14.3 samples/s +Step 38100/44000 | Loss: 2.8746 | LR: 2.26e-05 | 1390.2ms/step | 14.4 samples/s +Step 38110/44000 | Loss: 2.8834 | LR: 2.25e-05 | 1391.3ms/step | 14.4 samples/s +Step 38120/44000 | Loss: 2.8950 | LR: 2.25e-05 | 1390.2ms/step | 14.4 samples/s +Step 38130/44000 | Loss: 2.8706 | LR: 2.24e-05 | 1388.7ms/step | 14.4 samples/s +Step 38140/44000 | Loss: 2.8970 | LR: 2.23e-05 | 1391.1ms/step | 14.4 samples/s +Step 38150/44000 | Loss: 2.9153 | LR: 2.22e-05 | 1387.5ms/step | 14.4 samples/s +Step 38160/44000 | Loss: 2.9194 | LR: 2.22e-05 | 1392.1ms/step | 14.4 samples/s +Step 38170/44000 | Loss: 2.8834 | LR: 2.21e-05 | 1393.3ms/step | 14.4 samples/s +Step 38180/44000 | Loss: 2.9124 | LR: 2.20e-05 | 1391.2ms/step | 14.4 samples/s +Step 38190/44000 | Loss: 2.8530 | LR: 2.19e-05 | 1392.7ms/step | 14.4 samples/s +Step 38200/44000 | Loss: 2.8830 | LR: 2.19e-05 | 1393.7ms/step | 14.4 samples/s +Step 38210/44000 | Loss: 2.8824 | LR: 2.18e-05 | 1394.0ms/step | 14.3 samples/s +Step 38220/44000 | Loss: 2.9009 | LR: 2.17e-05 | 1390.2ms/step | 14.4 samples/s +Step 38230/44000 | Loss: 2.8857 | LR: 2.16e-05 | 1388.1ms/step | 14.4 samples/s +Step 38240/44000 | Loss: 2.8884 | LR: 2.16e-05 | 1390.8ms/step | 14.4 samples/s +Step 38250/44000 | Loss: 2.8875 | LR: 2.15e-05 | 1393.1ms/step | 14.4 samples/s +Validation at step 38250: Loss = 2.8938, Perplexity = 18.06 +Step 38260/44000 | Loss: 2.8603 | LR: 2.14e-05 | 3593.5ms/step | 5.6 samples/s +Step 38270/44000 | Loss: 2.8916 | LR: 2.13e-05 | 1392.9ms/step | 14.4 samples/s +Step 38280/44000 | Loss: 2.8510 | LR: 2.13e-05 | 1394.0ms/step | 14.3 samples/s +Step 38290/44000 | Loss: 2.8576 | LR: 2.12e-05 | 1395.5ms/step | 14.3 samples/s +Step 38300/44000 | Loss: 2.8912 | LR: 2.11e-05 | 1398.1ms/step | 14.3 samples/s +Step 38310/44000 | Loss: 2.8634 | LR: 2.11e-05 | 1394.3ms/step | 14.3 samples/s +Step 38320/44000 | Loss: 2.8803 | LR: 2.10e-05 | 1394.0ms/step | 14.3 samples/s +Step 38330/44000 | Loss: 2.8977 | LR: 2.09e-05 | 1394.2ms/step | 14.3 samples/s +Step 38340/44000 | Loss: 2.8481 | LR: 2.08e-05 | 1394.7ms/step | 14.3 samples/s +Step 38350/44000 | Loss: 2.8763 | LR: 2.08e-05 | 1390.9ms/step | 14.4 samples/s +Step 38360/44000 | Loss: 2.8840 | LR: 2.07e-05 | 1398.2ms/step | 14.3 samples/s +Step 38370/44000 | Loss: 2.8921 | LR: 2.06e-05 | 1400.0ms/step | 14.3 samples/s +Step 38380/44000 | Loss: 2.8954 | LR: 2.05e-05 | 1400.0ms/step | 14.3 samples/s +Step 38390/44000 | Loss: 2.8750 | LR: 2.05e-05 | 1398.0ms/step | 14.3 samples/s +Step 38400/44000 | Loss: 2.8953 | LR: 2.04e-05 | 1396.1ms/step | 14.3 samples/s +Step 38410/44000 | Loss: 2.8764 | LR: 2.03e-05 | 1395.3ms/step | 14.3 samples/s +Step 38420/44000 | Loss: 2.8877 | LR: 2.03e-05 | 1396.6ms/step | 14.3 samples/s +Step 38430/44000 | Loss: 2.8915 | LR: 2.02e-05 | 1399.8ms/step | 14.3 samples/s +Step 38440/44000 | Loss: 2.8449 | LR: 2.01e-05 | 1398.4ms/step | 14.3 samples/s +Step 38450/44000 | Loss: 2.8826 | LR: 2.00e-05 | 1400.0ms/step | 14.3 samples/s +Step 38460/44000 | Loss: 2.8811 | LR: 2.00e-05 | 1400.1ms/step | 14.3 samples/s +Step 38470/44000 | Loss: 2.8758 | LR: 1.99e-05 | 1399.8ms/step | 14.3 samples/s +Step 38480/44000 | Loss: 2.9003 | LR: 1.98e-05 | 1396.0ms/step | 14.3 samples/s +Step 38490/44000 | Loss: 2.8870 | LR: 1.98e-05 | 1395.4ms/step | 14.3 samples/s +Step 38500/44000 | Loss: 2.8892 | LR: 1.97e-05 | 1393.6ms/step | 14.4 samples/s +Validation at step 38500: Loss = 2.8937, Perplexity = 18.06 +Step 38510/44000 | Loss: 2.8811 | LR: 1.96e-05 | 3591.1ms/step | 5.6 samples/s +Step 38520/44000 | Loss: 2.8560 | LR: 1.95e-05 | 1392.8ms/step | 14.4 samples/s +Step 38530/44000 | Loss: 2.8800 | LR: 1.95e-05 | 1396.6ms/step | 14.3 samples/s +Step 38540/44000 | Loss: 2.8903 | LR: 1.94e-05 | 1396.4ms/step | 14.3 samples/s +Step 38550/44000 | Loss: 2.9001 | LR: 1.93e-05 | 1396.3ms/step | 14.3 samples/s +Step 38560/44000 | Loss: 2.8836 | LR: 1.93e-05 | 1395.9ms/step | 14.3 samples/s +Step 38570/44000 | Loss: 2.8749 | LR: 1.92e-05 | 1390.7ms/step | 14.4 samples/s +Step 38580/44000 | Loss: 2.8574 | LR: 1.91e-05 | 1396.3ms/step | 14.3 samples/s +Step 38590/44000 | Loss: 2.8871 | LR: 1.91e-05 | 1393.0ms/step | 14.4 samples/s +Step 38600/44000 | Loss: 2.8636 | LR: 1.90e-05 | 1393.3ms/step | 14.4 samples/s +Step 38610/44000 | Loss: 2.8680 | LR: 1.89e-05 | 1388.6ms/step | 14.4 samples/s +Step 38620/44000 | Loss: 2.8892 | LR: 1.88e-05 | 1391.8ms/step | 14.4 samples/s +Step 38630/44000 | Loss: 2.8631 | LR: 1.88e-05 | 1388.2ms/step | 14.4 samples/s +Step 38640/44000 | Loss: 2.9030 | LR: 1.87e-05 | 1388.4ms/step | 14.4 samples/s +Step 38650/44000 | Loss: 2.8587 | LR: 1.86e-05 | 1388.2ms/step | 14.4 samples/s +Step 38660/44000 | Loss: 2.8692 | LR: 1.86e-05 | 1388.3ms/step | 14.4 samples/s +Step 38670/44000 | Loss: 2.8834 | LR: 1.85e-05 | 1388.9ms/step | 14.4 samples/s +Step 38680/44000 | Loss: 2.9075 | LR: 1.84e-05 | 1390.2ms/step | 14.4 samples/s +Step 38690/44000 | Loss: 2.8821 | LR: 1.84e-05 | 1392.5ms/step | 14.4 samples/s +Step 38700/44000 | Loss: 2.9004 | LR: 1.83e-05 | 1389.6ms/step | 14.4 samples/s +Step 38710/44000 | Loss: 2.8849 | LR: 1.82e-05 | 1390.1ms/step | 14.4 samples/s +Step 38720/44000 | Loss: 2.8561 | LR: 1.82e-05 | 1394.2ms/step | 14.3 samples/s +Step 38730/44000 | Loss: 2.8739 | LR: 1.81e-05 | 1392.7ms/step | 14.4 samples/s +Step 38740/44000 | Loss: 2.8957 | LR: 1.80e-05 | 1394.3ms/step | 14.3 samples/s +Step 38750/44000 | Loss: 2.8764 | LR: 1.80e-05 | 1390.6ms/step | 14.4 samples/s +Validation at step 38750: Loss = 2.8937, Perplexity = 18.06 +Step 38760/44000 | Loss: 2.8859 | LR: 1.79e-05 | 3602.3ms/step | 5.6 samples/s +Step 38770/44000 | Loss: 2.8748 | LR: 1.78e-05 | 1398.3ms/step | 14.3 samples/s +Step 38780/44000 | Loss: 2.8672 | LR: 1.78e-05 | 1399.7ms/step | 14.3 samples/s +Step 38790/44000 | Loss: 2.8990 | LR: 1.77e-05 | 1397.9ms/step | 14.3 samples/s +Step 38800/44000 | Loss: 2.8663 | LR: 1.76e-05 | 1398.0ms/step | 14.3 samples/s +Step 38810/44000 | Loss: 2.8493 | LR: 1.76e-05 | 1396.2ms/step | 14.3 samples/s +Step 38820/44000 | Loss: 2.8853 | LR: 1.75e-05 | 1394.6ms/step | 14.3 samples/s +Step 38830/44000 | Loss: 2.8864 | LR: 1.74e-05 | 1393.5ms/step | 14.4 samples/s +Step 38840/44000 | Loss: 2.8499 | LR: 1.74e-05 | 1397.9ms/step | 14.3 samples/s +Step 38850/44000 | Loss: 2.8881 | LR: 1.73e-05 | 1400.0ms/step | 14.3 samples/s +Step 38860/44000 | Loss: 2.8759 | LR: 1.72e-05 | 1398.1ms/step | 14.3 samples/s +Step 38870/44000 | Loss: 2.8917 | LR: 1.72e-05 | 1400.0ms/step | 14.3 samples/s +Step 38880/44000 | Loss: 2.8938 | LR: 1.71e-05 | 1393.9ms/step | 14.3 samples/s +Step 38890/44000 | Loss: 2.8807 | LR: 1.70e-05 | 1392.3ms/step | 14.4 samples/s +Step 38900/44000 | Loss: 2.8763 | LR: 1.70e-05 | 1397.4ms/step | 14.3 samples/s +Step 38910/44000 | Loss: 2.8989 | LR: 1.69e-05 | 1397.0ms/step | 14.3 samples/s +Step 38920/44000 | Loss: 2.8816 | LR: 1.68e-05 | 1393.2ms/step | 14.4 samples/s +Step 38930/44000 | Loss: 2.8574 | LR: 1.68e-05 | 1394.2ms/step | 14.3 samples/s +Step 38940/44000 | Loss: 2.8658 | LR: 1.67e-05 | 1392.8ms/step | 14.4 samples/s +Step 38950/44000 | Loss: 2.8537 | LR: 1.66e-05 | 1394.6ms/step | 14.3 samples/s +Step 38960/44000 | Loss: 2.8721 | LR: 1.66e-05 | 1394.7ms/step | 14.3 samples/s +Step 38970/44000 | Loss: 2.8852 | LR: 1.65e-05 | 1399.9ms/step | 14.3 samples/s +Step 38980/44000 | Loss: 2.8610 | LR: 1.64e-05 | 1399.9ms/step | 14.3 samples/s +Step 38990/44000 | Loss: 2.8908 | LR: 1.64e-05 | 1398.1ms/step | 14.3 samples/s +Step 39000/44000 | Loss: 2.8789 | LR: 1.63e-05 | 1398.1ms/step | 14.3 samples/s +Validation at step 39000: Loss = 2.8937, Perplexity = 18.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-39000 +Step 39010/44000 | Loss: 2.9163 | LR: 1.62e-05 | 3885.9ms/step | 5.1 samples/s +Step 39020/44000 | Loss: 2.8680 | LR: 1.62e-05 | 1391.9ms/step | 14.4 samples/s +Step 39030/44000 | Loss: 2.9071 | LR: 1.61e-05 | 1392.3ms/step | 14.4 samples/s +Step 39040/44000 | Loss: 2.8606 | LR: 1.61e-05 | 1396.0ms/step | 14.3 samples/s +Step 39050/44000 | Loss: 2.8683 | LR: 1.60e-05 | 1395.4ms/step | 14.3 samples/s +Step 39060/44000 | Loss: 2.8815 | LR: 1.59e-05 | 1392.5ms/step | 14.4 samples/s +Step 39070/44000 | Loss: 2.8569 | LR: 1.59e-05 | 1391.1ms/step | 14.4 samples/s +Step 39080/44000 | Loss: 2.8877 | LR: 1.58e-05 | 1391.0ms/step | 14.4 samples/s +Step 39090/44000 | Loss: 2.8497 | LR: 1.57e-05 | 1388.6ms/step | 14.4 samples/s +Step 39100/44000 | Loss: 2.8537 | LR: 1.57e-05 | 1391.1ms/step | 14.4 samples/s +Step 39110/44000 | Loss: 2.8914 | LR: 1.56e-05 | 1394.2ms/step | 14.3 samples/s +Step 39120/44000 | Loss: 2.8828 | LR: 1.55e-05 | 1393.8ms/step | 14.3 samples/s +Step 39130/44000 | Loss: 2.8532 | LR: 1.55e-05 | 1396.3ms/step | 14.3 samples/s +Step 39140/44000 | Loss: 2.8716 | LR: 1.54e-05 | 1393.8ms/step | 14.3 samples/s +Step 39150/44000 | Loss: 2.8843 | LR: 1.54e-05 | 1392.4ms/step | 14.4 samples/s +Step 39160/44000 | Loss: 2.9114 | LR: 1.53e-05 | 1391.2ms/step | 14.4 samples/s +Step 39170/44000 | Loss: 2.8882 | LR: 1.52e-05 | 1386.9ms/step | 14.4 samples/s +Step 39180/44000 | Loss: 2.8918 | LR: 1.52e-05 | 1392.8ms/step | 14.4 samples/s +Step 39190/44000 | Loss: 2.8347 | LR: 1.51e-05 | 1388.9ms/step | 14.4 samples/s +Step 39200/44000 | Loss: 2.8585 | LR: 1.50e-05 | 1390.9ms/step | 14.4 samples/s +Step 39210/44000 | Loss: 2.8460 | LR: 1.50e-05 | 1391.5ms/step | 14.4 samples/s +Step 39220/44000 | Loss: 2.8850 | LR: 1.49e-05 | 1390.8ms/step | 14.4 samples/s +Step 39230/44000 | Loss: 2.8700 | LR: 1.49e-05 | 1392.8ms/step | 14.4 samples/s +Step 39240/44000 | Loss: 2.8715 | LR: 1.48e-05 | 1398.1ms/step | 14.3 samples/s +Step 39250/44000 | Loss: 2.8689 | LR: 1.47e-05 | 1399.5ms/step | 14.3 samples/s +Validation at step 39250: Loss = 2.8936, Perplexity = 18.06 +Step 39260/44000 | Loss: 2.8677 | LR: 1.47e-05 | 3622.5ms/step | 5.5 samples/s +Step 39270/44000 | Loss: 2.8625 | LR: 1.46e-05 | 1400.1ms/step | 14.3 samples/s +Step 39280/44000 | Loss: 2.8751 | LR: 1.46e-05 | 1399.6ms/step | 14.3 samples/s +Step 39290/44000 | Loss: 2.8729 | LR: 1.45e-05 | 1400.3ms/step | 14.3 samples/s +Step 39300/44000 | Loss: 2.8718 | LR: 1.44e-05 | 1400.1ms/step | 14.3 samples/s +Step 39310/44000 | Loss: 2.8640 | LR: 1.44e-05 | 1399.8ms/step | 14.3 samples/s +Step 39320/44000 | Loss: 2.8422 | LR: 1.43e-05 | 1398.3ms/step | 14.3 samples/s +Step 39330/44000 | Loss: 2.8856 | LR: 1.42e-05 | 1399.8ms/step | 14.3 samples/s +Step 39340/44000 | Loss: 2.8813 | LR: 1.42e-05 | 1395.9ms/step | 14.3 samples/s +Step 39350/44000 | Loss: 2.8733 | LR: 1.41e-05 | 1397.9ms/step | 14.3 samples/s +Step 39360/44000 | Loss: 2.9143 | LR: 1.41e-05 | 1396.3ms/step | 14.3 samples/s +Step 39370/44000 | Loss: 2.8578 | LR: 1.40e-05 | 1399.8ms/step | 14.3 samples/s +Step 39380/44000 | Loss: 2.8955 | LR: 1.39e-05 | 1393.5ms/step | 14.4 samples/s +Step 39390/44000 | Loss: 2.8805 | LR: 1.39e-05 | 1388.6ms/step | 14.4 samples/s +Step 39400/44000 | Loss: 2.8679 | LR: 1.38e-05 | 1394.0ms/step | 14.3 samples/s +Step 39410/44000 | Loss: 2.8785 | LR: 1.38e-05 | 1395.4ms/step | 14.3 samples/s +Step 39420/44000 | Loss: 2.9028 | LR: 1.37e-05 | 1398.6ms/step | 14.3 samples/s +Step 39430/44000 | Loss: 2.8976 | LR: 1.36e-05 | 1398.2ms/step | 14.3 samples/s +Step 39440/44000 | Loss: 2.8650 | LR: 1.36e-05 | 1400.0ms/step | 14.3 samples/s +Step 39450/44000 | Loss: 2.8993 | LR: 1.35e-05 | 1392.7ms/step | 14.4 samples/s +Step 39460/44000 | Loss: 2.8617 | LR: 1.35e-05 | 1390.6ms/step | 14.4 samples/s +Step 39470/44000 | Loss: 2.8667 | LR: 1.34e-05 | 1388.5ms/step | 14.4 samples/s +Step 39480/44000 | Loss: 2.8881 | LR: 1.34e-05 | 1390.2ms/step | 14.4 samples/s +Step 39490/44000 | Loss: 2.8900 | LR: 1.33e-05 | 1391.2ms/step | 14.4 samples/s +Step 39500/44000 | Loss: 2.8729 | LR: 1.32e-05 | 1393.2ms/step | 14.4 samples/s +Validation at step 39500: Loss = 2.8936, Perplexity = 18.06 +Step 39510/44000 | Loss: 2.8559 | LR: 1.32e-05 | 3594.7ms/step | 5.6 samples/s +Step 39520/44000 | Loss: 2.8575 | LR: 1.31e-05 | 1385.8ms/step | 14.4 samples/s +Step 39530/44000 | Loss: 2.8768 | LR: 1.31e-05 | 1390.6ms/step | 14.4 samples/s +Step 39540/44000 | Loss: 2.8811 | LR: 1.30e-05 | 1392.7ms/step | 14.4 samples/s +Step 39550/44000 | Loss: 2.8745 | LR: 1.29e-05 | 1398.0ms/step | 14.3 samples/s +Step 39560/44000 | Loss: 2.8863 | LR: 1.29e-05 | 1397.4ms/step | 14.3 samples/s +Step 39570/44000 | Loss: 2.8322 | LR: 1.28e-05 | 1390.6ms/step | 14.4 samples/s +Step 39580/44000 | Loss: 2.8747 | LR: 1.28e-05 | 1390.4ms/step | 14.4 samples/s +Step 39590/44000 | Loss: 2.8615 | LR: 1.27e-05 | 1393.4ms/step | 14.4 samples/s +Step 39600/44000 | Loss: 2.8703 | LR: 1.27e-05 | 1396.0ms/step | 14.3 samples/s +Step 39610/44000 | Loss: 2.8740 | LR: 1.26e-05 | 1397.9ms/step | 14.3 samples/s +Step 39620/44000 | Loss: 2.8816 | LR: 1.25e-05 | 1391.0ms/step | 14.4 samples/s +Step 39630/44000 | Loss: 2.8808 | LR: 1.25e-05 | 1385.4ms/step | 14.4 samples/s +Step 39640/44000 | Loss: 2.8842 | LR: 1.24e-05 | 1389.5ms/step | 14.4 samples/s +Step 39650/44000 | Loss: 2.8831 | LR: 1.24e-05 | 1394.2ms/step | 14.3 samples/s +Step 39660/44000 | Loss: 2.8695 | LR: 1.23e-05 | 1398.3ms/step | 14.3 samples/s +Step 39670/44000 | Loss: 2.8606 | LR: 1.23e-05 | 1399.9ms/step | 14.3 samples/s +Step 39680/44000 | Loss: 2.8919 | LR: 1.22e-05 | 1394.3ms/step | 14.3 samples/s +Step 39690/44000 | Loss: 2.8923 | LR: 1.22e-05 | 1392.1ms/step | 14.4 samples/s +Step 39700/44000 | Loss: 2.8822 | LR: 1.21e-05 | 1393.2ms/step | 14.4 samples/s +Step 39710/44000 | Loss: 2.9202 | LR: 1.20e-05 | 1392.7ms/step | 14.4 samples/s +Step 39720/44000 | Loss: 2.8895 | LR: 1.20e-05 | 1397.7ms/step | 14.3 samples/s +Step 39730/44000 | Loss: 2.8843 | LR: 1.19e-05 | 1395.7ms/step | 14.3 samples/s +Step 39740/44000 | Loss: 2.8898 | LR: 1.19e-05 | 1388.8ms/step | 14.4 samples/s +Step 39750/44000 | Loss: 2.9054 | LR: 1.18e-05 | 1393.5ms/step | 14.4 samples/s +Validation at step 39750: Loss = 2.8936, Perplexity = 18.06 +Step 39760/44000 | Loss: 2.9086 | LR: 1.18e-05 | 3606.2ms/step | 5.5 samples/s +Step 39770/44000 | Loss: 2.8734 | LR: 1.17e-05 | 1398.1ms/step | 14.3 samples/s +Step 39780/44000 | Loss: 2.8757 | LR: 1.17e-05 | 1391.4ms/step | 14.4 samples/s +Step 39790/44000 | Loss: 2.8610 | LR: 1.16e-05 | 1391.1ms/step | 14.4 samples/s +Step 39800/44000 | Loss: 2.8780 | LR: 1.15e-05 | 1389.3ms/step | 14.4 samples/s +Step 39810/44000 | Loss: 2.8620 | LR: 1.15e-05 | 1392.1ms/step | 14.4 samples/s +Step 39820/44000 | Loss: 2.8734 | LR: 1.14e-05 | 1398.2ms/step | 14.3 samples/s +Step 39830/44000 | Loss: 2.8838 | LR: 1.14e-05 | 1396.0ms/step | 14.3 samples/s +Step 39840/44000 | Loss: 2.8999 | LR: 1.13e-05 | 1397.9ms/step | 14.3 samples/s +Step 39850/44000 | Loss: 2.8897 | LR: 1.13e-05 | 1396.2ms/step | 14.3 samples/s +Step 39860/44000 | Loss: 2.8810 | LR: 1.12e-05 | 1397.6ms/step | 14.3 samples/s +Step 39870/44000 | Loss: 2.8743 | LR: 1.12e-05 | 1389.7ms/step | 14.4 samples/s +Step 39880/44000 | Loss: 2.8521 | LR: 1.11e-05 | 1388.2ms/step | 14.4 samples/s +Step 39890/44000 | Loss: 2.8671 | LR: 1.11e-05 | 1387.7ms/step | 14.4 samples/s +Step 39900/44000 | Loss: 2.8808 | LR: 1.10e-05 | 1386.6ms/step | 14.4 samples/s +Step 39910/44000 | Loss: 2.8498 | LR: 1.10e-05 | 1391.9ms/step | 14.4 samples/s +Step 39920/44000 | Loss: 2.9077 | LR: 1.09e-05 | 1396.0ms/step | 14.3 samples/s +Step 39930/44000 | Loss: 2.8792 | LR: 1.08e-05 | 1397.9ms/step | 14.3 samples/s +Step 39940/44000 | Loss: 2.9011 | LR: 1.08e-05 | 1391.9ms/step | 14.4 samples/s +Step 39950/44000 | Loss: 2.8630 | LR: 1.07e-05 | 1386.2ms/step | 14.4 samples/s +Step 39960/44000 | Loss: 2.8825 | LR: 1.07e-05 | 1386.8ms/step | 14.4 samples/s +Step 39970/44000 | Loss: 2.8768 | LR: 1.06e-05 | 1386.3ms/step | 14.4 samples/s +Step 39980/44000 | Loss: 2.8684 | LR: 1.06e-05 | 1389.0ms/step | 14.4 samples/s +Step 39990/44000 | Loss: 2.8906 | LR: 1.05e-05 | 1391.5ms/step | 14.4 samples/s +Step 40000/44000 | Loss: 2.8624 | LR: 1.05e-05 | 1390.5ms/step | 14.4 samples/s +Validation at step 40000: Loss = 2.8936, Perplexity = 18.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-40000 +Step 40010/44000 | Loss: 2.8809 | LR: 1.04e-05 | 3891.4ms/step | 5.1 samples/s +Step 40020/44000 | Loss: 2.8817 | LR: 1.04e-05 | 1390.7ms/step | 14.4 samples/s +Step 40030/44000 | Loss: 2.8904 | LR: 1.03e-05 | 1395.7ms/step | 14.3 samples/s +Step 40040/44000 | Loss: 2.8721 | LR: 1.03e-05 | 1393.8ms/step | 14.3 samples/s +Step 40050/44000 | Loss: 2.8551 | LR: 1.02e-05 | 1391.7ms/step | 14.4 samples/s +Step 40060/44000 | Loss: 2.8639 | LR: 1.02e-05 | 1392.9ms/step | 14.4 samples/s +Step 40070/44000 | Loss: 2.8869 | LR: 1.01e-05 | 1391.7ms/step | 14.4 samples/s +Step 40080/44000 | Loss: 2.8875 | LR: 1.01e-05 | 1390.9ms/step | 14.4 samples/s +Step 40090/44000 | Loss: 2.8684 | LR: 1.00e-05 | 1391.8ms/step | 14.4 samples/s +Step 40100/44000 | Loss: 2.8770 | LR: 9.96e-06 | 1391.3ms/step | 14.4 samples/s +Step 40110/44000 | Loss: 2.8769 | LR: 9.91e-06 | 1391.9ms/step | 14.4 samples/s +Step 40120/44000 | Loss: 2.8903 | LR: 9.86e-06 | 1390.8ms/step | 14.4 samples/s +Step 40130/44000 | Loss: 2.8865 | LR: 9.81e-06 | 1395.5ms/step | 14.3 samples/s +Step 40140/44000 | Loss: 2.8913 | LR: 9.76e-06 | 1393.7ms/step | 14.4 samples/s +Step 40150/44000 | Loss: 2.8503 | LR: 9.71e-06 | 1394.0ms/step | 14.3 samples/s +Step 40160/44000 | Loss: 2.8571 | LR: 9.66e-06 | 1394.3ms/step | 14.3 samples/s +Step 40170/44000 | Loss: 2.8627 | LR: 9.61e-06 | 1395.6ms/step | 14.3 samples/s +Step 40180/44000 | Loss: 2.9018 | LR: 9.56e-06 | 1389.8ms/step | 14.4 samples/s +Step 40190/44000 | Loss: 2.8784 | LR: 9.51e-06 | 1388.2ms/step | 14.4 samples/s +Step 40200/44000 | Loss: 2.9100 | LR: 9.46e-06 | 1392.5ms/step | 14.4 samples/s +Step 40210/44000 | Loss: 2.8758 | LR: 9.41e-06 | 1391.2ms/step | 14.4 samples/s +Step 40220/44000 | Loss: 2.8950 | LR: 9.36e-06 | 1396.7ms/step | 14.3 samples/s +Step 40230/44000 | Loss: 2.8693 | LR: 9.32e-06 | 1394.9ms/step | 14.3 samples/s +Step 40240/44000 | Loss: 2.8776 | LR: 9.27e-06 | 1395.0ms/step | 14.3 samples/s +Step 40250/44000 | Loss: 2.8409 | LR: 9.22e-06 | 1400.2ms/step | 14.3 samples/s +Validation at step 40250: Loss = 2.8936, Perplexity = 18.06 +Step 40260/44000 | Loss: 2.8695 | LR: 9.17e-06 | 3597.9ms/step | 5.6 samples/s +Step 40270/44000 | Loss: 2.8865 | LR: 9.12e-06 | 1395.9ms/step | 14.3 samples/s +Step 40280/44000 | Loss: 2.8816 | LR: 9.07e-06 | 1400.0ms/step | 14.3 samples/s +Step 40290/44000 | Loss: 2.8559 | LR: 9.02e-06 | 1398.1ms/step | 14.3 samples/s +Step 40300/44000 | Loss: 2.8856 | LR: 8.97e-06 | 1394.0ms/step | 14.3 samples/s +Step 40310/44000 | Loss: 2.8727 | LR: 8.93e-06 | 1395.6ms/step | 14.3 samples/s +Step 40320/44000 | Loss: 2.9007 | LR: 8.88e-06 | 1394.6ms/step | 14.3 samples/s +Step 40330/44000 | Loss: 2.8817 | LR: 8.83e-06 | 1395.3ms/step | 14.3 samples/s +Step 40340/44000 | Loss: 2.8781 | LR: 8.78e-06 | 1394.5ms/step | 14.3 samples/s +Step 40350/44000 | Loss: 2.8698 | LR: 8.74e-06 | 1400.0ms/step | 14.3 samples/s +Step 40360/44000 | Loss: 2.8841 | LR: 8.69e-06 | 1398.1ms/step | 14.3 samples/s +Step 40370/44000 | Loss: 2.8595 | LR: 8.64e-06 | 1392.9ms/step | 14.4 samples/s +Step 40380/44000 | Loss: 2.8811 | LR: 8.59e-06 | 1390.9ms/step | 14.4 samples/s +Step 40390/44000 | Loss: 2.8965 | LR: 8.55e-06 | 1391.5ms/step | 14.4 samples/s +Step 40400/44000 | Loss: 2.8636 | LR: 8.50e-06 | 1392.4ms/step | 14.4 samples/s +Step 40410/44000 | Loss: 2.8500 | LR: 8.45e-06 | 1390.9ms/step | 14.4 samples/s +Step 40420/44000 | Loss: 2.8735 | LR: 8.41e-06 | 1393.4ms/step | 14.4 samples/s +Step 40430/44000 | Loss: 2.8821 | LR: 8.36e-06 | 1395.8ms/step | 14.3 samples/s +Step 40440/44000 | Loss: 2.8608 | LR: 8.31e-06 | 1392.2ms/step | 14.4 samples/s +Step 40450/44000 | Loss: 2.8730 | LR: 8.27e-06 | 1392.0ms/step | 14.4 samples/s +Step 40460/44000 | Loss: 2.8392 | LR: 8.22e-06 | 1391.3ms/step | 14.4 samples/s +Step 40470/44000 | Loss: 2.8955 | LR: 8.17e-06 | 1388.6ms/step | 14.4 samples/s +Step 40480/44000 | Loss: 2.8839 | LR: 8.13e-06 | 1392.2ms/step | 14.4 samples/s +Step 40490/44000 | Loss: 2.8838 | LR: 8.08e-06 | 1395.9ms/step | 14.3 samples/s +Step 40500/44000 | Loss: 2.8782 | LR: 8.04e-06 | 1400.3ms/step | 14.3 samples/s +Validation at step 40500: Loss = 2.8936, Perplexity = 18.06 +Step 40510/44000 | Loss: 2.8665 | LR: 7.99e-06 | 3608.2ms/step | 5.5 samples/s +Step 40520/44000 | Loss: 2.9126 | LR: 7.94e-06 | 1399.6ms/step | 14.3 samples/s +Step 40530/44000 | Loss: 2.9019 | LR: 7.90e-06 | 1400.0ms/step | 14.3 samples/s +Step 40540/44000 | Loss: 2.8930 | LR: 7.85e-06 | 1400.2ms/step | 14.3 samples/s +Step 40550/44000 | Loss: 2.8499 | LR: 7.81e-06 | 1399.9ms/step | 14.3 samples/s +Step 40560/44000 | Loss: 2.8507 | LR: 7.76e-06 | 1398.2ms/step | 14.3 samples/s +Step 40570/44000 | Loss: 2.8859 | LR: 7.72e-06 | 1400.2ms/step | 14.3 samples/s +Step 40580/44000 | Loss: 2.8792 | LR: 7.67e-06 | 1399.8ms/step | 14.3 samples/s +Step 40590/44000 | Loss: 2.8532 | LR: 7.63e-06 | 1400.0ms/step | 14.3 samples/s +Step 40600/44000 | Loss: 2.8809 | LR: 7.59e-06 | 1399.9ms/step | 14.3 samples/s +Step 40610/44000 | Loss: 2.9117 | LR: 7.54e-06 | 1400.0ms/step | 14.3 samples/s +Step 40620/44000 | Loss: 2.8778 | LR: 7.50e-06 | 1398.1ms/step | 14.3 samples/s +Step 40630/44000 | Loss: 2.8922 | LR: 7.45e-06 | 1396.0ms/step | 14.3 samples/s +Step 40640/44000 | Loss: 2.8987 | LR: 7.41e-06 | 1393.9ms/step | 14.3 samples/s +Step 40650/44000 | Loss: 2.8968 | LR: 7.37e-06 | 1395.8ms/step | 14.3 samples/s +Step 40660/44000 | Loss: 2.8868 | LR: 7.32e-06 | 1393.6ms/step | 14.4 samples/s +Step 40670/44000 | Loss: 2.8957 | LR: 7.28e-06 | 1394.9ms/step | 14.3 samples/s +Step 40680/44000 | Loss: 2.8789 | LR: 7.23e-06 | 1391.9ms/step | 14.4 samples/s +Step 40690/44000 | Loss: 2.8918 | LR: 7.19e-06 | 1393.4ms/step | 14.4 samples/s +Step 40700/44000 | Loss: 2.8687 | LR: 7.15e-06 | 1392.5ms/step | 14.4 samples/s +Step 40710/44000 | Loss: 2.8839 | LR: 7.10e-06 | 1393.8ms/step | 14.3 samples/s +Step 40720/44000 | Loss: 2.8927 | LR: 7.06e-06 | 1394.2ms/step | 14.3 samples/s +Step 40730/44000 | Loss: 2.9005 | LR: 7.02e-06 | 1393.7ms/step | 14.4 samples/s +Step 40740/44000 | Loss: 2.8711 | LR: 6.98e-06 | 1397.5ms/step | 14.3 samples/s +Step 40750/44000 | Loss: 2.8671 | LR: 6.93e-06 | 1405.0ms/step | 14.2 samples/s +Validation at step 40750: Loss = 2.8935, Perplexity = 18.06 +Step 40760/44000 | Loss: 2.8953 | LR: 6.89e-06 | 3613.9ms/step | 5.5 samples/s +Step 40770/44000 | Loss: 2.8746 | LR: 6.85e-06 | 1399.7ms/step | 14.3 samples/s +Step 40780/44000 | Loss: 2.8962 | LR: 6.81e-06 | 1397.9ms/step | 14.3 samples/s +Step 40790/44000 | Loss: 2.8977 | LR: 6.77e-06 | 1396.3ms/step | 14.3 samples/s +Step 40800/44000 | Loss: 2.9022 | LR: 6.72e-06 | 1397.9ms/step | 14.3 samples/s +Step 40810/44000 | Loss: 2.8824 | LR: 6.68e-06 | 1396.1ms/step | 14.3 samples/s +Step 40820/44000 | Loss: 2.8584 | LR: 6.64e-06 | 1399.8ms/step | 14.3 samples/s +Step 40830/44000 | Loss: 2.8557 | LR: 6.60e-06 | 1392.2ms/step | 14.4 samples/s +Step 40840/44000 | Loss: 2.8995 | LR: 6.56e-06 | 1391.8ms/step | 14.4 samples/s +Step 40850/44000 | Loss: 2.8879 | LR: 6.52e-06 | 1394.2ms/step | 14.3 samples/s +Step 40860/44000 | Loss: 2.8980 | LR: 6.47e-06 | 1394.1ms/step | 14.3 samples/s +Step 40870/44000 | Loss: 2.9022 | LR: 6.43e-06 | 1398.0ms/step | 14.3 samples/s +Step 40880/44000 | Loss: 2.8737 | LR: 6.39e-06 | 1393.9ms/step | 14.3 samples/s +Step 40890/44000 | Loss: 2.8904 | LR: 6.35e-06 | 1395.6ms/step | 14.3 samples/s +Step 40900/44000 | Loss: 2.8673 | LR: 6.31e-06 | 1397.5ms/step | 14.3 samples/s +Step 40910/44000 | Loss: 2.9367 | LR: 6.27e-06 | 1395.0ms/step | 14.3 samples/s +Step 40920/44000 | Loss: 2.8844 | LR: 6.23e-06 | 1399.2ms/step | 14.3 samples/s +Step 40930/44000 | Loss: 2.8933 | LR: 6.19e-06 | 1398.4ms/step | 14.3 samples/s +Step 40940/44000 | Loss: 2.8973 | LR: 6.15e-06 | 1396.0ms/step | 14.3 samples/s +Step 40950/44000 | Loss: 2.8559 | LR: 6.11e-06 | 1397.9ms/step | 14.3 samples/s +Step 40960/44000 | Loss: 2.8463 | LR: 6.07e-06 | 1394.3ms/step | 14.3 samples/s +Step 40970/44000 | Loss: 2.8741 | LR: 6.03e-06 | 1394.2ms/step | 14.3 samples/s +Step 40980/44000 | Loss: 2.8606 | LR: 5.99e-06 | 1398.0ms/step | 14.3 samples/s +Step 40990/44000 | Loss: 2.8840 | LR: 5.95e-06 | 1400.0ms/step | 14.3 samples/s +Step 41000/44000 | Loss: 2.8888 | LR: 5.91e-06 | 1396.0ms/step | 14.3 samples/s +Validation at step 41000: Loss = 2.8935, Perplexity = 18.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000 +Step 41010/44000 | Loss: 2.8952 | LR: 5.87e-06 | 3902.2ms/step | 5.1 samples/s +Step 41020/44000 | Loss: 2.9042 | LR: 5.83e-06 | 1390.7ms/step | 14.4 samples/s +Step 41030/44000 | Loss: 2.8776 | LR: 5.80e-06 | 1392.1ms/step | 14.4 samples/s +Step 41040/44000 | Loss: 2.8795 | LR: 5.76e-06 | 1393.6ms/step | 14.4 samples/s +Step 41050/44000 | Loss: 2.8750 | LR: 5.72e-06 | 1393.3ms/step | 14.4 samples/s +Step 41060/44000 | Loss: 2.8481 | LR: 5.68e-06 | 1394.1ms/step | 14.3 samples/s +Step 41070/44000 | Loss: 2.8740 | LR: 5.64e-06 | 1393.8ms/step | 14.3 samples/s +Step 41080/44000 | Loss: 2.9064 | LR: 5.60e-06 | 1395.9ms/step | 14.3 samples/s +Step 41090/44000 | Loss: 2.9230 | LR: 5.56e-06 | 1391.5ms/step | 14.4 samples/s +Step 41100/44000 | Loss: 2.9019 | LR: 5.53e-06 | 1392.9ms/step | 14.4 samples/s +Step 41110/44000 | Loss: 2.9227 | LR: 5.49e-06 | 1394.1ms/step | 14.3 samples/s +Step 41120/44000 | Loss: 2.9100 | LR: 5.45e-06 | 1393.9ms/step | 14.3 samples/s +Step 41130/44000 | Loss: 2.8804 | LR: 5.41e-06 | 1392.4ms/step | 14.4 samples/s +Step 41140/44000 | Loss: 2.8596 | LR: 5.38e-06 | 1394.0ms/step | 14.3 samples/s +Step 41150/44000 | Loss: 2.9006 | LR: 5.34e-06 | 1395.8ms/step | 14.3 samples/s +Step 41160/44000 | Loss: 2.8730 | LR: 5.30e-06 | 1395.7ms/step | 14.3 samples/s +Step 41170/44000 | Loss: 2.8855 | LR: 5.26e-06 | 1400.3ms/step | 14.3 samples/s +Step 41180/44000 | Loss: 2.8594 | LR: 5.23e-06 | 1399.8ms/step | 14.3 samples/s +Step 41190/44000 | Loss: 2.8437 | LR: 5.19e-06 | 1395.9ms/step | 14.3 samples/s +Step 41200/44000 | Loss: 2.8836 | LR: 5.15e-06 | 1392.1ms/step | 14.4 samples/s +Step 41210/44000 | Loss: 2.8921 | LR: 5.12e-06 | 1396.0ms/step | 14.3 samples/s +Step 41220/44000 | Loss: 2.9002 | LR: 5.08e-06 | 1394.1ms/step | 14.3 samples/s +Step 41230/44000 | Loss: 2.8744 | LR: 5.04e-06 | 1395.9ms/step | 14.3 samples/s +Step 41240/44000 | Loss: 2.8603 | LR: 5.01e-06 | 1394.1ms/step | 14.3 samples/s +Step 41250/44000 | Loss: 2.8783 | LR: 4.97e-06 | 1396.1ms/step | 14.3 samples/s +Validation at step 41250: Loss = 2.8935, Perplexity = 18.06 +Step 41260/44000 | Loss: 2.9057 | LR: 4.94e-06 | 3582.0ms/step | 5.6 samples/s +Step 41270/44000 | Loss: 2.8846 | LR: 4.90e-06 | 1384.8ms/step | 14.4 samples/s +Step 41280/44000 | Loss: 2.8823 | LR: 4.86e-06 | 1384.0ms/step | 14.5 samples/s +Step 41290/44000 | Loss: 2.8698 | LR: 4.83e-06 | 1386.3ms/step | 14.4 samples/s +Step 41300/44000 | Loss: 2.8765 | LR: 4.79e-06 | 1383.5ms/step | 14.5 samples/s +Step 41310/44000 | Loss: 2.8625 | LR: 4.76e-06 | 1385.6ms/step | 14.4 samples/s +Step 41320/44000 | Loss: 2.8869 | LR: 4.72e-06 | 1387.3ms/step | 14.4 samples/s +Step 41330/44000 | Loss: 2.8725 | LR: 4.69e-06 | 1386.4ms/step | 14.4 samples/s diff --git a/epfl_final/logs/l8k365m_dense_resume2.log b/epfl_final/logs/l8k365m_dense_resume2.log new file mode 100644 index 0000000000000000000000000000000000000000..b70aacb1d96e2a64ad22f909fff57390bcc0a7be --- /dev/null +++ b/epfl_final/logs/l8k365m_dense_resume2.log @@ -0,0 +1,209 @@ +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run qlqhc3gy +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_045920-qlqhc3gy +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m dense 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qlqhc3gy +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 8k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "dense" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 41000 (epoch 0, micro_step 410000) +Resuming training from step 41000 +Resuming wandb run qlqhc3gy +Starting training... +Validation at step 41000: Loss = 2.8935, Perplexity = 18.06 +wandb: WARNING Tried to log to step 41000 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41010 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41020 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41030 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41040 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41050 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41060 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41070 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41080 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41090 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41100 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41110 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41120 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41130 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41140 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41150 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41160 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41170 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41180 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41190 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41200 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41210 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41220 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41230 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41240 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41250 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41260 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41270 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41280 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41290 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41300 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41310 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 41320 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. diff --git a/epfl_final/logs/l8k365m_dense_resume3.log b/epfl_final/logs/l8k365m_dense_resume3.log new file mode 100644 index 0000000000000000000000000000000000000000..5a9bfd5c7ddfe42ec990288e5f4d5906e8f5e3c0 --- /dev/null +++ b/epfl_final/logs/l8k365m_dense_resume3.log @@ -0,0 +1,513 @@ +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run qlqhc3gy +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_060003-qlqhc3gy +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m dense 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qlqhc3gy +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-42000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-42000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 8k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "dense" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-42000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-42000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 42000 (epoch 0, micro_step 420000) +Resuming training from step 42000 +Resuming wandb run qlqhc3gy +Starting training... +Validation at step 42000: Loss = 2.8956, Perplexity = 18.09 +wandb: WARNING Tried to log to step 42000 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42010 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42020 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42030 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42040 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42050 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42060 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42070 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42080 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42090 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42100 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42110 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42120 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42130 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42140 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42150 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42160 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42170 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42180 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42190 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42200 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42210 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42220 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42230 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42240 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42250 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42260 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42270 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42280 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42290 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42300 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42310 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42320 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42330 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42340 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42350 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42360 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42370 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42380 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42390 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42400 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42410 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42420 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42430 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42440 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42450 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42460 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42470 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42480 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42490 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42500 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42510 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42520 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42530 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42540 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42550 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42560 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42570 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42580 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42590 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42600 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42610 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42620 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42630 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42640 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42650 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42660 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42670 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42680 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42690 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42700 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42710 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42720 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42730 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42740 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42750 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42760 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42770 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42780 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42790 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42800 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42810 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42820 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42830 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42840 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42850 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42860 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42870 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42880 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 42890 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: updating run metadata +wandb: uploading wandb-summary.json; uploading config.yaml +wandb: uploading history steps 4399-4399, summary, console lines 302-303 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁ +wandb: eval/loss █▂▁▁▃ +wandb: eval/perplexity █▂▁▁▃ +wandb: eval/uses_dense_attention ▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁█▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec █████▁███████████████████████████▁██████ +wandb: speed/steps_per_sec ███████████████▁████████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.91276 +wandb: eval/perplexity 18.40744 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 2472.70107 +wandb: speed/samples_per_sec 8.08832 +wandb: speed/steps_per_sec 0.40442 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m dense 8k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qlqhc3gy +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260708_060003-qlqhc3gy/logs +Switching attention mode for step 42001: dense (dense step interval) +Step 42010/44000 | Loss: 2.9026 | LR: 2.61e-06 | 58442.7ms/step | 0.3 samples/s +Step 42020/44000 | Loss: 2.8940 | LR: 2.58e-06 | 2471.8ms/step | 8.1 samples/s +Step 42030/44000 | Loss: 2.8854 | LR: 2.56e-06 | 2472.3ms/step | 8.1 samples/s +Step 42040/44000 | Loss: 2.8688 | LR: 2.53e-06 | 2473.4ms/step | 8.1 samples/s +Step 42050/44000 | Loss: 2.8986 | LR: 2.50e-06 | 2475.4ms/step | 8.1 samples/s +Step 42060/44000 | Loss: 2.8971 | LR: 2.48e-06 | 2474.1ms/step | 8.1 samples/s +Step 42070/44000 | Loss: 2.9161 | LR: 2.45e-06 | 2473.5ms/step | 8.1 samples/s +Step 42080/44000 | Loss: 2.8754 | LR: 2.43e-06 | 2474.6ms/step | 8.1 samples/s +Step 42090/44000 | Loss: 2.8948 | LR: 2.40e-06 | 2475.4ms/step | 8.1 samples/s +Step 42100/44000 | Loss: 2.8972 | LR: 2.38e-06 | 2476.0ms/step | 8.1 samples/s +Step 42110/44000 | Loss: 2.9053 | LR: 2.35e-06 | 2475.6ms/step | 8.1 samples/s +Step 42120/44000 | Loss: 2.9192 | LR: 2.33e-06 | 2474.4ms/step | 8.1 samples/s +Step 42130/44000 | Loss: 2.8948 | LR: 2.30e-06 | 2473.3ms/step | 8.1 samples/s +Step 42140/44000 | Loss: 2.9002 | LR: 2.28e-06 | 2473.8ms/step | 8.1 samples/s +Step 42150/44000 | Loss: 2.8784 | LR: 2.25e-06 | 2474.0ms/step | 8.1 samples/s +Step 42160/44000 | Loss: 2.9180 | LR: 2.23e-06 | 2473.2ms/step | 8.1 samples/s +Step 42170/44000 | Loss: 2.9129 | LR: 2.21e-06 | 2473.5ms/step | 8.1 samples/s +Step 42180/44000 | Loss: 2.9178 | LR: 2.18e-06 | 2473.3ms/step | 8.1 samples/s +Step 42190/44000 | Loss: 2.9356 | LR: 2.16e-06 | 2473.3ms/step | 8.1 samples/s +Step 42200/44000 | Loss: 2.9144 | LR: 2.13e-06 | 2473.3ms/step | 8.1 samples/s +Step 42210/44000 | Loss: 2.8649 | LR: 2.11e-06 | 2474.5ms/step | 8.1 samples/s +Step 42220/44000 | Loss: 2.9082 | LR: 2.09e-06 | 2475.2ms/step | 8.1 samples/s +Step 42230/44000 | Loss: 2.9055 | LR: 2.06e-06 | 2474.6ms/step | 8.1 samples/s +Step 42240/44000 | Loss: 2.8562 | LR: 2.04e-06 | 2474.7ms/step | 8.1 samples/s +Step 42250/44000 | Loss: 2.8947 | LR: 2.02e-06 | 2474.9ms/step | 8.1 samples/s +Validation at step 42250: Loss = 2.9152, Perplexity = 18.45 +Step 42260/44000 | Loss: 2.9025 | LR: 1.99e-06 | 6679.7ms/step | 3.0 samples/s +Step 42270/44000 | Loss: 2.9179 | LR: 1.97e-06 | 2474.1ms/step | 8.1 samples/s +Step 42280/44000 | Loss: 2.8910 | LR: 1.95e-06 | 2474.6ms/step | 8.1 samples/s +Step 42290/44000 | Loss: 2.9246 | LR: 1.93e-06 | 2474.4ms/step | 8.1 samples/s +Step 42300/44000 | Loss: 2.8877 | LR: 1.90e-06 | 2473.5ms/step | 8.1 samples/s +Step 42310/44000 | Loss: 2.9260 | LR: 1.88e-06 | 2474.1ms/step | 8.1 samples/s +Step 42320/44000 | Loss: 2.9189 | LR: 1.86e-06 | 2473.8ms/step | 8.1 samples/s +Step 42330/44000 | Loss: 2.9417 | LR: 1.84e-06 | 2474.1ms/step | 8.1 samples/s +Step 42340/44000 | Loss: 2.8960 | LR: 1.82e-06 | 2474.1ms/step | 8.1 samples/s +Step 42350/44000 | Loss: 2.8536 | LR: 1.79e-06 | 2474.2ms/step | 8.1 samples/s +Step 42360/44000 | Loss: 2.9162 | LR: 1.77e-06 | 2474.4ms/step | 8.1 samples/s +Step 42370/44000 | Loss: 2.9019 | LR: 1.75e-06 | 2473.8ms/step | 8.1 samples/s +Step 42380/44000 | Loss: 2.9031 | LR: 1.73e-06 | 2473.9ms/step | 8.1 samples/s +Step 42390/44000 | Loss: 2.9117 | LR: 1.71e-06 | 2472.9ms/step | 8.1 samples/s +Step 42400/44000 | Loss: 2.8752 | LR: 1.69e-06 | 2472.6ms/step | 8.1 samples/s +Step 42410/44000 | Loss: 2.8923 | LR: 1.67e-06 | 2477.5ms/step | 8.1 samples/s +Step 42420/44000 | Loss: 2.8904 | LR: 1.64e-06 | 2478.4ms/step | 8.1 samples/s +Step 42430/44000 | Loss: 2.8909 | LR: 1.62e-06 | 2474.9ms/step | 8.1 samples/s +Step 42440/44000 | Loss: 2.8826 | LR: 1.60e-06 | 2473.5ms/step | 8.1 samples/s +Step 42450/44000 | Loss: 2.8496 | LR: 1.58e-06 | 2472.3ms/step | 8.1 samples/s +Step 42460/44000 | Loss: 2.8867 | LR: 1.56e-06 | 2473.0ms/step | 8.1 samples/s +Step 42470/44000 | Loss: 2.8888 | LR: 1.54e-06 | 2473.8ms/step | 8.1 samples/s +Step 42480/44000 | Loss: 2.8738 | LR: 1.52e-06 | 2474.0ms/step | 8.1 samples/s +Step 42490/44000 | Loss: 2.9266 | LR: 1.50e-06 | 2474.3ms/step | 8.1 samples/s +Step 42500/44000 | Loss: 2.9400 | LR: 1.48e-06 | 2473.3ms/step | 8.1 samples/s +Validation at step 42500: Loss = 2.9134, Perplexity = 18.42 +Step 42510/44000 | Loss: 2.8661 | LR: 1.46e-06 | 6676.2ms/step | 3.0 samples/s +Step 42520/44000 | Loss: 2.8836 | LR: 1.44e-06 | 2473.9ms/step | 8.1 samples/s +Step 42530/44000 | Loss: 2.8974 | LR: 1.42e-06 | 2473.8ms/step | 8.1 samples/s +Step 42540/44000 | Loss: 2.8869 | LR: 1.40e-06 | 2473.9ms/step | 8.1 samples/s +Step 42550/44000 | Loss: 2.9057 | LR: 1.39e-06 | 2473.6ms/step | 8.1 samples/s +Step 42560/44000 | Loss: 2.9137 | LR: 1.37e-06 | 2476.8ms/step | 8.1 samples/s +Step 42570/44000 | Loss: 2.9186 | LR: 1.35e-06 | 2474.4ms/step | 8.1 samples/s +Step 42580/44000 | Loss: 2.9066 | LR: 1.33e-06 | 2472.8ms/step | 8.1 samples/s +Step 42590/44000 | Loss: 2.9172 | LR: 1.31e-06 | 2473.4ms/step | 8.1 samples/s +Step 42600/44000 | Loss: 2.9135 | LR: 1.29e-06 | 2472.6ms/step | 8.1 samples/s +Step 42610/44000 | Loss: 2.8956 | LR: 1.27e-06 | 2474.9ms/step | 8.1 samples/s +Step 42620/44000 | Loss: 2.8944 | LR: 1.25e-06 | 2475.7ms/step | 8.1 samples/s +Step 42630/44000 | Loss: 2.8900 | LR: 1.24e-06 | 2474.4ms/step | 8.1 samples/s +Step 42640/44000 | Loss: 2.9035 | LR: 1.22e-06 | 2474.4ms/step | 8.1 samples/s +Step 42650/44000 | Loss: 2.8893 | LR: 1.20e-06 | 2477.1ms/step | 8.1 samples/s +Step 42660/44000 | Loss: 2.9111 | LR: 1.18e-06 | 2474.4ms/step | 8.1 samples/s +Step 42670/44000 | Loss: 2.8768 | LR: 1.17e-06 | 2473.4ms/step | 8.1 samples/s +Step 42680/44000 | Loss: 2.8596 | LR: 1.15e-06 | 2474.7ms/step | 8.1 samples/s +Step 42690/44000 | Loss: 2.8910 | LR: 1.13e-06 | 2475.0ms/step | 8.1 samples/s +Step 42700/44000 | Loss: 2.9024 | LR: 1.11e-06 | 2474.9ms/step | 8.1 samples/s +Step 42710/44000 | Loss: 2.9106 | LR: 1.10e-06 | 2475.7ms/step | 8.1 samples/s +Step 42720/44000 | Loss: 2.9081 | LR: 1.08e-06 | 2475.6ms/step | 8.1 samples/s +Step 42730/44000 | Loss: 2.8979 | LR: 1.06e-06 | 2475.9ms/step | 8.1 samples/s +Step 42740/44000 | Loss: 2.9098 | LR: 1.05e-06 | 2473.4ms/step | 8.1 samples/s +Step 42750/44000 | Loss: 2.9176 | LR: 1.03e-06 | 2474.7ms/step | 8.1 samples/s +Validation at step 42750: Loss = 2.9128, Perplexity = 18.41 +Step 42760/44000 | Loss: 2.8979 | LR: 1.01e-06 | 6675.5ms/step | 3.0 samples/s +Step 42770/44000 | Loss: 2.9136 | LR: 9.97e-07 | 2473.9ms/step | 8.1 samples/s +Step 42780/44000 | Loss: 2.8889 | LR: 9.81e-07 | 2474.6ms/step | 8.1 samples/s +Step 42790/44000 | Loss: 2.9061 | LR: 9.65e-07 | 2475.3ms/step | 8.1 samples/s +Step 42800/44000 | Loss: 2.9060 | LR: 9.49e-07 | 2474.3ms/step | 8.1 samples/s +Step 42810/44000 | Loss: 2.8747 | LR: 9.33e-07 | 2474.0ms/step | 8.1 samples/s +Step 42820/44000 | Loss: 2.9063 | LR: 9.18e-07 | 2474.3ms/step | 8.1 samples/s +Step 42830/44000 | Loss: 2.9125 | LR: 9.02e-07 | 2475.6ms/step | 8.1 samples/s +Step 42840/44000 | Loss: 2.9042 | LR: 8.87e-07 | 2474.3ms/step | 8.1 samples/s +Step 42850/44000 | Loss: 2.8695 | LR: 8.72e-07 | 2475.6ms/step | 8.1 samples/s +Step 42860/44000 | Loss: 2.8815 | LR: 8.57e-07 | 2478.8ms/step | 8.1 samples/s +Step 42870/44000 | Loss: 2.9173 | LR: 8.42e-07 | 2474.9ms/step | 8.1 samples/s +Step 42880/44000 | Loss: 2.9004 | LR: 8.27e-07 | 2472.8ms/step | 8.1 samples/s +Step 42890/44000 | Loss: 2.9127 | LR: 8.12e-07 | 2472.7ms/step | 8.1 samples/s +Step 42900/44000 | Loss: 2.9098 | LR: 7.98e-07 | 2472.7ms/step | 8.1 samples/s +Step 42910/44000 | Loss: 2.8639 | LR: 7.83e-07 | 2472.6ms/step | 8.1 samples/s +Step 42920/44000 | Loss: 2.8815 | LR: 7.69e-07 | 2473.5ms/step | 8.1 samples/s +Step 42930/44000 | Loss: 2.8930 | LR: 7.55e-07 | 2473.4ms/step | 8.1 samples/s +Step 42940/44000 | Loss: 2.8811 | LR: 7.41e-07 | 2473.3ms/step | 8.1 samples/s +Step 42950/44000 | Loss: 2.9038 | LR: 7.27e-07 | 2473.6ms/step | 8.1 samples/s +Step 42960/44000 | Loss: 2.8809 | LR: 7.13e-07 | 2472.9ms/step | 8.1 samples/s +Step 42970/44000 | Loss: 2.9096 | LR: 6.99e-07 | 2472.7ms/step | 8.1 samples/s +Step 42980/44000 | Loss: 2.9116 | LR: 6.86e-07 | 2472.9ms/step | 8.1 samples/s +Step 42990/44000 | Loss: 2.8679 | LR: 6.72e-07 | 2473.2ms/step | 8.1 samples/s +Step 43000/44000 | Loss: 2.8914 | LR: 6.59e-07 | 2473.3ms/step | 8.1 samples/s +Validation at step 43000: Loss = 2.9128, Perplexity = 18.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-43000 +Step 43010/44000 | Loss: 2.8783 | LR: 6.46e-07 | 6981.5ms/step | 2.9 samples/s +Step 43020/44000 | Loss: 2.9055 | LR: 6.33e-07 | 2473.4ms/step | 8.1 samples/s +Step 43030/44000 | Loss: 2.9116 | LR: 6.20e-07 | 2473.2ms/step | 8.1 samples/s +Step 43040/44000 | Loss: 2.8814 | LR: 6.08e-07 | 2472.8ms/step | 8.1 samples/s +Step 43050/44000 | Loss: 2.8901 | LR: 5.95e-07 | 2473.4ms/step | 8.1 samples/s +Step 43060/44000 | Loss: 2.8589 | LR: 5.83e-07 | 2473.9ms/step | 8.1 samples/s +Step 43070/44000 | Loss: 2.8816 | LR: 5.70e-07 | 2474.0ms/step | 8.1 samples/s +Step 43080/44000 | Loss: 2.8898 | LR: 5.58e-07 | 2474.0ms/step | 8.1 samples/s +Step 43090/44000 | Loss: 2.9181 | LR: 5.46e-07 | 2474.5ms/step | 8.1 samples/s +Step 43100/44000 | Loss: 2.8988 | LR: 5.34e-07 | 2473.4ms/step | 8.1 samples/s +Step 43110/44000 | Loss: 2.8700 | LR: 5.22e-07 | 2472.6ms/step | 8.1 samples/s +Step 43120/44000 | Loss: 2.8807 | LR: 5.11e-07 | 2472.0ms/step | 8.1 samples/s +Step 43130/44000 | Loss: 2.9162 | LR: 4.99e-07 | 2473.0ms/step | 8.1 samples/s +Step 43140/44000 | Loss: 2.9092 | LR: 4.88e-07 | 2472.3ms/step | 8.1 samples/s +Step 43150/44000 | Loss: 2.8604 | LR: 4.76e-07 | 2472.6ms/step | 8.1 samples/s +Step 43160/44000 | Loss: 2.8885 | LR: 4.65e-07 | 2471.5ms/step | 8.1 samples/s +Step 43170/44000 | Loss: 2.8890 | LR: 4.54e-07 | 2472.7ms/step | 8.1 samples/s +Step 43180/44000 | Loss: 2.8879 | LR: 4.43e-07 | 2474.7ms/step | 8.1 samples/s +Step 43190/44000 | Loss: 2.8875 | LR: 4.33e-07 | 2473.3ms/step | 8.1 samples/s +Step 43200/44000 | Loss: 2.9041 | LR: 4.22e-07 | 2473.3ms/step | 8.1 samples/s +Step 43210/44000 | Loss: 2.8635 | LR: 4.12e-07 | 2473.5ms/step | 8.1 samples/s +Step 43220/44000 | Loss: 2.9215 | LR: 4.01e-07 | 2473.8ms/step | 8.1 samples/s +Step 43230/44000 | Loss: 2.9011 | LR: 3.91e-07 | 2473.3ms/step | 8.1 samples/s +Step 43240/44000 | Loss: 2.9095 | LR: 3.81e-07 | 2473.7ms/step | 8.1 samples/s +Step 43250/44000 | Loss: 2.9180 | LR: 3.71e-07 | 2473.4ms/step | 8.1 samples/s +Validation at step 43250: Loss = 2.9128, Perplexity = 18.41 +Step 43260/44000 | Loss: 2.9016 | LR: 3.61e-07 | 6676.0ms/step | 3.0 samples/s +Step 43270/44000 | Loss: 2.9167 | LR: 3.51e-07 | 2472.6ms/step | 8.1 samples/s +Step 43280/44000 | Loss: 2.8757 | LR: 3.42e-07 | 2472.6ms/step | 8.1 samples/s +Step 43290/44000 | Loss: 2.8804 | LR: 3.32e-07 | 2472.1ms/step | 8.1 samples/s +Step 43300/44000 | Loss: 2.9320 | LR: 3.23e-07 | 2471.2ms/step | 8.1 samples/s +Step 43310/44000 | Loss: 2.8923 | LR: 3.14e-07 | 2471.9ms/step | 8.1 samples/s +Step 43320/44000 | Loss: 2.8882 | LR: 3.05e-07 | 2480.6ms/step | 8.1 samples/s +Step 43330/44000 | Loss: 2.9103 | LR: 2.96e-07 | 2472.7ms/step | 8.1 samples/s +Step 43340/44000 | Loss: 2.8948 | LR: 2.87e-07 | 2473.0ms/step | 8.1 samples/s +Step 43350/44000 | Loss: 2.8576 | LR: 2.79e-07 | 2473.1ms/step | 8.1 samples/s +Step 43360/44000 | Loss: 2.8828 | LR: 2.70e-07 | 2473.6ms/step | 8.1 samples/s +Step 43370/44000 | Loss: 2.8778 | LR: 2.62e-07 | 2474.4ms/step | 8.1 samples/s +Step 43380/44000 | Loss: 2.8832 | LR: 2.53e-07 | 2474.2ms/step | 8.1 samples/s +Step 43390/44000 | Loss: 2.8876 | LR: 2.45e-07 | 2473.4ms/step | 8.1 samples/s +Step 43400/44000 | Loss: 2.8796 | LR: 2.37e-07 | 2475.7ms/step | 8.1 samples/s +Step 43410/44000 | Loss: 2.8704 | LR: 2.30e-07 | 2480.9ms/step | 8.1 samples/s +Step 43420/44000 | Loss: 2.9239 | LR: 2.22e-07 | 2479.2ms/step | 8.1 samples/s +Step 43430/44000 | Loss: 2.8647 | LR: 2.14e-07 | 2482.4ms/step | 8.1 samples/s +Step 43440/44000 | Loss: 2.8877 | LR: 2.07e-07 | 2481.7ms/step | 8.1 samples/s +Step 43450/44000 | Loss: 2.8725 | LR: 1.99e-07 | 2479.4ms/step | 8.1 samples/s +Step 43460/44000 | Loss: 2.8947 | LR: 1.92e-07 | 2475.6ms/step | 8.1 samples/s +Step 43470/44000 | Loss: 2.8734 | LR: 1.85e-07 | 2477.6ms/step | 8.1 samples/s +Step 43480/44000 | Loss: 2.8769 | LR: 1.78e-07 | 2477.4ms/step | 8.1 samples/s +Step 43490/44000 | Loss: 2.9095 | LR: 1.72e-07 | 2476.5ms/step | 8.1 samples/s +Step 43500/44000 | Loss: 2.8874 | LR: 1.65e-07 | 2473.5ms/step | 8.1 samples/s +Validation at step 43500: Loss = 2.9128, Perplexity = 18.41 +Step 43510/44000 | Loss: 2.8898 | LR: 1.58e-07 | 6675.9ms/step | 3.0 samples/s +Step 43520/44000 | Loss: 2.8896 | LR: 1.52e-07 | 2471.9ms/step | 8.1 samples/s +Step 43530/44000 | Loss: 2.8807 | LR: 1.46e-07 | 2472.7ms/step | 8.1 samples/s +Step 43540/44000 | Loss: 2.8856 | LR: 1.40e-07 | 2472.7ms/step | 8.1 samples/s +Step 43550/44000 | Loss: 2.9021 | LR: 1.34e-07 | 2472.8ms/step | 8.1 samples/s +Step 43560/44000 | Loss: 2.8786 | LR: 1.28e-07 | 2473.0ms/step | 8.1 samples/s +Step 43570/44000 | Loss: 2.8951 | LR: 1.22e-07 | 2472.2ms/step | 8.1 samples/s +Step 43580/44000 | Loss: 2.9028 | LR: 1.16e-07 | 2472.8ms/step | 8.1 samples/s +Step 43590/44000 | Loss: 2.9086 | LR: 1.11e-07 | 2472.4ms/step | 8.1 samples/s +Step 43600/44000 | Loss: 2.8982 | LR: 1.06e-07 | 2472.2ms/step | 8.1 samples/s +Step 43610/44000 | Loss: 2.8895 | LR: 1.00e-07 | 2472.7ms/step | 8.1 samples/s +Step 43620/44000 | Loss: 2.8601 | LR: 9.52e-08 | 2473.7ms/step | 8.1 samples/s +Step 43630/44000 | Loss: 2.9137 | LR: 9.03e-08 | 2474.1ms/step | 8.1 samples/s +Step 43640/44000 | Loss: 2.9201 | LR: 8.55e-08 | 2479.9ms/step | 8.1 samples/s +Step 43650/44000 | Loss: 2.8716 | LR: 8.08e-08 | 2475.0ms/step | 8.1 samples/s +Step 43660/44000 | Loss: 2.8757 | LR: 7.62e-08 | 2473.3ms/step | 8.1 samples/s +Step 43670/44000 | Loss: 2.9088 | LR: 7.18e-08 | 2473.4ms/step | 8.1 samples/s +Step 43680/44000 | Loss: 2.9063 | LR: 6.75e-08 | 2476.3ms/step | 8.1 samples/s +Step 43690/44000 | Loss: 2.9049 | LR: 6.34e-08 | 2474.2ms/step | 8.1 samples/s +Step 43700/44000 | Loss: 2.9062 | LR: 5.94e-08 | 2472.7ms/step | 8.1 samples/s +Step 43710/44000 | Loss: 2.8842 | LR: 5.55e-08 | 2472.9ms/step | 8.1 samples/s +Step 43720/44000 | Loss: 2.8903 | LR: 5.17e-08 | 2472.2ms/step | 8.1 samples/s +Step 43730/44000 | Loss: 2.9026 | LR: 4.81e-08 | 2472.5ms/step | 8.1 samples/s +Step 43740/44000 | Loss: 2.8897 | LR: 4.46e-08 | 2473.9ms/step | 8.1 samples/s +Step 43750/44000 | Loss: 2.8944 | LR: 4.12e-08 | 2473.0ms/step | 8.1 samples/s +Validation at step 43750: Loss = 2.9127, Perplexity = 18.41 +Step 43760/44000 | Loss: 2.8659 | LR: 3.80e-08 | 6671.9ms/step | 3.0 samples/s +Step 43770/44000 | Loss: 2.8963 | LR: 3.49e-08 | 2475.3ms/step | 8.1 samples/s +Step 43780/44000 | Loss: 2.8845 | LR: 3.19e-08 | 2475.8ms/step | 8.1 samples/s +Step 43790/44000 | Loss: 2.9188 | LR: 2.91e-08 | 2477.7ms/step | 8.1 samples/s +Step 43800/44000 | Loss: 2.8794 | LR: 2.64e-08 | 2474.1ms/step | 8.1 samples/s +Step 43810/44000 | Loss: 2.8787 | LR: 2.38e-08 | 2474.4ms/step | 8.1 samples/s +Step 43820/44000 | Loss: 2.8945 | LR: 2.14e-08 | 2474.2ms/step | 8.1 samples/s +Step 43830/44000 | Loss: 2.8698 | LR: 1.91e-08 | 2474.8ms/step | 8.1 samples/s +Step 43840/44000 | Loss: 2.8995 | LR: 1.69e-08 | 2474.3ms/step | 8.1 samples/s +Step 43850/44000 | Loss: 2.8989 | LR: 1.48e-08 | 2474.5ms/step | 8.1 samples/s +Step 43860/44000 | Loss: 2.9028 | LR: 1.29e-08 | 2474.0ms/step | 8.1 samples/s +Step 43870/44000 | Loss: 2.8806 | LR: 1.11e-08 | 2473.2ms/step | 8.1 samples/s +Step 43880/44000 | Loss: 2.8919 | LR: 9.50e-09 | 2475.0ms/step | 8.1 samples/s +Step 43890/44000 | Loss: 2.9090 | LR: 7.98e-09 | 2473.2ms/step | 8.1 samples/s +Step 43900/44000 | Loss: 2.8898 | LR: 6.60e-09 | 2475.8ms/step | 8.1 samples/s +Step 43910/44000 | Loss: 2.9074 | LR: 5.34e-09 | 2474.6ms/step | 8.1 samples/s +Step 43920/44000 | Loss: 2.8697 | LR: 4.22e-09 | 2476.8ms/step | 8.1 samples/s +Step 43930/44000 | Loss: 2.9084 | LR: 3.23e-09 | 2474.6ms/step | 8.1 samples/s +Step 43940/44000 | Loss: 2.8652 | LR: 2.37e-09 | 2478.6ms/step | 8.1 samples/s +Step 43950/44000 | Loss: 2.9014 | LR: 1.65e-09 | 2473.5ms/step | 8.1 samples/s +Step 43960/44000 | Loss: 2.8861 | LR: 1.06e-09 | 2473.2ms/step | 8.1 samples/s +Step 43970/44000 | Loss: 2.9254 | LR: 5.94e-10 | 2474.1ms/step | 8.1 samples/s +Step 43980/44000 | Loss: 2.8674 | LR: 2.64e-10 | 2473.4ms/step | 8.1 samples/s +Step 43990/44000 | Loss: 2.9117 | LR: 6.60e-11 | 2474.6ms/step | 8.1 samples/s +Step 44000/44000 | Loss: 2.8938 | LR: 0.00e+00 | 2472.7ms/step | 8.1 samples/s +Validation at step 44000: Loss = 2.9128, Perplexity = 18.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000 +Training complete! diff --git a/epfl_final/logs/l8k365m_fixed_s1.log b/epfl_final/logs/l8k365m_fixed_s1.log new file mode 100644 index 0000000000000000000000000000000000000000..442ed6837c793d5d4c185f5a60321d4e229e4c34 --- /dev/null +++ b/epfl_final/logs/l8k365m_fixed_s1.log @@ -0,0 +1,1747 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m fixed s1 8k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "fixed", + "seed1" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_fixed_s1", + "sparse_mask_seed": 1, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run dbtl6d2o +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_034612-dbtl6d2o +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m fixed s1 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/dbtl6d2o +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0399 | LR: 6.67e-06 | 6242.3ms/step | 3.2 samples/s +Step 20/44000 | Loss: 11.9183 | LR: 1.33e-05 | 1251.7ms/step | 16.0 samples/s +Step 30/44000 | Loss: 11.5362 | LR: 2.00e-05 | 1251.7ms/step | 16.0 samples/s +Step 40/44000 | Loss: 11.0000 | LR: 2.67e-05 | 1248.0ms/step | 16.0 samples/s +Step 50/44000 | Loss: 10.7288 | LR: 3.33e-05 | 1249.6ms/step | 16.0 samples/s +Step 60/44000 | Loss: 10.4874 | LR: 4.00e-05 | 1249.8ms/step | 16.0 samples/s +Step 70/44000 | Loss: 10.2507 | LR: 4.67e-05 | 1249.8ms/step | 16.0 samples/s +Step 80/44000 | Loss: 9.9442 | LR: 5.33e-05 | 1250.8ms/step | 16.0 samples/s +Step 90/44000 | Loss: 9.5784 | LR: 6.00e-05 | 1252.6ms/step | 16.0 samples/s +Step 100/44000 | Loss: 9.1218 | LR: 6.67e-05 | 1251.2ms/step | 16.0 samples/s +Step 110/44000 | Loss: 8.5918 | LR: 7.33e-05 | 1253.4ms/step | 16.0 samples/s +Step 120/44000 | Loss: 8.1441 | LR: 8.00e-05 | 1256.2ms/step | 15.9 samples/s +Step 130/44000 | Loss: 7.7648 | LR: 8.67e-05 | 1256.5ms/step | 15.9 samples/s +Step 140/44000 | Loss: 7.5316 | LR: 9.33e-05 | 1258.5ms/step | 15.9 samples/s +Step 150/44000 | Loss: 7.3415 | LR: 1.00e-04 | 1260.4ms/step | 15.9 samples/s +Step 160/44000 | Loss: 7.2319 | LR: 1.07e-04 | 1261.7ms/step | 15.9 samples/s +Step 170/44000 | Loss: 7.1245 | LR: 1.13e-04 | 1263.1ms/step | 15.8 samples/s +Step 180/44000 | Loss: 7.0297 | LR: 1.20e-04 | 1265.4ms/step | 15.8 samples/s +Step 190/44000 | Loss: 6.9441 | LR: 1.27e-04 | 1266.7ms/step | 15.8 samples/s +Step 200/44000 | Loss: 6.8844 | LR: 1.33e-04 | 1267.1ms/step | 15.8 samples/s +Step 210/44000 | Loss: 6.8134 | LR: 1.40e-04 | 1266.1ms/step | 15.8 samples/s +Step 220/44000 | Loss: 6.7140 | LR: 1.47e-04 | 1267.4ms/step | 15.8 samples/s +Step 230/44000 | Loss: 6.6094 | LR: 1.53e-04 | 1267.4ms/step | 15.8 samples/s +Step 240/44000 | Loss: 6.5445 | LR: 1.60e-04 | 1267.2ms/step | 15.8 samples/s +Step 250/44000 | Loss: 6.4573 | LR: 1.67e-04 | 1266.9ms/step | 15.8 samples/s +Validation at step 250: Loss = 6.4496, Perplexity = 632.45 +Step 260/44000 | Loss: 6.4300 | LR: 1.73e-04 | 5458.7ms/step | 3.7 samples/s +Step 270/44000 | Loss: 6.3543 | LR: 1.80e-04 | 1263.1ms/step | 15.8 samples/s +Step 280/44000 | Loss: 6.3163 | LR: 1.87e-04 | 1266.9ms/step | 15.8 samples/s +Step 290/44000 | Loss: 6.2799 | LR: 1.93e-04 | 1267.1ms/step | 15.8 samples/s +Step 300/44000 | Loss: 6.2121 | LR: 2.00e-04 | 1269.8ms/step | 15.7 samples/s +Step 310/44000 | Loss: 6.1845 | LR: 2.07e-04 | 1271.4ms/step | 15.7 samples/s +Step 320/44000 | Loss: 6.1355 | LR: 2.13e-04 | 1271.6ms/step | 15.7 samples/s +Step 330/44000 | Loss: 6.0789 | LR: 2.20e-04 | 1269.9ms/step | 15.7 samples/s +Step 340/44000 | Loss: 6.0623 | LR: 2.27e-04 | 1266.4ms/step | 15.8 samples/s +Step 350/44000 | Loss: 5.9903 | LR: 2.33e-04 | 1269.4ms/step | 15.8 samples/s +Step 360/44000 | Loss: 5.9558 | LR: 2.40e-04 | 1272.0ms/step | 15.7 samples/s +Step 370/44000 | Loss: 5.9256 | LR: 2.47e-04 | 1272.2ms/step | 15.7 samples/s +Step 380/44000 | Loss: 5.9157 | LR: 2.53e-04 | 1271.4ms/step | 15.7 samples/s +Step 390/44000 | Loss: 5.8353 | LR: 2.60e-04 | 1273.0ms/step | 15.7 samples/s +Step 400/44000 | Loss: 5.8585 | LR: 2.67e-04 | 1273.0ms/step | 15.7 samples/s +Step 410/44000 | Loss: 5.8053 | LR: 2.73e-04 | 1273.5ms/step | 15.7 samples/s +Step 420/44000 | Loss: 5.7999 | LR: 2.80e-04 | 1273.2ms/step | 15.7 samples/s +Step 430/44000 | Loss: 5.7359 | LR: 2.87e-04 | 1273.2ms/step | 15.7 samples/s +Step 440/44000 | Loss: 5.7068 | LR: 2.93e-04 | 1272.6ms/step | 15.7 samples/s +Step 450/44000 | Loss: 5.6716 | LR: 3.00e-04 | 1272.4ms/step | 15.7 samples/s +Step 460/44000 | Loss: 5.6015 | LR: 3.07e-04 | 1273.0ms/step | 15.7 samples/s +Step 470/44000 | Loss: 5.6159 | LR: 3.13e-04 | 1273.3ms/step | 15.7 samples/s +Step 480/44000 | Loss: 5.5625 | LR: 3.20e-04 | 1274.6ms/step | 15.7 samples/s +Step 490/44000 | Loss: 5.5100 | LR: 3.27e-04 | 1274.4ms/step | 15.7 samples/s +Step 500/44000 | Loss: 5.4977 | LR: 3.33e-04 | 1273.7ms/step | 15.7 samples/s +Validation at step 500: Loss = 5.4838, Perplexity = 240.76 +Step 510/44000 | Loss: 5.4662 | LR: 3.40e-04 | 3417.0ms/step | 5.9 samples/s +Step 520/44000 | Loss: 5.4221 | LR: 3.47e-04 | 1275.5ms/step | 15.7 samples/s +Step 530/44000 | Loss: 5.4416 | LR: 3.53e-04 | 1274.9ms/step | 15.7 samples/s +Step 540/44000 | Loss: 5.3638 | LR: 3.60e-04 | 1275.0ms/step | 15.7 samples/s +Step 550/44000 | Loss: 5.3360 | LR: 3.67e-04 | 1274.4ms/step | 15.7 samples/s +Step 560/44000 | Loss: 5.3138 | LR: 3.73e-04 | 1274.6ms/step | 15.7 samples/s +Step 570/44000 | Loss: 5.3098 | LR: 3.80e-04 | 1274.9ms/step | 15.7 samples/s +Step 580/44000 | Loss: 5.2659 | LR: 3.87e-04 | 1274.3ms/step | 15.7 samples/s +Step 590/44000 | Loss: 5.2098 | LR: 3.93e-04 | 1275.0ms/step | 15.7 samples/s +Step 600/44000 | Loss: 5.1960 | LR: 4.00e-04 | 1275.1ms/step | 15.7 samples/s +Step 610/44000 | Loss: 5.1884 | LR: 4.07e-04 | 1274.7ms/step | 15.7 samples/s +Step 620/44000 | Loss: 5.1685 | LR: 4.13e-04 | 1274.7ms/step | 15.7 samples/s +Step 630/44000 | Loss: 5.0688 | LR: 4.20e-04 | 1275.1ms/step | 15.7 samples/s +Step 640/44000 | Loss: 5.0573 | LR: 4.27e-04 | 1275.0ms/step | 15.7 samples/s +Step 650/44000 | Loss: 5.0435 | LR: 4.33e-04 | 1274.4ms/step | 15.7 samples/s +Step 660/44000 | Loss: 5.0296 | LR: 4.40e-04 | 1274.2ms/step | 15.7 samples/s +Step 670/44000 | Loss: 4.9937 | LR: 4.47e-04 | 1274.9ms/step | 15.7 samples/s +Step 680/44000 | Loss: 4.9516 | LR: 4.53e-04 | 1276.0ms/step | 15.7 samples/s +Step 690/44000 | Loss: 4.9208 | LR: 4.60e-04 | 1274.9ms/step | 15.7 samples/s +Step 700/44000 | Loss: 4.9123 | LR: 4.67e-04 | 1275.6ms/step | 15.7 samples/s +Step 710/44000 | Loss: 4.8776 | LR: 4.73e-04 | 1275.6ms/step | 15.7 samples/s +Step 720/44000 | Loss: 4.8233 | LR: 4.80e-04 | 1275.3ms/step | 15.7 samples/s +Step 730/44000 | Loss: 4.8206 | LR: 4.87e-04 | 1273.2ms/step | 15.7 samples/s +Step 740/44000 | Loss: 4.7523 | LR: 4.93e-04 | 1272.0ms/step | 15.7 samples/s +Step 750/44000 | Loss: 4.7825 | LR: 5.00e-04 | 1272.0ms/step | 15.7 samples/s +Validation at step 750: Loss = 4.7673, Perplexity = 117.60 +Step 760/44000 | Loss: 4.7578 | LR: 5.00e-04 | 3416.5ms/step | 5.9 samples/s +Step 770/44000 | Loss: 4.7063 | LR: 5.00e-04 | 1275.0ms/step | 15.7 samples/s +Step 780/44000 | Loss: 4.6996 | LR: 5.00e-04 | 1273.4ms/step | 15.7 samples/s +Step 790/44000 | Loss: 4.6809 | LR: 5.00e-04 | 1274.5ms/step | 15.7 samples/s +Step 800/44000 | Loss: 4.6427 | LR: 5.00e-04 | 1275.5ms/step | 15.7 samples/s +Step 810/44000 | Loss: 4.6331 | LR: 5.00e-04 | 1274.2ms/step | 15.7 samples/s +Step 820/44000 | Loss: 4.6061 | LR: 5.00e-04 | 1275.0ms/step | 15.7 samples/s +Step 830/44000 | Loss: 4.5853 | LR: 5.00e-04 | 1274.8ms/step | 15.7 samples/s +Step 840/44000 | Loss: 4.5512 | LR: 5.00e-04 | 1275.5ms/step | 15.7 samples/s +Step 850/44000 | Loss: 4.5440 | LR: 5.00e-04 | 1274.0ms/step | 15.7 samples/s +Step 860/44000 | Loss: 4.5593 | LR: 5.00e-04 | 1273.4ms/step | 15.7 samples/s +Step 870/44000 | Loss: 4.4851 | LR: 5.00e-04 | 1274.3ms/step | 15.7 samples/s +Step 880/44000 | Loss: 4.4988 | LR: 5.00e-04 | 1274.8ms/step | 15.7 samples/s +Step 890/44000 | Loss: 4.4711 | LR: 5.00e-04 | 1275.3ms/step | 15.7 samples/s +Step 900/44000 | Loss: 4.4524 | LR: 5.00e-04 | 1275.3ms/step | 15.7 samples/s +Step 910/44000 | Loss: 4.4280 | LR: 5.00e-04 | 1275.1ms/step | 15.7 samples/s +Step 920/44000 | Loss: 4.4207 | LR: 5.00e-04 | 1275.0ms/step | 15.7 samples/s +Step 930/44000 | Loss: 4.3925 | LR: 5.00e-04 | 1277.0ms/step | 15.7 samples/s +Step 940/44000 | Loss: 4.3870 | LR: 5.00e-04 | 1275.6ms/step | 15.7 samples/s +Step 950/44000 | Loss: 4.3782 | LR: 5.00e-04 | 1275.3ms/step | 15.7 samples/s +Step 960/44000 | Loss: 4.3351 | LR: 5.00e-04 | 1274.4ms/step | 15.7 samples/s +Step 970/44000 | Loss: 4.3661 | LR: 5.00e-04 | 1275.0ms/step | 15.7 samples/s +Step 980/44000 | Loss: 4.3487 | LR: 5.00e-04 | 1274.4ms/step | 15.7 samples/s +Step 990/44000 | Loss: 4.3253 | LR: 5.00e-04 | 1276.6ms/step | 15.7 samples/s +Step 1000/44000 | Loss: 4.3188 | LR: 5.00e-04 | 1274.3ms/step | 15.7 samples/s +Validation at step 1000: Loss = 4.3212, Perplexity = 75.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-1000 +Step 1010/44000 | Loss: 4.2780 | LR: 5.00e-04 | 3713.7ms/step | 5.4 samples/s +Step 1020/44000 | Loss: 4.2860 | LR: 5.00e-04 | 1270.6ms/step | 15.7 samples/s +Step 1030/44000 | Loss: 4.2369 | LR: 5.00e-04 | 1271.3ms/step | 15.7 samples/s +Step 1040/44000 | Loss: 4.2526 | LR: 5.00e-04 | 1273.2ms/step | 15.7 samples/s +Step 1050/44000 | Loss: 4.2458 | LR: 5.00e-04 | 1273.1ms/step | 15.7 samples/s +Step 1060/44000 | Loss: 4.2363 | LR: 5.00e-04 | 1272.4ms/step | 15.7 samples/s +Step 1070/44000 | Loss: 4.2202 | LR: 5.00e-04 | 1272.9ms/step | 15.7 samples/s +Step 1080/44000 | Loss: 4.2063 | LR: 5.00e-04 | 1274.2ms/step | 15.7 samples/s +Step 1090/44000 | Loss: 4.2271 | LR: 5.00e-04 | 1274.9ms/step | 15.7 samples/s +Step 1100/44000 | Loss: 4.2037 | LR: 5.00e-04 | 1274.9ms/step | 15.7 samples/s +Step 1110/44000 | Loss: 4.1832 | LR: 5.00e-04 | 1273.8ms/step | 15.7 samples/s +Step 1120/44000 | Loss: 4.1684 | LR: 5.00e-04 | 1274.3ms/step | 15.7 samples/s +Step 1130/44000 | Loss: 4.1425 | LR: 5.00e-04 | 1273.7ms/step | 15.7 samples/s +Step 1140/44000 | Loss: 4.1656 | LR: 5.00e-04 | 1273.3ms/step | 15.7 samples/s +Step 1150/44000 | Loss: 4.1151 | LR: 5.00e-04 | 1273.0ms/step | 15.7 samples/s +Step 1160/44000 | Loss: 4.1147 | LR: 5.00e-04 | 1272.6ms/step | 15.7 samples/s +Step 1170/44000 | Loss: 4.1227 | LR: 5.00e-04 | 1273.4ms/step | 15.7 samples/s +Step 1180/44000 | Loss: 4.1042 | LR: 5.00e-04 | 1273.3ms/step | 15.7 samples/s +Step 1190/44000 | Loss: 4.1265 | LR: 5.00e-04 | 1273.8ms/step | 15.7 samples/s +Step 1200/44000 | Loss: 4.1177 | LR: 5.00e-04 | 1274.5ms/step | 15.7 samples/s +Step 1210/44000 | Loss: 4.1236 | LR: 5.00e-04 | 1274.9ms/step | 15.7 samples/s +Step 1220/44000 | Loss: 4.0773 | LR: 5.00e-04 | 1273.5ms/step | 15.7 samples/s +Step 1230/44000 | Loss: 4.0701 | LR: 5.00e-04 | 1273.9ms/step | 15.7 samples/s +Step 1240/44000 | Loss: 4.0728 | LR: 5.00e-04 | 1275.1ms/step | 15.7 samples/s +Step 1250/44000 | Loss: 4.0770 | LR: 5.00e-04 | 1274.8ms/step | 15.7 samples/s +Validation at step 1250: Loss = 4.0714, Perplexity = 58.64 +Step 1260/44000 | Loss: 4.0471 | LR: 5.00e-04 | 3425.2ms/step | 5.8 samples/s +Step 1270/44000 | Loss: 4.0466 | LR: 5.00e-04 | 1274.4ms/step | 15.7 samples/s +Step 1280/44000 | Loss: 4.0544 | LR: 5.00e-04 | 1275.3ms/step | 15.7 samples/s +Step 1290/44000 | Loss: 4.0026 | LR: 5.00e-04 | 1275.8ms/step | 15.7 samples/s +Step 1300/44000 | Loss: 4.0536 | LR: 5.00e-04 | 1273.0ms/step | 15.7 samples/s +Step 1310/44000 | Loss: 3.9897 | LR: 5.00e-04 | 1273.3ms/step | 15.7 samples/s +Step 1320/44000 | Loss: 4.0189 | LR: 5.00e-04 | 1272.7ms/step | 15.7 samples/s +Step 1330/44000 | Loss: 4.0253 | LR: 5.00e-04 | 1274.1ms/step | 15.7 samples/s +Step 1340/44000 | Loss: 3.9873 | LR: 5.00e-04 | 1275.0ms/step | 15.7 samples/s +Step 1350/44000 | Loss: 3.9907 | LR: 5.00e-04 | 1273.5ms/step | 15.7 samples/s +Step 1360/44000 | Loss: 3.9838 | LR: 5.00e-04 | 1273.4ms/step | 15.7 samples/s +Step 1370/44000 | Loss: 3.9626 | LR: 5.00e-04 | 1274.0ms/step | 15.7 samples/s +Step 1380/44000 | Loss: 3.9727 | LR: 5.00e-04 | 1273.4ms/step | 15.7 samples/s +Step 1390/44000 | Loss: 3.9746 | LR: 5.00e-04 | 1273.7ms/step | 15.7 samples/s +Step 1400/44000 | Loss: 3.9693 | LR: 5.00e-04 | 1274.6ms/step | 15.7 samples/s +Step 1410/44000 | Loss: 3.9617 | LR: 5.00e-04 | 1274.2ms/step | 15.7 samples/s +Step 1420/44000 | Loss: 3.9528 | LR: 5.00e-04 | 1273.6ms/step | 15.7 samples/s +Step 1430/44000 | Loss: 3.9690 | LR: 5.00e-04 | 1275.2ms/step | 15.7 samples/s +Step 1440/44000 | Loss: 3.9628 | LR: 5.00e-04 | 1274.2ms/step | 15.7 samples/s +Step 1450/44000 | Loss: 3.9493 | LR: 5.00e-04 | 1273.0ms/step | 15.7 samples/s +Step 1460/44000 | Loss: 3.9457 | LR: 5.00e-04 | 1273.4ms/step | 15.7 samples/s +Step 1470/44000 | Loss: 3.9565 | LR: 5.00e-04 | 1272.8ms/step | 15.7 samples/s +Step 1480/44000 | Loss: 3.9492 | LR: 5.00e-04 | 1273.3ms/step | 15.7 samples/s +Step 1490/44000 | Loss: 3.9452 | LR: 5.00e-04 | 1273.1ms/step | 15.7 samples/s +Step 1500/44000 | Loss: 3.9227 | LR: 5.00e-04 | 1274.5ms/step | 15.7 samples/s +Validation at step 1500: Loss = 3.9212, Perplexity = 50.46 +Step 1510/44000 | Loss: 3.9346 | LR: 5.00e-04 | 3421.8ms/step | 5.8 samples/s +Step 1520/44000 | Loss: 3.9353 | LR: 5.00e-04 | 1275.2ms/step | 15.7 samples/s +Step 1530/44000 | Loss: 3.9040 | LR: 5.00e-04 | 1274.4ms/step | 15.7 samples/s +Step 1540/44000 | Loss: 3.9131 | LR: 5.00e-04 | 1275.1ms/step | 15.7 samples/s +Step 1550/44000 | Loss: 3.8778 | LR: 5.00e-04 | 1275.6ms/step | 15.7 samples/s +Step 1560/44000 | Loss: 3.8705 | LR: 5.00e-04 | 1275.6ms/step | 15.7 samples/s +Step 1570/44000 | Loss: 3.8639 | LR: 5.00e-04 | 1276.4ms/step | 15.7 samples/s +Step 1580/44000 | Loss: 3.8858 | LR: 5.00e-04 | 1274.5ms/step | 15.7 samples/s +Step 1590/44000 | Loss: 3.8581 | LR: 5.00e-04 | 1275.6ms/step | 15.7 samples/s +Step 1600/44000 | Loss: 3.8646 | LR: 5.00e-04 | 1276.1ms/step | 15.7 samples/s +Step 1610/44000 | Loss: 3.8451 | LR: 5.00e-04 | 1275.6ms/step | 15.7 samples/s +Step 1620/44000 | Loss: 3.8278 | LR: 5.00e-04 | 1274.7ms/step | 15.7 samples/s +Step 1630/44000 | Loss: 3.8459 | LR: 4.99e-04 | 1273.0ms/step | 15.7 samples/s +Step 1640/44000 | Loss: 3.8496 | LR: 4.99e-04 | 1274.0ms/step | 15.7 samples/s +Step 1650/44000 | Loss: 3.8250 | LR: 4.99e-04 | 1274.4ms/step | 15.7 samples/s +Step 1660/44000 | Loss: 3.8288 | LR: 4.99e-04 | 1275.7ms/step | 15.7 samples/s +Step 1670/44000 | Loss: 3.8207 | LR: 4.99e-04 | 1274.6ms/step | 15.7 samples/s +Step 1680/44000 | Loss: 3.8255 | LR: 4.99e-04 | 1274.2ms/step | 15.7 samples/s +Step 1690/44000 | Loss: 3.8122 | LR: 4.99e-04 | 1274.5ms/step | 15.7 samples/s +Step 1700/44000 | Loss: 3.8506 | LR: 4.99e-04 | 1274.8ms/step | 15.7 samples/s +Step 1710/44000 | Loss: 3.7988 | LR: 4.99e-04 | 1274.1ms/step | 15.7 samples/s +Step 1720/44000 | Loss: 3.8089 | LR: 4.99e-04 | 1274.5ms/step | 15.7 samples/s +Step 1730/44000 | Loss: 3.8119 | LR: 4.99e-04 | 1275.7ms/step | 15.7 samples/s +Step 1740/44000 | Loss: 3.7763 | LR: 4.99e-04 | 1274.9ms/step | 15.7 samples/s +Step 1750/44000 | Loss: 3.7958 | LR: 4.99e-04 | 1276.2ms/step | 15.7 samples/s +Validation at step 1750: Loss = 3.8131, Perplexity = 45.29 +Step 1760/44000 | Loss: 3.7813 | LR: 4.99e-04 | 3421.5ms/step | 5.8 samples/s +Step 1770/44000 | Loss: 3.7791 | LR: 4.99e-04 | 1275.5ms/step | 15.7 samples/s +Step 1780/44000 | Loss: 3.8139 | LR: 4.99e-04 | 1275.7ms/step | 15.7 samples/s +Step 1790/44000 | Loss: 3.7783 | LR: 4.99e-04 | 1274.3ms/step | 15.7 samples/s +Step 1800/44000 | Loss: 3.7770 | LR: 4.99e-04 | 1274.3ms/step | 15.7 samples/s +Step 1810/44000 | Loss: 3.7937 | LR: 4.99e-04 | 1275.5ms/step | 15.7 samples/s +Step 1820/44000 | Loss: 3.7834 | LR: 4.99e-04 | 1275.8ms/step | 15.7 samples/s +Step 1830/44000 | Loss: 3.7720 | LR: 4.99e-04 | 1273.8ms/step | 15.7 samples/s +Step 1840/44000 | Loss: 3.7580 | LR: 4.99e-04 | 1273.3ms/step | 15.7 samples/s +Step 1850/44000 | Loss: 3.7833 | LR: 4.99e-04 | 1273.5ms/step | 15.7 samples/s +Step 1860/44000 | Loss: 3.7674 | LR: 4.99e-04 | 1274.0ms/step | 15.7 samples/s +Step 1870/44000 | Loss: 3.7712 | LR: 4.99e-04 | 1275.1ms/step | 15.7 samples/s +Step 1880/44000 | Loss: 3.7734 | LR: 4.99e-04 | 1274.7ms/step | 15.7 samples/s +Step 1890/44000 | Loss: 3.7666 | LR: 4.99e-04 | 1274.1ms/step | 15.7 samples/s +Step 1900/44000 | Loss: 3.7750 | LR: 4.99e-04 | 1272.5ms/step | 15.7 samples/s +Step 1910/44000 | Loss: 3.7460 | LR: 4.99e-04 | 1273.4ms/step | 15.7 samples/s +Step 1920/44000 | Loss: 3.7663 | LR: 4.99e-04 | 1273.7ms/step | 15.7 samples/s +Step 1930/44000 | Loss: 3.7545 | LR: 4.99e-04 | 1273.6ms/step | 15.7 samples/s +Step 1940/44000 | Loss: 3.7517 | LR: 4.99e-04 | 1273.4ms/step | 15.7 samples/s +Step 1950/44000 | Loss: 3.7493 | LR: 4.99e-04 | 1274.7ms/step | 15.7 samples/s +Step 1960/44000 | Loss: 3.7358 | LR: 4.99e-04 | 1273.7ms/step | 15.7 samples/s +Step 1970/44000 | Loss: 3.7414 | LR: 4.99e-04 | 1273.4ms/step | 15.7 samples/s +Step 1980/44000 | Loss: 3.7159 | LR: 4.99e-04 | 1272.9ms/step | 15.7 samples/s +Step 1990/44000 | Loss: 3.7000 | LR: 4.99e-04 | 1274.2ms/step | 15.7 samples/s +Step 2000/44000 | Loss: 3.7113 | LR: 4.99e-04 | 1272.9ms/step | 15.7 samples/s +Validation at step 2000: Loss = 3.7290, Perplexity = 41.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-2000 +Step 2010/44000 | Loss: 3.6895 | LR: 4.99e-04 | 3707.4ms/step | 5.4 samples/s +Step 2020/44000 | Loss: 3.7487 | LR: 4.99e-04 | 1268.7ms/step | 15.8 samples/s +Step 2030/44000 | Loss: 3.7241 | LR: 4.99e-04 | 1269.1ms/step | 15.8 samples/s +Step 2040/44000 | Loss: 3.7092 | LR: 4.99e-04 | 1270.1ms/step | 15.7 samples/s +Step 2050/44000 | Loss: 3.6981 | LR: 4.99e-04 | 1272.1ms/step | 15.7 samples/s +Step 2060/44000 | Loss: 3.6717 | LR: 4.99e-04 | 1272.4ms/step | 15.7 samples/s +Step 2070/44000 | Loss: 3.6927 | LR: 4.99e-04 | 1272.1ms/step | 15.7 samples/s +Step 2080/44000 | Loss: 3.6629 | LR: 4.99e-04 | 1271.4ms/step | 15.7 samples/s +Step 2090/44000 | Loss: 3.7005 | LR: 4.99e-04 | 1273.5ms/step | 15.7 samples/s +Step 2100/44000 | Loss: 3.6922 | LR: 4.99e-04 | 1272.9ms/step | 15.7 samples/s +Step 2110/44000 | Loss: 3.7092 | LR: 4.99e-04 | 1273.8ms/step | 15.7 samples/s +Step 2120/44000 | Loss: 3.6974 | LR: 4.99e-04 | 1273.8ms/step | 15.7 samples/s +Step 2130/44000 | Loss: 3.7064 | LR: 4.99e-04 | 1275.7ms/step | 15.7 samples/s +Step 2140/44000 | Loss: 3.6561 | LR: 4.99e-04 | 1274.0ms/step | 15.7 samples/s +Step 2150/44000 | Loss: 3.6706 | LR: 4.99e-04 | 1274.6ms/step | 15.7 samples/s +Step 2160/44000 | Loss: 3.6771 | LR: 4.99e-04 | 1275.1ms/step | 15.7 samples/s +Step 2170/44000 | Loss: 3.6787 | LR: 4.99e-04 | 1274.6ms/step | 15.7 samples/s +Step 2180/44000 | Loss: 3.6639 | LR: 4.99e-04 | 1274.7ms/step | 15.7 samples/s +Step 2190/44000 | Loss: 3.6737 | LR: 4.99e-04 | 1274.0ms/step | 15.7 samples/s +Step 2200/44000 | Loss: 3.6469 | LR: 4.99e-04 | 1273.4ms/step | 15.7 samples/s +Step 2210/44000 | Loss: 3.6494 | LR: 4.99e-04 | 1273.0ms/step | 15.7 samples/s +Step 2220/44000 | Loss: 3.6290 | LR: 4.99e-04 | 1273.0ms/step | 15.7 samples/s +Step 2230/44000 | Loss: 3.6734 | LR: 4.99e-04 | 1273.6ms/step | 15.7 samples/s +Step 2240/44000 | Loss: 3.6522 | LR: 4.99e-04 | 1272.9ms/step | 15.7 samples/s +Step 2250/44000 | Loss: 3.6537 | LR: 4.99e-04 | 1272.9ms/step | 15.7 samples/s +Validation at step 2250: Loss = 3.6654, Perplexity = 39.07 +Step 2260/44000 | Loss: 3.6491 | LR: 4.98e-04 | 3421.9ms/step | 5.8 samples/s +Step 2270/44000 | Loss: 3.6353 | LR: 4.98e-04 | 1274.5ms/step | 15.7 samples/s +Step 2280/44000 | Loss: 3.6652 | LR: 4.98e-04 | 1275.6ms/step | 15.7 samples/s +Step 2290/44000 | Loss: 3.6549 | LR: 4.98e-04 | 1275.3ms/step | 15.7 samples/s +Step 2300/44000 | Loss: 3.6341 | LR: 4.98e-04 | 1275.2ms/step | 15.7 samples/s +Step 2310/44000 | Loss: 3.6482 | LR: 4.98e-04 | 1275.2ms/step | 15.7 samples/s +Step 2320/44000 | Loss: 3.6348 | LR: 4.98e-04 | 1274.9ms/step | 15.7 samples/s +Step 2330/44000 | Loss: 3.6393 | LR: 4.98e-04 | 1275.7ms/step | 15.7 samples/s +Step 2340/44000 | Loss: 3.6164 | LR: 4.98e-04 | 1275.5ms/step | 15.7 samples/s +Step 2350/44000 | Loss: 3.6260 | LR: 4.98e-04 | 1273.8ms/step | 15.7 samples/s +Step 2360/44000 | Loss: 3.6094 | LR: 4.98e-04 | 1275.4ms/step | 15.7 samples/s +Step 2370/44000 | Loss: 3.6490 | LR: 4.98e-04 | 1275.8ms/step | 15.7 samples/s +Step 2380/44000 | Loss: 3.6260 | LR: 4.98e-04 | 1275.3ms/step | 15.7 samples/s +Step 2390/44000 | Loss: 3.5900 | LR: 4.98e-04 | 1274.4ms/step | 15.7 samples/s +Step 2400/44000 | Loss: 3.5862 | LR: 4.98e-04 | 1275.3ms/step | 15.7 samples/s +Step 2410/44000 | Loss: 3.6454 | LR: 4.98e-04 | 1275.0ms/step | 15.7 samples/s +Step 2420/44000 | Loss: 3.6516 | LR: 4.98e-04 | 1274.6ms/step | 15.7 samples/s +Step 2430/44000 | Loss: 3.6187 | LR: 4.98e-04 | 1273.9ms/step | 15.7 samples/s +Step 2440/44000 | Loss: 3.5954 | LR: 4.98e-04 | 1274.1ms/step | 15.7 samples/s +Step 2450/44000 | Loss: 3.6231 | LR: 4.98e-04 | 1274.9ms/step | 15.7 samples/s +Step 2460/44000 | Loss: 3.6163 | LR: 4.98e-04 | 1274.4ms/step | 15.7 samples/s +Step 2470/44000 | Loss: 3.5958 | LR: 4.98e-04 | 1274.6ms/step | 15.7 samples/s +Step 2480/44000 | Loss: 3.6267 | LR: 4.98e-04 | 1274.7ms/step | 15.7 samples/s +Step 2490/44000 | Loss: 3.5821 | LR: 4.98e-04 | 1274.6ms/step | 15.7 samples/s +Step 2500/44000 | Loss: 3.5997 | LR: 4.98e-04 | 1274.9ms/step | 15.7 samples/s +Validation at step 2500: Loss = 3.6144, Perplexity = 37.13 +Step 2510/44000 | Loss: 3.5821 | LR: 4.98e-04 | 3415.8ms/step | 5.9 samples/s +Step 2520/44000 | Loss: 3.6011 | LR: 4.98e-04 | 1273.2ms/step | 15.7 samples/s +Step 2530/44000 | Loss: 3.6002 | LR: 4.98e-04 | 1272.8ms/step | 15.7 samples/s +Step 2540/44000 | Loss: 3.6111 | LR: 4.98e-04 | 1273.5ms/step | 15.7 samples/s +Step 2550/44000 | Loss: 3.6028 | LR: 4.98e-04 | 1273.7ms/step | 15.7 samples/s +Step 2560/44000 | Loss: 3.6128 | LR: 4.98e-04 | 1273.5ms/step | 15.7 samples/s +Step 2570/44000 | Loss: 3.5840 | LR: 4.98e-04 | 1273.5ms/step | 15.7 samples/s +Step 2580/44000 | Loss: 3.6226 | LR: 4.98e-04 | 1275.2ms/step | 15.7 samples/s +Step 2590/44000 | Loss: 3.5789 | LR: 4.98e-04 | 1275.5ms/step | 15.7 samples/s +Step 2600/44000 | Loss: 3.5864 | LR: 4.98e-04 | 1275.6ms/step | 15.7 samples/s +Step 2610/44000 | Loss: 3.5803 | LR: 4.98e-04 | 1274.9ms/step | 15.7 samples/s +Step 2620/44000 | Loss: 3.5831 | LR: 4.98e-04 | 1275.5ms/step | 15.7 samples/s +Step 2630/44000 | Loss: 3.5725 | LR: 4.98e-04 | 1274.6ms/step | 15.7 samples/s +Step 2640/44000 | Loss: 3.5857 | LR: 4.98e-04 | 1274.4ms/step | 15.7 samples/s +Step 2650/44000 | Loss: 3.6017 | LR: 4.98e-04 | 1273.9ms/step | 15.7 samples/s +Step 2660/44000 | Loss: 3.5728 | LR: 4.98e-04 | 1274.0ms/step | 15.7 samples/s +Step 2670/44000 | Loss: 3.5648 | LR: 4.98e-04 | 1273.6ms/step | 15.7 samples/s +Step 2680/44000 | Loss: 3.5865 | LR: 4.98e-04 | 1275.0ms/step | 15.7 samples/s +Step 2690/44000 | Loss: 3.5752 | LR: 4.98e-04 | 1274.9ms/step | 15.7 samples/s +Step 2700/44000 | Loss: 3.6080 | LR: 4.97e-04 | 1274.6ms/step | 15.7 samples/s +Step 2710/44000 | Loss: 3.5728 | LR: 4.97e-04 | 1274.9ms/step | 15.7 samples/s +Step 2720/44000 | Loss: 3.5721 | LR: 4.97e-04 | 1274.9ms/step | 15.7 samples/s +Step 2730/44000 | Loss: 3.5670 | LR: 4.97e-04 | 1274.1ms/step | 15.7 samples/s +Step 2740/44000 | Loss: 3.5674 | LR: 4.97e-04 | 1273.3ms/step | 15.7 samples/s +Step 2750/44000 | Loss: 3.5572 | LR: 4.97e-04 | 1273.7ms/step | 15.7 samples/s +Validation at step 2750: Loss = 3.5719, Perplexity = 35.58 +Step 2760/44000 | Loss: 3.5299 | LR: 4.97e-04 | 3417.6ms/step | 5.9 samples/s +Step 2770/44000 | Loss: 3.5540 | LR: 4.97e-04 | 1274.2ms/step | 15.7 samples/s +Step 2780/44000 | Loss: 3.5318 | LR: 4.97e-04 | 1273.9ms/step | 15.7 samples/s +Step 2790/44000 | Loss: 3.5578 | LR: 4.97e-04 | 1273.2ms/step | 15.7 samples/s +Step 2800/44000 | Loss: 3.5516 | LR: 4.97e-04 | 1273.4ms/step | 15.7 samples/s +Step 2810/44000 | Loss: 3.5342 | LR: 4.97e-04 | 1273.8ms/step | 15.7 samples/s +Step 2820/44000 | Loss: 3.5450 | LR: 4.97e-04 | 1272.0ms/step | 15.7 samples/s +Step 2830/44000 | Loss: 3.5218 | LR: 4.97e-04 | 1272.9ms/step | 15.7 samples/s +Step 2840/44000 | Loss: 3.5314 | LR: 4.97e-04 | 1272.7ms/step | 15.7 samples/s +Step 2850/44000 | Loss: 3.5403 | LR: 4.97e-04 | 1273.2ms/step | 15.7 samples/s +Step 2860/44000 | Loss: 3.5342 | LR: 4.97e-04 | 1273.3ms/step | 15.7 samples/s +Step 2870/44000 | Loss: 3.5437 | LR: 4.97e-04 | 1273.3ms/step | 15.7 samples/s +Step 2880/44000 | Loss: 3.5057 | LR: 4.97e-04 | 1273.9ms/step | 15.7 samples/s +Step 2890/44000 | Loss: 3.5494 | LR: 4.97e-04 | 1273.7ms/step | 15.7 samples/s +Step 2900/44000 | Loss: 3.5471 | LR: 4.97e-04 | 1273.1ms/step | 15.7 samples/s +Step 2910/44000 | Loss: 3.5203 | LR: 4.97e-04 | 1272.6ms/step | 15.7 samples/s +Step 2920/44000 | Loss: 3.5110 | LR: 4.97e-04 | 1272.8ms/step | 15.7 samples/s +Step 2930/44000 | Loss: 3.5071 | LR: 4.97e-04 | 1274.9ms/step | 15.7 samples/s +Step 2940/44000 | Loss: 3.5267 | LR: 4.97e-04 | 1274.2ms/step | 15.7 samples/s +Step 2950/44000 | Loss: 3.5272 | LR: 4.97e-04 | 1273.6ms/step | 15.7 samples/s +Step 2960/44000 | Loss: 3.5415 | LR: 4.97e-04 | 1274.9ms/step | 15.7 samples/s +Step 2970/44000 | Loss: 3.5255 | LR: 4.97e-04 | 1275.9ms/step | 15.7 samples/s +Step 2980/44000 | Loss: 3.5267 | LR: 4.97e-04 | 1275.3ms/step | 15.7 samples/s +Step 2990/44000 | Loss: 3.5158 | LR: 4.97e-04 | 1274.4ms/step | 15.7 samples/s +Step 3000/44000 | Loss: 3.5443 | LR: 4.97e-04 | 1273.9ms/step | 15.7 samples/s +Validation at step 3000: Loss = 3.5321, Perplexity = 34.19 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-3000 +Step 3010/44000 | Loss: 3.5248 | LR: 4.97e-04 | 3713.4ms/step | 5.4 samples/s +Step 3020/44000 | Loss: 3.5028 | LR: 4.97e-04 | 1276.5ms/step | 15.7 samples/s +Step 3030/44000 | Loss: 3.5231 | LR: 4.97e-04 | 1278.0ms/step | 15.7 samples/s +Step 3040/44000 | Loss: 3.4848 | LR: 4.97e-04 | 1275.3ms/step | 15.7 samples/s +Step 3050/44000 | Loss: 3.5109 | LR: 4.97e-04 | 1273.2ms/step | 15.7 samples/s +Step 3060/44000 | Loss: 3.4832 | LR: 4.96e-04 | 1272.2ms/step | 15.7 samples/s +Step 3070/44000 | Loss: 3.5297 | LR: 4.96e-04 | 1273.7ms/step | 15.7 samples/s +Step 3080/44000 | Loss: 3.5483 | LR: 4.96e-04 | 1274.5ms/step | 15.7 samples/s +Step 3090/44000 | Loss: 3.5387 | LR: 4.96e-04 | 1274.2ms/step | 15.7 samples/s +Step 3100/44000 | Loss: 3.4977 | LR: 4.96e-04 | 1273.5ms/step | 15.7 samples/s +Step 3110/44000 | Loss: 3.4996 | LR: 4.96e-04 | 1274.5ms/step | 15.7 samples/s +Step 3120/44000 | Loss: 3.5185 | LR: 4.96e-04 | 1274.4ms/step | 15.7 samples/s +Step 3130/44000 | Loss: 3.4859 | LR: 4.96e-04 | 1274.5ms/step | 15.7 samples/s +Step 3140/44000 | Loss: 3.5039 | LR: 4.96e-04 | 1274.3ms/step | 15.7 samples/s +Step 3150/44000 | Loss: 3.4992 | LR: 4.96e-04 | 1274.1ms/step | 15.7 samples/s +Step 3160/44000 | Loss: 3.5185 | LR: 4.96e-04 | 1271.6ms/step | 15.7 samples/s +Step 3170/44000 | Loss: 3.4629 | LR: 4.96e-04 | 1268.5ms/step | 15.8 samples/s +Step 3180/44000 | Loss: 3.5052 | LR: 4.96e-04 | 1271.3ms/step | 15.7 samples/s +Step 3190/44000 | Loss: 3.4937 | LR: 4.96e-04 | 1273.1ms/step | 15.7 samples/s +Step 3200/44000 | Loss: 3.5191 | LR: 4.96e-04 | 1273.2ms/step | 15.7 samples/s +Step 3210/44000 | Loss: 3.4910 | LR: 4.96e-04 | 1273.5ms/step | 15.7 samples/s +Step 3220/44000 | Loss: 3.4935 | LR: 4.96e-04 | 1273.6ms/step | 15.7 samples/s +Step 3230/44000 | Loss: 3.4901 | LR: 4.96e-04 | 1274.3ms/step | 15.7 samples/s +Step 3240/44000 | Loss: 3.4891 | LR: 4.96e-04 | 1274.2ms/step | 15.7 samples/s +Step 3250/44000 | Loss: 3.4918 | LR: 4.96e-04 | 1274.4ms/step | 15.7 samples/s +Validation at step 3250: Loss = 3.4991, Perplexity = 33.09 +Step 3260/44000 | Loss: 3.5108 | LR: 4.96e-04 | 3421.4ms/step | 5.8 samples/s +Step 3270/44000 | Loss: 3.4761 | LR: 4.96e-04 | 1275.0ms/step | 15.7 samples/s +Step 3280/44000 | Loss: 3.4643 | LR: 4.96e-04 | 1274.7ms/step | 15.7 samples/s +Step 3290/44000 | Loss: 3.4849 | LR: 4.96e-04 | 1273.0ms/step | 15.7 samples/s +Step 3300/44000 | Loss: 3.5132 | LR: 4.96e-04 | 1275.0ms/step | 15.7 samples/s +Step 3310/44000 | Loss: 3.4850 | LR: 4.96e-04 | 1274.3ms/step | 15.7 samples/s +Step 3320/44000 | Loss: 3.5082 | LR: 4.96e-04 | 1274.2ms/step | 15.7 samples/s +Step 3330/44000 | Loss: 3.5106 | LR: 4.96e-04 | 1274.4ms/step | 15.7 samples/s +Step 3340/44000 | Loss: 3.4680 | LR: 4.96e-04 | 1274.1ms/step | 15.7 samples/s +Step 3350/44000 | Loss: 3.4828 | LR: 4.96e-04 | 1273.6ms/step | 15.7 samples/s +Step 3360/44000 | Loss: 3.4645 | LR: 4.96e-04 | 1274.3ms/step | 15.7 samples/s +Step 3370/44000 | Loss: 3.4377 | LR: 4.95e-04 | 1273.6ms/step | 15.7 samples/s +Step 3380/44000 | Loss: 3.4846 | LR: 4.95e-04 | 1274.8ms/step | 15.7 samples/s +Step 3390/44000 | Loss: 3.4638 | LR: 4.95e-04 | 1275.7ms/step | 15.7 samples/s +Step 3400/44000 | Loss: 3.4956 | LR: 4.95e-04 | 1274.9ms/step | 15.7 samples/s +Step 3410/44000 | Loss: 3.4712 | LR: 4.95e-04 | 1273.5ms/step | 15.7 samples/s +Step 3420/44000 | Loss: 3.4815 | LR: 4.95e-04 | 1272.9ms/step | 15.7 samples/s +Step 3430/44000 | Loss: 3.4731 | LR: 4.95e-04 | 1273.9ms/step | 15.7 samples/s +Step 3440/44000 | Loss: 3.4638 | LR: 4.95e-04 | 1274.7ms/step | 15.7 samples/s +Step 3450/44000 | Loss: 3.4488 | LR: 4.95e-04 | 1274.0ms/step | 15.7 samples/s +Step 3460/44000 | Loss: 3.4848 | LR: 4.95e-04 | 1274.0ms/step | 15.7 samples/s +Step 3470/44000 | Loss: 3.4576 | LR: 4.95e-04 | 1274.7ms/step | 15.7 samples/s +Step 3480/44000 | Loss: 3.5024 | LR: 4.95e-04 | 1274.0ms/step | 15.7 samples/s +Step 3490/44000 | Loss: 3.4367 | LR: 4.95e-04 | 1273.6ms/step | 15.7 samples/s +Step 3500/44000 | Loss: 3.4448 | LR: 4.95e-04 | 1275.0ms/step | 15.7 samples/s +Validation at step 3500: Loss = 3.4739, Perplexity = 32.26 +Step 3510/44000 | Loss: 3.4768 | LR: 4.95e-04 | 3419.6ms/step | 5.8 samples/s +Step 3520/44000 | Loss: 3.4628 | LR: 4.95e-04 | 1274.4ms/step | 15.7 samples/s +Step 3530/44000 | Loss: 3.4594 | LR: 4.95e-04 | 1274.0ms/step | 15.7 samples/s +Step 3540/44000 | Loss: 3.4692 | LR: 4.95e-04 | 1274.8ms/step | 15.7 samples/s +Step 3550/44000 | Loss: 3.4442 | LR: 4.95e-04 | 1273.7ms/step | 15.7 samples/s +Step 3560/44000 | Loss: 3.4311 | LR: 4.95e-04 | 1274.7ms/step | 15.7 samples/s +Step 3570/44000 | Loss: 3.4563 | LR: 4.95e-04 | 1274.9ms/step | 15.7 samples/s +Step 3580/44000 | Loss: 3.4320 | LR: 4.95e-04 | 1275.4ms/step | 15.7 samples/s +Step 3590/44000 | Loss: 3.4460 | LR: 4.95e-04 | 1275.5ms/step | 15.7 samples/s +Step 3600/44000 | Loss: 3.4348 | LR: 4.95e-04 | 1275.5ms/step | 15.7 samples/s +Step 3610/44000 | Loss: 3.4485 | LR: 4.95e-04 | 1275.2ms/step | 15.7 samples/s +Step 3620/44000 | Loss: 3.4476 | LR: 4.95e-04 | 1275.9ms/step | 15.7 samples/s +Step 3630/44000 | Loss: 3.4371 | LR: 4.95e-04 | 1273.8ms/step | 15.7 samples/s +Step 3640/44000 | Loss: 3.4503 | LR: 4.95e-04 | 1273.9ms/step | 15.7 samples/s +Step 3650/44000 | Loss: 3.4396 | LR: 4.94e-04 | 1274.5ms/step | 15.7 samples/s +Step 3660/44000 | Loss: 3.4366 | LR: 4.94e-04 | 1274.5ms/step | 15.7 samples/s +Step 3670/44000 | Loss: 3.4306 | LR: 4.94e-04 | 1276.2ms/step | 15.7 samples/s +Step 3680/44000 | Loss: 3.4426 | LR: 4.94e-04 | 1276.6ms/step | 15.7 samples/s +Step 3690/44000 | Loss: 3.4383 | LR: 4.94e-04 | 1276.8ms/step | 15.7 samples/s +Step 3700/44000 | Loss: 3.4515 | LR: 4.94e-04 | 1276.0ms/step | 15.7 samples/s +Step 3710/44000 | Loss: 3.4445 | LR: 4.94e-04 | 1275.2ms/step | 15.7 samples/s +Step 3720/44000 | Loss: 3.3982 | LR: 4.94e-04 | 1274.7ms/step | 15.7 samples/s +Step 3730/44000 | Loss: 3.4477 | LR: 4.94e-04 | 1275.3ms/step | 15.7 samples/s +Step 3740/44000 | Loss: 3.4437 | LR: 4.94e-04 | 1276.4ms/step | 15.7 samples/s +Step 3750/44000 | Loss: 3.4501 | LR: 4.94e-04 | 1275.6ms/step | 15.7 samples/s +Validation at step 3750: Loss = 3.4430, Perplexity = 31.28 +Step 3760/44000 | Loss: 3.4292 | LR: 4.94e-04 | 3418.6ms/step | 5.9 samples/s +Step 3770/44000 | Loss: 3.4250 | LR: 4.94e-04 | 1274.7ms/step | 15.7 samples/s +Step 3780/44000 | Loss: 3.4187 | LR: 4.94e-04 | 1274.6ms/step | 15.7 samples/s +Step 3790/44000 | Loss: 3.4396 | LR: 4.94e-04 | 1274.1ms/step | 15.7 samples/s +Step 3800/44000 | Loss: 3.4230 | LR: 4.94e-04 | 1273.4ms/step | 15.7 samples/s +Step 3810/44000 | Loss: 3.4278 | LR: 4.94e-04 | 1273.9ms/step | 15.7 samples/s +Step 3820/44000 | Loss: 3.4341 | LR: 4.94e-04 | 1275.7ms/step | 15.7 samples/s +Step 3830/44000 | Loss: 3.4251 | LR: 4.94e-04 | 1275.4ms/step | 15.7 samples/s +Step 3840/44000 | Loss: 3.4135 | LR: 4.94e-04 | 1275.3ms/step | 15.7 samples/s +Step 3850/44000 | Loss: 3.4204 | LR: 4.94e-04 | 1275.5ms/step | 15.7 samples/s +Step 3860/44000 | Loss: 3.4203 | LR: 4.94e-04 | 1275.4ms/step | 15.7 samples/s +Step 3870/44000 | Loss: 3.4267 | LR: 4.94e-04 | 1274.6ms/step | 15.7 samples/s +Step 3880/44000 | Loss: 3.3946 | LR: 4.94e-04 | 1274.5ms/step | 15.7 samples/s +Step 3890/44000 | Loss: 3.4520 | LR: 4.94e-04 | 1275.0ms/step | 15.7 samples/s +Step 3900/44000 | Loss: 3.4095 | LR: 4.93e-04 | 1274.9ms/step | 15.7 samples/s +Step 3910/44000 | Loss: 3.4014 | LR: 4.93e-04 | 1272.8ms/step | 15.7 samples/s +Step 3920/44000 | Loss: 3.4032 | LR: 4.93e-04 | 1274.1ms/step | 15.7 samples/s +Step 3930/44000 | Loss: 3.4142 | LR: 4.93e-04 | 1273.2ms/step | 15.7 samples/s +Step 3940/44000 | Loss: 3.4140 | LR: 4.93e-04 | 1273.5ms/step | 15.7 samples/s +Step 3950/44000 | Loss: 3.4404 | LR: 4.93e-04 | 1273.2ms/step | 15.7 samples/s +Step 3960/44000 | Loss: 3.4328 | LR: 4.93e-04 | 1275.5ms/step | 15.7 samples/s +Step 3970/44000 | Loss: 3.3859 | LR: 4.93e-04 | 1273.2ms/step | 15.7 samples/s +Step 3980/44000 | Loss: 3.3974 | LR: 4.93e-04 | 1270.0ms/step | 15.7 samples/s +Step 3990/44000 | Loss: 3.4237 | LR: 4.93e-04 | 1269.3ms/step | 15.8 samples/s +Step 4000/44000 | Loss: 3.4068 | LR: 4.93e-04 | 1270.8ms/step | 15.7 samples/s +Validation at step 4000: Loss = 3.4214, Perplexity = 30.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-4000 +Step 4010/44000 | Loss: 3.4587 | LR: 4.93e-04 | 3698.2ms/step | 5.4 samples/s +Step 4020/44000 | Loss: 3.4185 | LR: 4.93e-04 | 1272.0ms/step | 15.7 samples/s +Step 4030/44000 | Loss: 3.3948 | LR: 4.93e-04 | 1271.7ms/step | 15.7 samples/s +Step 4040/44000 | Loss: 3.4096 | LR: 4.93e-04 | 1273.0ms/step | 15.7 samples/s +Step 4050/44000 | Loss: 3.4442 | LR: 4.93e-04 | 1273.1ms/step | 15.7 samples/s +Step 4060/44000 | Loss: 3.3910 | LR: 4.93e-04 | 1273.4ms/step | 15.7 samples/s +Step 4070/44000 | Loss: 3.4107 | LR: 4.93e-04 | 1273.4ms/step | 15.7 samples/s +Step 4080/44000 | Loss: 3.4292 | LR: 4.93e-04 | 1274.7ms/step | 15.7 samples/s +Step 4090/44000 | Loss: 3.3809 | LR: 4.93e-04 | 1274.8ms/step | 15.7 samples/s +Step 4100/44000 | Loss: 3.4117 | LR: 4.93e-04 | 1272.9ms/step | 15.7 samples/s +Step 4110/44000 | Loss: 3.3833 | LR: 4.93e-04 | 1272.8ms/step | 15.7 samples/s +Step 4120/44000 | Loss: 3.4108 | LR: 4.93e-04 | 1274.2ms/step | 15.7 samples/s +Step 4130/44000 | Loss: 3.4117 | LR: 4.93e-04 | 1273.1ms/step | 15.7 samples/s +Step 4140/44000 | Loss: 3.3925 | LR: 4.92e-04 | 1274.2ms/step | 15.7 samples/s +Step 4150/44000 | Loss: 3.4384 | LR: 4.92e-04 | 1274.6ms/step | 15.7 samples/s +Step 4160/44000 | Loss: 3.4175 | LR: 4.92e-04 | 1273.9ms/step | 15.7 samples/s +Step 4170/44000 | Loss: 3.4011 | LR: 4.92e-04 | 1274.1ms/step | 15.7 samples/s +Step 4180/44000 | Loss: 3.3806 | LR: 4.92e-04 | 1273.7ms/step | 15.7 samples/s +Step 4190/44000 | Loss: 3.3910 | LR: 4.92e-04 | 1273.6ms/step | 15.7 samples/s +Step 4200/44000 | Loss: 3.3831 | LR: 4.92e-04 | 1274.8ms/step | 15.7 samples/s +Step 4210/44000 | Loss: 3.4170 | LR: 4.92e-04 | 1276.1ms/step | 15.7 samples/s +Step 4220/44000 | Loss: 3.3823 | LR: 4.92e-04 | 1273.7ms/step | 15.7 samples/s +Step 4230/44000 | Loss: 3.4049 | LR: 4.92e-04 | 1273.3ms/step | 15.7 samples/s +Step 4240/44000 | Loss: 3.4650 | LR: 4.92e-04 | 1274.3ms/step | 15.7 samples/s +Step 4250/44000 | Loss: 3.4345 | LR: 4.92e-04 | 1274.3ms/step | 15.7 samples/s +Validation at step 4250: Loss = 3.4042, Perplexity = 30.09 +Step 4260/44000 | Loss: 3.4082 | LR: 4.92e-04 | 3418.9ms/step | 5.8 samples/s +Step 4270/44000 | Loss: 3.3432 | LR: 4.92e-04 | 1275.3ms/step | 15.7 samples/s +Step 4280/44000 | Loss: 3.3673 | LR: 4.92e-04 | 1274.0ms/step | 15.7 samples/s +Step 4290/44000 | Loss: 3.3994 | LR: 4.92e-04 | 1274.7ms/step | 15.7 samples/s +Step 4300/44000 | Loss: 3.3894 | LR: 4.92e-04 | 1274.6ms/step | 15.7 samples/s +Step 4310/44000 | Loss: 3.3920 | LR: 4.92e-04 | 1273.8ms/step | 15.7 samples/s +Step 4320/44000 | Loss: 3.3963 | LR: 4.92e-04 | 1273.9ms/step | 15.7 samples/s +Step 4330/44000 | Loss: 3.3827 | LR: 4.92e-04 | 1274.0ms/step | 15.7 samples/s +Step 4340/44000 | Loss: 3.3965 | LR: 4.92e-04 | 1273.2ms/step | 15.7 samples/s +Step 4350/44000 | Loss: 3.4041 | LR: 4.92e-04 | 1274.4ms/step | 15.7 samples/s +Step 4360/44000 | Loss: 3.3534 | LR: 4.91e-04 | 1273.6ms/step | 15.7 samples/s +Step 4370/44000 | Loss: 3.3717 | LR: 4.91e-04 | 1272.5ms/step | 15.7 samples/s +Step 4380/44000 | Loss: 3.4149 | LR: 4.91e-04 | 1272.8ms/step | 15.7 samples/s +Step 4390/44000 | Loss: 3.4046 | LR: 4.91e-04 | 1273.2ms/step | 15.7 samples/s +Step 4400/44000 | Loss: 3.4038 | LR: 4.91e-04 | 1273.2ms/step | 15.7 samples/s +Step 4410/44000 | Loss: 3.3759 | LR: 4.91e-04 | 1273.5ms/step | 15.7 samples/s +Step 4420/44000 | Loss: 3.3832 | LR: 4.91e-04 | 1274.9ms/step | 15.7 samples/s +Step 4430/44000 | Loss: 3.3847 | LR: 4.91e-04 | 1273.4ms/step | 15.7 samples/s +Step 4440/44000 | Loss: 3.3959 | LR: 4.91e-04 | 1275.0ms/step | 15.7 samples/s +Step 4450/44000 | Loss: 3.3859 | LR: 4.91e-04 | 1273.6ms/step | 15.7 samples/s +Step 4460/44000 | Loss: 3.3847 | LR: 4.91e-04 | 1273.1ms/step | 15.7 samples/s +Step 4470/44000 | Loss: 3.3477 | LR: 4.91e-04 | 1273.7ms/step | 15.7 samples/s +Step 4480/44000 | Loss: 3.3816 | LR: 4.91e-04 | 1272.8ms/step | 15.7 samples/s +Step 4490/44000 | Loss: 3.3874 | LR: 4.91e-04 | 1272.2ms/step | 15.7 samples/s +Step 4500/44000 | Loss: 3.3805 | LR: 4.91e-04 | 1273.3ms/step | 15.7 samples/s +Validation at step 4500: Loss = 3.3835, Perplexity = 29.47 +Step 4510/44000 | Loss: 3.3717 | LR: 4.91e-04 | 3418.7ms/step | 5.9 samples/s +Step 4520/44000 | Loss: 3.3949 | LR: 4.91e-04 | 1274.5ms/step | 15.7 samples/s +Step 4530/44000 | Loss: 3.3932 | LR: 4.91e-04 | 1276.1ms/step | 15.7 samples/s +Step 4540/44000 | Loss: 3.3540 | LR: 4.91e-04 | 1275.7ms/step | 15.7 samples/s +Step 4550/44000 | Loss: 3.3475 | LR: 4.91e-04 | 1274.3ms/step | 15.7 samples/s +Step 4560/44000 | Loss: 3.3540 | LR: 4.90e-04 | 1273.8ms/step | 15.7 samples/s +Step 4570/44000 | Loss: 3.3738 | LR: 4.90e-04 | 1274.3ms/step | 15.7 samples/s +Step 4580/44000 | Loss: 3.3314 | LR: 4.90e-04 | 1275.8ms/step | 15.7 samples/s +Step 4590/44000 | Loss: 3.3327 | LR: 4.90e-04 | 1274.9ms/step | 15.7 samples/s +Step 4600/44000 | Loss: 3.3539 | LR: 4.90e-04 | 1274.9ms/step | 15.7 samples/s +Step 4610/44000 | Loss: 3.3612 | LR: 4.90e-04 | 1274.4ms/step | 15.7 samples/s +Step 4620/44000 | Loss: 3.3517 | LR: 4.90e-04 | 1273.3ms/step | 15.7 samples/s +Step 4630/44000 | Loss: 3.3620 | LR: 4.90e-04 | 1273.4ms/step | 15.7 samples/s +Step 4640/44000 | Loss: 3.3728 | LR: 4.90e-04 | 1273.4ms/step | 15.7 samples/s +Step 4650/44000 | Loss: 3.3627 | LR: 4.90e-04 | 1274.6ms/step | 15.7 samples/s +Step 4660/44000 | Loss: 3.3405 | LR: 4.90e-04 | 1273.3ms/step | 15.7 samples/s +Step 4670/44000 | Loss: 3.3329 | LR: 4.90e-04 | 1273.7ms/step | 15.7 samples/s +Step 4680/44000 | Loss: 3.3571 | LR: 4.90e-04 | 1274.1ms/step | 15.7 samples/s +Step 4690/44000 | Loss: 3.3509 | LR: 4.90e-04 | 1274.3ms/step | 15.7 samples/s +Step 4700/44000 | Loss: 3.3842 | LR: 4.90e-04 | 1274.7ms/step | 15.7 samples/s +Step 4710/44000 | Loss: 3.3607 | LR: 4.90e-04 | 1275.2ms/step | 15.7 samples/s +Step 4720/44000 | Loss: 3.3395 | LR: 4.90e-04 | 1275.5ms/step | 15.7 samples/s +Step 4730/44000 | Loss: 3.3582 | LR: 4.90e-04 | 1275.0ms/step | 15.7 samples/s +Step 4740/44000 | Loss: 3.3496 | LR: 4.90e-04 | 1275.4ms/step | 15.7 samples/s +Step 4750/44000 | Loss: 3.3599 | LR: 4.90e-04 | 1274.4ms/step | 15.7 samples/s +Validation at step 4750: Loss = 3.3649, Perplexity = 28.93 +Step 4760/44000 | Loss: 3.3385 | LR: 4.89e-04 | 3422.1ms/step | 5.8 samples/s +Step 4770/44000 | Loss: 3.3623 | LR: 4.89e-04 | 1274.8ms/step | 15.7 samples/s +Step 4780/44000 | Loss: 3.3576 | LR: 4.89e-04 | 1274.7ms/step | 15.7 samples/s +Step 4790/44000 | Loss: 3.3288 | LR: 4.89e-04 | 1274.8ms/step | 15.7 samples/s +Step 4800/44000 | Loss: 3.3541 | LR: 4.89e-04 | 1274.2ms/step | 15.7 samples/s +Step 4810/44000 | Loss: 3.3533 | LR: 4.89e-04 | 1274.7ms/step | 15.7 samples/s +Step 4820/44000 | Loss: 3.3567 | LR: 4.89e-04 | 1274.3ms/step | 15.7 samples/s +Step 4830/44000 | Loss: 3.3440 | LR: 4.89e-04 | 1274.7ms/step | 15.7 samples/s +Step 4840/44000 | Loss: 3.3579 | LR: 4.89e-04 | 1274.6ms/step | 15.7 samples/s +Step 4850/44000 | Loss: 3.3592 | LR: 4.89e-04 | 1273.9ms/step | 15.7 samples/s +Step 4860/44000 | Loss: 3.3660 | LR: 4.89e-04 | 1274.6ms/step | 15.7 samples/s +Step 4870/44000 | Loss: 3.3301 | LR: 4.89e-04 | 1275.3ms/step | 15.7 samples/s +Step 4880/44000 | Loss: 3.3294 | LR: 4.89e-04 | 1275.2ms/step | 15.7 samples/s +Step 4890/44000 | Loss: 3.3210 | LR: 4.89e-04 | 1275.2ms/step | 15.7 samples/s +Step 4900/44000 | Loss: 3.3034 | LR: 4.89e-04 | 1274.3ms/step | 15.7 samples/s +Step 4910/44000 | Loss: 3.3416 | LR: 4.89e-04 | 1275.4ms/step | 15.7 samples/s +Step 4920/44000 | Loss: 3.3417 | LR: 4.89e-04 | 1274.5ms/step | 15.7 samples/s +Step 4930/44000 | Loss: 3.3252 | LR: 4.89e-04 | 1274.3ms/step | 15.7 samples/s +Step 4940/44000 | Loss: 3.3345 | LR: 4.89e-04 | 1274.6ms/step | 15.7 samples/s +Step 4950/44000 | Loss: 3.3361 | LR: 4.88e-04 | 1275.1ms/step | 15.7 samples/s +Step 4960/44000 | Loss: 3.3267 | LR: 4.88e-04 | 1274.0ms/step | 15.7 samples/s +Step 4970/44000 | Loss: 3.3179 | LR: 4.88e-04 | 1275.0ms/step | 15.7 samples/s +Step 4980/44000 | Loss: 3.3156 | LR: 4.88e-04 | 1274.0ms/step | 15.7 samples/s +Step 4990/44000 | Loss: 3.3411 | LR: 4.88e-04 | 1275.0ms/step | 15.7 samples/s +Step 5000/44000 | Loss: 3.3377 | LR: 4.88e-04 | 1274.4ms/step | 15.7 samples/s +Validation at step 5000: Loss = 3.3476, Perplexity = 28.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-5000 +Step 5010/44000 | Loss: 3.3463 | LR: 4.88e-04 | 3705.6ms/step | 5.4 samples/s +Step 5020/44000 | Loss: 3.3255 | LR: 4.88e-04 | 1273.2ms/step | 15.7 samples/s +Step 5030/44000 | Loss: 3.3350 | LR: 4.88e-04 | 1273.7ms/step | 15.7 samples/s +Step 5040/44000 | Loss: 3.3435 | LR: 4.88e-04 | 1274.8ms/step | 15.7 samples/s +Step 5050/44000 | Loss: 3.3186 | LR: 4.88e-04 | 1273.3ms/step | 15.7 samples/s +Step 5060/44000 | Loss: 3.3486 | LR: 4.88e-04 | 1276.4ms/step | 15.7 samples/s +Step 5070/44000 | Loss: 3.3527 | LR: 4.88e-04 | 1275.3ms/step | 15.7 samples/s +Step 5080/44000 | Loss: 3.3446 | LR: 4.88e-04 | 1275.4ms/step | 15.7 samples/s +Step 5090/44000 | Loss: 3.3163 | LR: 4.88e-04 | 1275.6ms/step | 15.7 samples/s +Step 5100/44000 | Loss: 3.3433 | LR: 4.88e-04 | 1275.8ms/step | 15.7 samples/s +Step 5110/44000 | Loss: 3.3623 | LR: 4.88e-04 | 1275.0ms/step | 15.7 samples/s +Step 5120/44000 | Loss: 3.3588 | LR: 4.88e-04 | 1274.2ms/step | 15.7 samples/s +Step 5130/44000 | Loss: 3.3404 | LR: 4.87e-04 | 1274.7ms/step | 15.7 samples/s +Step 5140/44000 | Loss: 3.3152 | LR: 4.87e-04 | 1275.0ms/step | 15.7 samples/s +Step 5150/44000 | Loss: 3.3604 | LR: 4.87e-04 | 1276.3ms/step | 15.7 samples/s +Step 5160/44000 | Loss: 3.3588 | LR: 4.87e-04 | 1276.3ms/step | 15.7 samples/s +Step 5170/44000 | Loss: 3.3491 | LR: 4.87e-04 | 1276.5ms/step | 15.7 samples/s +Step 5180/44000 | Loss: 3.3065 | LR: 4.87e-04 | 1276.0ms/step | 15.7 samples/s +Step 5190/44000 | Loss: 3.3323 | LR: 4.87e-04 | 1275.9ms/step | 15.7 samples/s +Step 5200/44000 | Loss: 3.3005 | LR: 4.87e-04 | 1274.0ms/step | 15.7 samples/s +Step 5210/44000 | Loss: 3.3359 | LR: 4.87e-04 | 1274.7ms/step | 15.7 samples/s +Step 5220/44000 | Loss: 3.3309 | LR: 4.87e-04 | 1273.8ms/step | 15.7 samples/s +Step 5230/44000 | Loss: 3.2839 | LR: 4.87e-04 | 1274.4ms/step | 15.7 samples/s +Step 5240/44000 | Loss: 3.3135 | LR: 4.87e-04 | 1275.2ms/step | 15.7 samples/s +Step 5250/44000 | Loss: 3.3124 | LR: 4.87e-04 | 1274.8ms/step | 15.7 samples/s +Validation at step 5250: Loss = 3.3340, Perplexity = 28.05 +Step 5260/44000 | Loss: 3.3360 | LR: 4.87e-04 | 3423.7ms/step | 5.8 samples/s +Step 5270/44000 | Loss: 3.3046 | LR: 4.87e-04 | 1275.4ms/step | 15.7 samples/s +Step 5280/44000 | Loss: 3.3322 | LR: 4.87e-04 | 1276.1ms/step | 15.7 samples/s +Step 5290/44000 | Loss: 3.3182 | LR: 4.87e-04 | 1275.2ms/step | 15.7 samples/s +Step 5300/44000 | Loss: 3.2697 | LR: 4.86e-04 | 1275.2ms/step | 15.7 samples/s +Step 5310/44000 | Loss: 3.3120 | LR: 4.86e-04 | 1275.2ms/step | 15.7 samples/s +Step 5320/44000 | Loss: 3.3248 | LR: 4.86e-04 | 1274.6ms/step | 15.7 samples/s +Step 5330/44000 | Loss: 3.3201 | LR: 4.86e-04 | 1273.4ms/step | 15.7 samples/s +Step 5340/44000 | Loss: 3.3310 | LR: 4.86e-04 | 1273.8ms/step | 15.7 samples/s +Step 5350/44000 | Loss: 3.2981 | LR: 4.86e-04 | 1273.8ms/step | 15.7 samples/s +Step 5360/44000 | Loss: 3.3229 | LR: 4.86e-04 | 1276.0ms/step | 15.7 samples/s +Step 5370/44000 | Loss: 3.3211 | LR: 4.86e-04 | 1275.8ms/step | 15.7 samples/s +Step 5380/44000 | Loss: 3.3087 | LR: 4.86e-04 | 1273.8ms/step | 15.7 samples/s +Step 5390/44000 | Loss: 3.3321 | LR: 4.86e-04 | 1273.5ms/step | 15.7 samples/s +Step 5400/44000 | Loss: 3.3362 | LR: 4.86e-04 | 1273.6ms/step | 15.7 samples/s +Step 5410/44000 | Loss: 3.3230 | LR: 4.86e-04 | 1273.9ms/step | 15.7 samples/s +Step 5420/44000 | Loss: 3.3193 | LR: 4.86e-04 | 1273.2ms/step | 15.7 samples/s +Step 5430/44000 | Loss: 3.3026 | LR: 4.86e-04 | 1273.9ms/step | 15.7 samples/s +Step 5440/44000 | Loss: 3.3135 | LR: 4.86e-04 | 1273.3ms/step | 15.7 samples/s +Step 5450/44000 | Loss: 3.3261 | LR: 4.86e-04 | 1272.8ms/step | 15.7 samples/s +Step 5460/44000 | Loss: 3.3426 | LR: 4.86e-04 | 1273.2ms/step | 15.7 samples/s +Step 5470/44000 | Loss: 3.3212 | LR: 4.85e-04 | 1273.3ms/step | 15.7 samples/s +Step 5480/44000 | Loss: 3.3028 | LR: 4.85e-04 | 1272.7ms/step | 15.7 samples/s +Step 5490/44000 | Loss: 3.2904 | LR: 4.85e-04 | 1272.5ms/step | 15.7 samples/s +Step 5500/44000 | Loss: 3.3131 | LR: 4.85e-04 | 1273.4ms/step | 15.7 samples/s +Validation at step 5500: Loss = 3.3194, Perplexity = 27.64 +Step 5510/44000 | Loss: 3.3090 | LR: 4.85e-04 | 3427.6ms/step | 5.8 samples/s +Step 5520/44000 | Loss: 3.3120 | LR: 4.85e-04 | 1275.2ms/step | 15.7 samples/s +Step 5530/44000 | Loss: 3.2812 | LR: 4.85e-04 | 1275.0ms/step | 15.7 samples/s +Step 5540/44000 | Loss: 3.3158 | LR: 4.85e-04 | 1274.9ms/step | 15.7 samples/s +Step 5550/44000 | Loss: 3.2972 | LR: 4.85e-04 | 1274.3ms/step | 15.7 samples/s +Step 5560/44000 | Loss: 3.3064 | LR: 4.85e-04 | 1274.1ms/step | 15.7 samples/s +Step 5570/44000 | Loss: 3.3016 | LR: 4.85e-04 | 1274.0ms/step | 15.7 samples/s +Step 5580/44000 | Loss: 3.2890 | LR: 4.85e-04 | 1275.2ms/step | 15.7 samples/s +Step 5590/44000 | Loss: 3.2831 | LR: 4.85e-04 | 1275.6ms/step | 15.7 samples/s +Step 5600/44000 | Loss: 3.2841 | LR: 4.85e-04 | 1273.7ms/step | 15.7 samples/s +Step 5610/44000 | Loss: 3.3145 | LR: 4.85e-04 | 1274.7ms/step | 15.7 samples/s +Step 5620/44000 | Loss: 3.2979 | LR: 4.85e-04 | 1274.6ms/step | 15.7 samples/s +Step 5630/44000 | Loss: 3.3185 | LR: 4.84e-04 | 1274.8ms/step | 15.7 samples/s +Step 5640/44000 | Loss: 3.2895 | LR: 4.84e-04 | 1275.6ms/step | 15.7 samples/s +Step 5650/44000 | Loss: 3.2875 | LR: 4.84e-04 | 1274.9ms/step | 15.7 samples/s +Step 5660/44000 | Loss: 3.2743 | LR: 4.84e-04 | 1274.1ms/step | 15.7 samples/s +Step 5670/44000 | Loss: 3.2933 | LR: 4.84e-04 | 1273.8ms/step | 15.7 samples/s +Step 5680/44000 | Loss: 3.2751 | LR: 4.84e-04 | 1273.6ms/step | 15.7 samples/s +Step 5690/44000 | Loss: 3.3228 | LR: 4.84e-04 | 1274.7ms/step | 15.7 samples/s +Step 5700/44000 | Loss: 3.3129 | LR: 4.84e-04 | 1275.6ms/step | 15.7 samples/s +Step 5710/44000 | Loss: 3.2608 | LR: 4.84e-04 | 1274.9ms/step | 15.7 samples/s +Step 5720/44000 | Loss: 3.2602 | LR: 4.84e-04 | 1275.2ms/step | 15.7 samples/s +Step 5730/44000 | Loss: 3.2800 | LR: 4.84e-04 | 1275.8ms/step | 15.7 samples/s +Step 5740/44000 | Loss: 3.2827 | LR: 4.84e-04 | 1274.8ms/step | 15.7 samples/s +Step 5750/44000 | Loss: 3.2860 | LR: 4.84e-04 | 1274.9ms/step | 15.7 samples/s +Validation at step 5750: Loss = 3.3061, Perplexity = 27.28 +Step 5760/44000 | Loss: 3.3092 | LR: 4.84e-04 | 3416.9ms/step | 5.9 samples/s +Step 5770/44000 | Loss: 3.3148 | LR: 4.84e-04 | 1273.7ms/step | 15.7 samples/s +Step 5780/44000 | Loss: 3.2928 | LR: 4.83e-04 | 1274.6ms/step | 15.7 samples/s +Step 5790/44000 | Loss: 3.2770 | LR: 4.83e-04 | 1275.1ms/step | 15.7 samples/s +Step 5800/44000 | Loss: 3.3020 | LR: 4.83e-04 | 1274.8ms/step | 15.7 samples/s +Step 5810/44000 | Loss: 3.3157 | LR: 4.83e-04 | 1274.6ms/step | 15.7 samples/s +Step 5820/44000 | Loss: 3.2978 | LR: 4.83e-04 | 1274.2ms/step | 15.7 samples/s +Step 5830/44000 | Loss: 3.2941 | LR: 4.83e-04 | 1275.1ms/step | 15.7 samples/s +Step 5840/44000 | Loss: 3.2880 | LR: 4.83e-04 | 1274.5ms/step | 15.7 samples/s +Step 5850/44000 | Loss: 3.2693 | LR: 4.83e-04 | 1274.7ms/step | 15.7 samples/s +Step 5860/44000 | Loss: 3.2974 | LR: 4.83e-04 | 1275.6ms/step | 15.7 samples/s +Step 5870/44000 | Loss: 3.3039 | LR: 4.83e-04 | 1274.9ms/step | 15.7 samples/s +Step 5880/44000 | Loss: 3.2822 | LR: 4.83e-04 | 1275.6ms/step | 15.7 samples/s +Step 5890/44000 | Loss: 3.2987 | LR: 4.83e-04 | 1275.3ms/step | 15.7 samples/s +Step 5900/44000 | Loss: 3.2868 | LR: 4.83e-04 | 1275.1ms/step | 15.7 samples/s +Step 5910/44000 | Loss: 3.2941 | LR: 4.83e-04 | 1275.0ms/step | 15.7 samples/s +Step 5920/44000 | Loss: 3.3024 | LR: 4.83e-04 | 1273.9ms/step | 15.7 samples/s +Step 5930/44000 | Loss: 3.2543 | LR: 4.83e-04 | 1274.8ms/step | 15.7 samples/s +Step 5940/44000 | Loss: 3.3042 | LR: 4.82e-04 | 1273.2ms/step | 15.7 samples/s +Step 5950/44000 | Loss: 3.2665 | LR: 4.82e-04 | 1274.3ms/step | 15.7 samples/s +Step 5960/44000 | Loss: 3.2591 | LR: 4.82e-04 | 1273.0ms/step | 15.7 samples/s +Step 5970/44000 | Loss: 3.2643 | LR: 4.82e-04 | 1273.3ms/step | 15.7 samples/s +Step 5980/44000 | Loss: 3.2586 | LR: 4.82e-04 | 1274.9ms/step | 15.7 samples/s +Step 5990/44000 | Loss: 3.2583 | LR: 4.82e-04 | 1275.1ms/step | 15.7 samples/s +Step 6000/44000 | Loss: 3.2842 | LR: 4.82e-04 | 1274.2ms/step | 15.7 samples/s +Validation at step 6000: Loss = 3.2927, Perplexity = 26.92 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-6000 +Step 6010/44000 | Loss: 3.3052 | LR: 4.82e-04 | 3704.9ms/step | 5.4 samples/s +Step 6020/44000 | Loss: 3.2683 | LR: 4.82e-04 | 1271.6ms/step | 15.7 samples/s +Step 6030/44000 | Loss: 3.2725 | LR: 4.82e-04 | 1271.2ms/step | 15.7 samples/s +Step 6040/44000 | Loss: 3.2759 | LR: 4.82e-04 | 1273.4ms/step | 15.7 samples/s +Step 6050/44000 | Loss: 3.2798 | LR: 4.82e-04 | 1273.5ms/step | 15.7 samples/s +Step 6060/44000 | Loss: 3.3038 | LR: 4.82e-04 | 1274.6ms/step | 15.7 samples/s +Step 6070/44000 | Loss: 3.2835 | LR: 4.82e-04 | 1273.3ms/step | 15.7 samples/s +Step 6080/44000 | Loss: 3.2762 | LR: 4.81e-04 | 1273.5ms/step | 15.7 samples/s +Step 6090/44000 | Loss: 3.3038 | LR: 4.81e-04 | 1274.3ms/step | 15.7 samples/s +Step 6100/44000 | Loss: 3.2576 | LR: 4.81e-04 | 1274.0ms/step | 15.7 samples/s +Step 6110/44000 | Loss: 3.2731 | LR: 4.81e-04 | 1273.4ms/step | 15.7 samples/s +Step 6120/44000 | Loss: 3.2540 | LR: 4.81e-04 | 1275.7ms/step | 15.7 samples/s +Step 6130/44000 | Loss: 3.2720 | LR: 4.81e-04 | 1273.8ms/step | 15.7 samples/s +Step 6140/44000 | Loss: 3.2581 | LR: 4.81e-04 | 1274.8ms/step | 15.7 samples/s +Step 6150/44000 | Loss: 3.2666 | LR: 4.81e-04 | 1273.6ms/step | 15.7 samples/s +Step 6160/44000 | Loss: 3.2642 | LR: 4.81e-04 | 1274.6ms/step | 15.7 samples/s +Step 6170/44000 | Loss: 3.2807 | LR: 4.81e-04 | 1275.8ms/step | 15.7 samples/s +Step 6180/44000 | Loss: 3.2772 | LR: 4.81e-04 | 1275.5ms/step | 15.7 samples/s +Step 6190/44000 | Loss: 3.2613 | LR: 4.81e-04 | 1274.9ms/step | 15.7 samples/s +Step 6200/44000 | Loss: 3.2759 | LR: 4.81e-04 | 1274.7ms/step | 15.7 samples/s +Step 6210/44000 | Loss: 3.2725 | LR: 4.81e-04 | 1274.8ms/step | 15.7 samples/s +Step 6220/44000 | Loss: 3.3054 | LR: 4.81e-04 | 1275.8ms/step | 15.7 samples/s +Step 6230/44000 | Loss: 3.2722 | LR: 4.80e-04 | 1272.9ms/step | 15.7 samples/s +Step 6240/44000 | Loss: 3.2904 | LR: 4.80e-04 | 1273.5ms/step | 15.7 samples/s +Step 6250/44000 | Loss: 3.2852 | LR: 4.80e-04 | 1275.5ms/step | 15.7 samples/s +Validation at step 6250: Loss = 3.2891, Perplexity = 26.82 +Step 6260/44000 | Loss: 3.2792 | LR: 4.80e-04 | 3415.8ms/step | 5.9 samples/s +Step 6270/44000 | Loss: 3.3030 | LR: 4.80e-04 | 1273.5ms/step | 15.7 samples/s +Step 6280/44000 | Loss: 3.2562 | LR: 4.80e-04 | 1274.2ms/step | 15.7 samples/s +Step 6290/44000 | Loss: 3.2857 | LR: 4.80e-04 | 1274.7ms/step | 15.7 samples/s +Step 6300/44000 | Loss: 3.2626 | LR: 4.80e-04 | 1274.6ms/step | 15.7 samples/s +Step 6310/44000 | Loss: 3.2672 | LR: 4.80e-04 | 1273.4ms/step | 15.7 samples/s +Step 6320/44000 | Loss: 3.2605 | LR: 4.80e-04 | 1273.9ms/step | 15.7 samples/s +Step 6330/44000 | Loss: 3.2687 | LR: 4.80e-04 | 1274.5ms/step | 15.7 samples/s +Step 6340/44000 | Loss: 3.2789 | LR: 4.80e-04 | 1273.9ms/step | 15.7 samples/s +Step 6350/44000 | Loss: 3.2389 | LR: 4.80e-04 | 1274.7ms/step | 15.7 samples/s +Step 6360/44000 | Loss: 3.2938 | LR: 4.80e-04 | 1273.7ms/step | 15.7 samples/s +Step 6370/44000 | Loss: 3.2624 | LR: 4.79e-04 | 1273.2ms/step | 15.7 samples/s +Step 6380/44000 | Loss: 3.2509 | LR: 4.79e-04 | 1273.6ms/step | 15.7 samples/s +Step 6390/44000 | Loss: 3.2421 | LR: 4.79e-04 | 1273.5ms/step | 15.7 samples/s +Step 6400/44000 | Loss: 3.2749 | LR: 4.79e-04 | 1273.6ms/step | 15.7 samples/s +Step 6410/44000 | Loss: 3.2454 | LR: 4.79e-04 | 1272.9ms/step | 15.7 samples/s +Step 6420/44000 | Loss: 3.2753 | LR: 4.79e-04 | 1275.9ms/step | 15.7 samples/s +Step 6430/44000 | Loss: 3.2384 | LR: 4.79e-04 | 1275.2ms/step | 15.7 samples/s +Step 6440/44000 | Loss: 3.2541 | LR: 4.79e-04 | 1274.2ms/step | 15.7 samples/s +Step 6450/44000 | Loss: 3.2507 | LR: 4.79e-04 | 1275.4ms/step | 15.7 samples/s +Step 6460/44000 | Loss: 3.2617 | LR: 4.79e-04 | 1274.0ms/step | 15.7 samples/s +Step 6470/44000 | Loss: 3.2415 | LR: 4.79e-04 | 1273.0ms/step | 15.7 samples/s +Step 6480/44000 | Loss: 3.2552 | LR: 4.79e-04 | 1273.7ms/step | 15.7 samples/s +Step 6490/44000 | Loss: 3.2480 | LR: 4.79e-04 | 1273.1ms/step | 15.7 samples/s +Step 6500/44000 | Loss: 3.2453 | LR: 4.79e-04 | 1273.4ms/step | 15.7 samples/s +Validation at step 6500: Loss = 3.2730, Perplexity = 26.39 +Step 6510/44000 | Loss: 3.2717 | LR: 4.78e-04 | 3421.9ms/step | 5.8 samples/s +Step 6520/44000 | Loss: 3.2776 | LR: 4.78e-04 | 1274.8ms/step | 15.7 samples/s +Step 6530/44000 | Loss: 3.2737 | LR: 4.78e-04 | 1275.9ms/step | 15.7 samples/s +Step 6540/44000 | Loss: 3.2704 | LR: 4.78e-04 | 1275.7ms/step | 15.7 samples/s +Step 6550/44000 | Loss: 3.2712 | LR: 4.78e-04 | 1274.2ms/step | 15.7 samples/s +Step 6560/44000 | Loss: 3.2520 | LR: 4.78e-04 | 1273.5ms/step | 15.7 samples/s +Step 6570/44000 | Loss: 3.2540 | LR: 4.78e-04 | 1273.3ms/step | 15.7 samples/s +Step 6580/44000 | Loss: 3.2927 | LR: 4.78e-04 | 1274.3ms/step | 15.7 samples/s +Step 6590/44000 | Loss: 3.2582 | LR: 4.78e-04 | 1274.4ms/step | 15.7 samples/s +Step 6600/44000 | Loss: 3.2480 | LR: 4.78e-04 | 1274.5ms/step | 15.7 samples/s +Step 6610/44000 | Loss: 3.2136 | LR: 4.78e-04 | 1275.0ms/step | 15.7 samples/s +Step 6620/44000 | Loss: 3.2459 | LR: 4.78e-04 | 1274.5ms/step | 15.7 samples/s +Step 6630/44000 | Loss: 3.2497 | LR: 4.78e-04 | 1273.4ms/step | 15.7 samples/s +Step 6640/44000 | Loss: 3.2396 | LR: 4.77e-04 | 1273.8ms/step | 15.7 samples/s +Step 6650/44000 | Loss: 3.2663 | LR: 4.77e-04 | 1273.7ms/step | 15.7 samples/s +Step 6660/44000 | Loss: 3.2912 | LR: 4.77e-04 | 1274.3ms/step | 15.7 samples/s +Step 6670/44000 | Loss: 3.2384 | LR: 4.77e-04 | 1274.8ms/step | 15.7 samples/s +Step 6680/44000 | Loss: 3.2613 | LR: 4.77e-04 | 1275.1ms/step | 15.7 samples/s +Step 6690/44000 | Loss: 3.2619 | LR: 4.77e-04 | 1274.1ms/step | 15.7 samples/s +Step 6700/44000 | Loss: 3.2633 | LR: 4.77e-04 | 1273.3ms/step | 15.7 samples/s +Step 6710/44000 | Loss: 3.2611 | LR: 4.77e-04 | 1274.9ms/step | 15.7 samples/s +Step 6720/44000 | Loss: 3.2268 | LR: 4.77e-04 | 1274.9ms/step | 15.7 samples/s +Step 6730/44000 | Loss: 3.2604 | LR: 4.77e-04 | 1275.5ms/step | 15.7 samples/s +Step 6740/44000 | Loss: 3.2362 | LR: 4.77e-04 | 1273.6ms/step | 15.7 samples/s +Step 6750/44000 | Loss: 3.2433 | LR: 4.77e-04 | 1274.4ms/step | 15.7 samples/s +Validation at step 6750: Loss = 3.2632, Perplexity = 26.13 +Step 6760/44000 | Loss: 3.2683 | LR: 4.77e-04 | 3420.2ms/step | 5.8 samples/s +Step 6770/44000 | Loss: 3.2662 | LR: 4.76e-04 | 1275.0ms/step | 15.7 samples/s +Step 6780/44000 | Loss: 3.2624 | LR: 4.76e-04 | 1275.0ms/step | 15.7 samples/s +Step 6790/44000 | Loss: 3.2544 | LR: 4.76e-04 | 1275.7ms/step | 15.7 samples/s +Step 6800/44000 | Loss: 3.2075 | LR: 4.76e-04 | 1275.3ms/step | 15.7 samples/s +Step 6810/44000 | Loss: 3.2700 | LR: 4.76e-04 | 1274.4ms/step | 15.7 samples/s +Step 6820/44000 | Loss: 3.2325 | LR: 4.76e-04 | 1275.0ms/step | 15.7 samples/s +Step 6830/44000 | Loss: 3.2504 | LR: 4.76e-04 | 1273.5ms/step | 15.7 samples/s +Step 6840/44000 | Loss: 3.2543 | LR: 4.76e-04 | 1275.3ms/step | 15.7 samples/s +Step 6850/44000 | Loss: 3.2568 | LR: 4.76e-04 | 1274.7ms/step | 15.7 samples/s +Step 6860/44000 | Loss: 3.2557 | LR: 4.76e-04 | 1275.9ms/step | 15.7 samples/s +Step 6870/44000 | Loss: 3.2316 | LR: 4.76e-04 | 1276.7ms/step | 15.7 samples/s +Step 6880/44000 | Loss: 3.2037 | LR: 4.76e-04 | 1275.4ms/step | 15.7 samples/s +Step 6890/44000 | Loss: 3.2591 | LR: 4.76e-04 | 1276.0ms/step | 15.7 samples/s +Step 6900/44000 | Loss: 3.2718 | LR: 4.75e-04 | 1275.8ms/step | 15.7 samples/s +Step 6910/44000 | Loss: 3.2343 | LR: 4.75e-04 | 1275.4ms/step | 15.7 samples/s +Step 6920/44000 | Loss: 3.2532 | LR: 4.75e-04 | 1275.0ms/step | 15.7 samples/s +Step 6930/44000 | Loss: 3.2353 | LR: 4.75e-04 | 1273.9ms/step | 15.7 samples/s +Step 6940/44000 | Loss: 3.2608 | LR: 4.75e-04 | 1272.4ms/step | 15.7 samples/s +Step 6950/44000 | Loss: 3.2613 | LR: 4.75e-04 | 1272.1ms/step | 15.7 samples/s +Step 6960/44000 | Loss: 3.2326 | LR: 4.75e-04 | 1273.8ms/step | 15.7 samples/s +Step 6970/44000 | Loss: 3.2557 | LR: 4.75e-04 | 1273.5ms/step | 15.7 samples/s +Step 6980/44000 | Loss: 3.2223 | LR: 4.75e-04 | 1273.4ms/step | 15.7 samples/s +Step 6990/44000 | Loss: 3.2305 | LR: 4.75e-04 | 1273.9ms/step | 15.7 samples/s +Step 7000/44000 | Loss: 3.2376 | LR: 4.75e-04 | 1273.9ms/step | 15.7 samples/s +Validation at step 7000: Loss = 3.2526, Perplexity = 25.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-7000 +Step 7010/44000 | Loss: 3.2295 | LR: 4.75e-04 | 3703.9ms/step | 5.4 samples/s +Step 7020/44000 | Loss: 3.2303 | LR: 4.75e-04 | 1268.7ms/step | 15.8 samples/s +Step 7030/44000 | Loss: 3.2539 | LR: 4.74e-04 | 1270.0ms/step | 15.7 samples/s +Step 7040/44000 | Loss: 3.2444 | LR: 4.74e-04 | 1270.6ms/step | 15.7 samples/s +Step 7050/44000 | Loss: 3.2693 | LR: 4.74e-04 | 1273.6ms/step | 15.7 samples/s +Step 7060/44000 | Loss: 3.2508 | LR: 4.74e-04 | 1273.3ms/step | 15.7 samples/s +Step 7070/44000 | Loss: 3.2388 | LR: 4.74e-04 | 1273.4ms/step | 15.7 samples/s +Step 7080/44000 | Loss: 3.2307 | LR: 4.74e-04 | 1274.7ms/step | 15.7 samples/s +Step 7090/44000 | Loss: 3.2100 | LR: 4.74e-04 | 1274.9ms/step | 15.7 samples/s +Step 7100/44000 | Loss: 3.2362 | LR: 4.74e-04 | 1275.4ms/step | 15.7 samples/s +Step 7110/44000 | Loss: 3.2104 | LR: 4.74e-04 | 1274.5ms/step | 15.7 samples/s +Step 7120/44000 | Loss: 3.2300 | LR: 4.74e-04 | 1273.4ms/step | 15.7 samples/s +Step 7130/44000 | Loss: 3.2427 | LR: 4.74e-04 | 1273.6ms/step | 15.7 samples/s +Step 7140/44000 | Loss: 3.2628 | LR: 4.74e-04 | 1275.6ms/step | 15.7 samples/s +Step 7150/44000 | Loss: 3.2428 | LR: 4.73e-04 | 1276.1ms/step | 15.7 samples/s +Step 7160/44000 | Loss: 3.2526 | LR: 4.73e-04 | 1276.3ms/step | 15.7 samples/s +Step 7170/44000 | Loss: 3.2551 | LR: 4.73e-04 | 1275.0ms/step | 15.7 samples/s +Step 7180/44000 | Loss: 3.2381 | LR: 4.73e-04 | 1274.3ms/step | 15.7 samples/s +Step 7190/44000 | Loss: 3.1996 | LR: 4.73e-04 | 1273.7ms/step | 15.7 samples/s +Step 7200/44000 | Loss: 3.2519 | LR: 4.73e-04 | 1273.3ms/step | 15.7 samples/s +Step 7210/44000 | Loss: 3.2725 | LR: 4.73e-04 | 1274.5ms/step | 15.7 samples/s +Step 7220/44000 | Loss: 3.2549 | LR: 4.73e-04 | 1273.7ms/step | 15.7 samples/s +Step 7230/44000 | Loss: 3.2521 | LR: 4.73e-04 | 1275.9ms/step | 15.7 samples/s +Step 7240/44000 | Loss: 3.2410 | LR: 4.73e-04 | 1275.7ms/step | 15.7 samples/s +Step 7250/44000 | Loss: 3.2456 | LR: 4.73e-04 | 1275.4ms/step | 15.7 samples/s +Validation at step 7250: Loss = 3.2434, Perplexity = 25.62 +Step 7260/44000 | Loss: 3.2266 | LR: 4.73e-04 | 3408.9ms/step | 5.9 samples/s +Step 7270/44000 | Loss: 3.2370 | LR: 4.72e-04 | 1271.2ms/step | 15.7 samples/s +Step 7280/44000 | Loss: 3.2163 | LR: 4.72e-04 | 1272.7ms/step | 15.7 samples/s +Step 7290/44000 | Loss: 3.2575 | LR: 4.72e-04 | 1273.6ms/step | 15.7 samples/s +Step 7300/44000 | Loss: 3.2400 | LR: 4.72e-04 | 1274.1ms/step | 15.7 samples/s +Step 7310/44000 | Loss: 3.2564 | LR: 4.72e-04 | 1274.0ms/step | 15.7 samples/s +Step 7320/44000 | Loss: 3.2566 | LR: 4.72e-04 | 1274.9ms/step | 15.7 samples/s +Step 7330/44000 | Loss: 3.2500 | LR: 4.72e-04 | 1274.4ms/step | 15.7 samples/s +Step 7340/44000 | Loss: 3.2591 | LR: 4.72e-04 | 1276.2ms/step | 15.7 samples/s +Step 7350/44000 | Loss: 3.2064 | LR: 4.72e-04 | 1275.1ms/step | 15.7 samples/s +Step 7360/44000 | Loss: 3.2148 | LR: 4.72e-04 | 1275.2ms/step | 15.7 samples/s +Step 7370/44000 | Loss: 3.2056 | LR: 4.72e-04 | 1274.5ms/step | 15.7 samples/s +Step 7380/44000 | Loss: 3.2353 | LR: 4.72e-04 | 1274.2ms/step | 15.7 samples/s +Step 7390/44000 | Loss: 3.2426 | LR: 4.71e-04 | 1274.0ms/step | 15.7 samples/s +Step 7400/44000 | Loss: 3.2336 | LR: 4.71e-04 | 1275.5ms/step | 15.7 samples/s +Step 7410/44000 | Loss: 3.2552 | LR: 4.71e-04 | 1274.3ms/step | 15.7 samples/s +Step 7420/44000 | Loss: 3.2577 | LR: 4.71e-04 | 1275.5ms/step | 15.7 samples/s +Step 7430/44000 | Loss: 3.2380 | LR: 4.71e-04 | 1275.7ms/step | 15.7 samples/s +Step 7440/44000 | Loss: 3.2388 | LR: 4.71e-04 | 1275.7ms/step | 15.7 samples/s +Step 7450/44000 | Loss: 3.2540 | LR: 4.71e-04 | 1275.5ms/step | 15.7 samples/s +Step 7460/44000 | Loss: 3.1974 | LR: 4.71e-04 | 1276.0ms/step | 15.7 samples/s +Step 7470/44000 | Loss: 3.2406 | LR: 4.71e-04 | 1275.3ms/step | 15.7 samples/s +Step 7480/44000 | Loss: 3.1945 | LR: 4.71e-04 | 1274.0ms/step | 15.7 samples/s +Step 7490/44000 | Loss: 3.2363 | LR: 4.71e-04 | 1275.2ms/step | 15.7 samples/s +Step 7500/44000 | Loss: 3.2001 | LR: 4.71e-04 | 1275.2ms/step | 15.7 samples/s +Validation at step 7500: Loss = 3.2367, Perplexity = 25.45 +Step 7510/44000 | Loss: 3.2114 | LR: 4.70e-04 | 3419.7ms/step | 5.8 samples/s +Step 7520/44000 | Loss: 3.2272 | LR: 4.70e-04 | 1275.6ms/step | 15.7 samples/s +Step 7530/44000 | Loss: 3.2552 | LR: 4.70e-04 | 1274.9ms/step | 15.7 samples/s +Step 7540/44000 | Loss: 3.2112 | LR: 4.70e-04 | 1275.7ms/step | 15.7 samples/s +Step 7550/44000 | Loss: 3.2174 | LR: 4.70e-04 | 1276.0ms/step | 15.7 samples/s +Step 7560/44000 | Loss: 3.2190 | LR: 4.70e-04 | 1275.4ms/step | 15.7 samples/s +Step 7570/44000 | Loss: 3.2496 | LR: 4.70e-04 | 1274.0ms/step | 15.7 samples/s +Step 7580/44000 | Loss: 3.2061 | LR: 4.70e-04 | 1274.2ms/step | 15.7 samples/s +Step 7590/44000 | Loss: 3.2509 | LR: 4.70e-04 | 1274.2ms/step | 15.7 samples/s +Step 7600/44000 | Loss: 3.2304 | LR: 4.70e-04 | 1274.7ms/step | 15.7 samples/s +Step 7610/44000 | Loss: 3.2476 | LR: 4.70e-04 | 1275.4ms/step | 15.7 samples/s +Step 7620/44000 | Loss: 3.2013 | LR: 4.70e-04 | 1274.7ms/step | 15.7 samples/s +Step 7630/44000 | Loss: 3.2549 | LR: 4.69e-04 | 1275.4ms/step | 15.7 samples/s +Step 7640/44000 | Loss: 3.2143 | LR: 4.69e-04 | 1274.4ms/step | 15.7 samples/s +Step 7650/44000 | Loss: 3.2267 | LR: 4.69e-04 | 1275.5ms/step | 15.7 samples/s +Step 7660/44000 | Loss: 3.2369 | LR: 4.69e-04 | 1275.9ms/step | 15.7 samples/s +Step 7670/44000 | Loss: 3.2157 | LR: 4.69e-04 | 1275.2ms/step | 15.7 samples/s +Step 7680/44000 | Loss: 3.2359 | LR: 4.69e-04 | 1275.2ms/step | 15.7 samples/s +Step 7690/44000 | Loss: 3.1979 | LR: 4.69e-04 | 1275.6ms/step | 15.7 samples/s +Step 7700/44000 | Loss: 3.2230 | LR: 4.69e-04 | 1273.5ms/step | 15.7 samples/s +Step 7710/44000 | Loss: 3.2472 | LR: 4.69e-04 | 1275.6ms/step | 15.7 samples/s +Step 7720/44000 | Loss: 3.2467 | LR: 4.69e-04 | 1274.7ms/step | 15.7 samples/s +Step 7730/44000 | Loss: 3.2416 | LR: 4.69e-04 | 1272.6ms/step | 15.7 samples/s +Step 7740/44000 | Loss: 3.2109 | LR: 4.68e-04 | 1273.5ms/step | 15.7 samples/s +Step 7750/44000 | Loss: 3.1882 | LR: 4.68e-04 | 1273.7ms/step | 15.7 samples/s +Validation at step 7750: Loss = 3.2268, Perplexity = 25.20 +Step 7760/44000 | Loss: 3.1989 | LR: 4.68e-04 | 3419.3ms/step | 5.8 samples/s +Step 7770/44000 | Loss: 3.2020 | LR: 4.68e-04 | 1273.8ms/step | 15.7 samples/s +Step 7780/44000 | Loss: 3.1985 | LR: 4.68e-04 | 1275.1ms/step | 15.7 samples/s +Step 7790/44000 | Loss: 3.2227 | LR: 4.68e-04 | 1276.2ms/step | 15.7 samples/s +Step 7800/44000 | Loss: 3.2125 | LR: 4.68e-04 | 1275.1ms/step | 15.7 samples/s +Step 7810/44000 | Loss: 3.2004 | LR: 4.68e-04 | 1274.7ms/step | 15.7 samples/s +Step 7820/44000 | Loss: 3.2150 | LR: 4.68e-04 | 1274.1ms/step | 15.7 samples/s +Step 7830/44000 | Loss: 3.2312 | LR: 4.68e-04 | 1275.3ms/step | 15.7 samples/s +Step 7840/44000 | Loss: 3.2099 | LR: 4.68e-04 | 1274.6ms/step | 15.7 samples/s +Step 7850/44000 | Loss: 3.1914 | LR: 4.67e-04 | 1273.3ms/step | 15.7 samples/s +Step 7860/44000 | Loss: 3.2079 | LR: 4.67e-04 | 1274.4ms/step | 15.7 samples/s +Step 7870/44000 | Loss: 3.2114 | LR: 4.67e-04 | 1275.1ms/step | 15.7 samples/s +Step 7880/44000 | Loss: 3.1719 | LR: 4.67e-04 | 1275.2ms/step | 15.7 samples/s +Step 7890/44000 | Loss: 3.1906 | LR: 4.67e-04 | 1274.7ms/step | 15.7 samples/s +Step 7900/44000 | Loss: 3.2387 | LR: 4.67e-04 | 1274.0ms/step | 15.7 samples/s +Step 7910/44000 | Loss: 3.1866 | LR: 4.67e-04 | 1274.7ms/step | 15.7 samples/s +Step 7920/44000 | Loss: 3.2233 | LR: 4.67e-04 | 1271.7ms/step | 15.7 samples/s +Step 7930/44000 | Loss: 3.2017 | LR: 4.67e-04 | 1269.0ms/step | 15.8 samples/s +Step 7940/44000 | Loss: 3.2178 | LR: 4.67e-04 | 1271.1ms/step | 15.7 samples/s +Step 7950/44000 | Loss: 3.2115 | LR: 4.67e-04 | 1272.8ms/step | 15.7 samples/s +Step 7960/44000 | Loss: 3.2062 | LR: 4.66e-04 | 1274.1ms/step | 15.7 samples/s +Step 7970/44000 | Loss: 3.2006 | LR: 4.66e-04 | 1274.0ms/step | 15.7 samples/s +Step 7980/44000 | Loss: 3.2265 | LR: 4.66e-04 | 1274.2ms/step | 15.7 samples/s +Step 7990/44000 | Loss: 3.2236 | LR: 4.66e-04 | 1275.0ms/step | 15.7 samples/s +Step 8000/44000 | Loss: 3.1949 | LR: 4.66e-04 | 1274.6ms/step | 15.7 samples/s +Validation at step 8000: Loss = 3.2188, Perplexity = 25.00 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-8000 +Step 8010/44000 | Loss: 3.1660 | LR: 4.66e-04 | 3717.2ms/step | 5.4 samples/s +Step 8020/44000 | Loss: 3.1868 | LR: 4.66e-04 | 1273.7ms/step | 15.7 samples/s +Step 8030/44000 | Loss: 3.1881 | LR: 4.66e-04 | 1275.0ms/step | 15.7 samples/s +Step 8040/44000 | Loss: 3.2133 | LR: 4.66e-04 | 1274.9ms/step | 15.7 samples/s +Step 8050/44000 | Loss: 3.1949 | LR: 4.66e-04 | 1273.9ms/step | 15.7 samples/s +Step 8060/44000 | Loss: 3.2290 | LR: 4.66e-04 | 1275.1ms/step | 15.7 samples/s +Step 8070/44000 | Loss: 3.1943 | LR: 4.65e-04 | 1274.1ms/step | 15.7 samples/s +Step 8080/44000 | Loss: 3.1840 | LR: 4.65e-04 | 1273.4ms/step | 15.7 samples/s +Step 8090/44000 | Loss: 3.1858 | LR: 4.65e-04 | 1273.1ms/step | 15.7 samples/s +Step 8100/44000 | Loss: 3.2084 | LR: 4.65e-04 | 1273.7ms/step | 15.7 samples/s +Step 8110/44000 | Loss: 3.2027 | LR: 4.65e-04 | 1274.0ms/step | 15.7 samples/s +Step 8120/44000 | Loss: 3.1961 | LR: 4.65e-04 | 1274.9ms/step | 15.7 samples/s +Step 8130/44000 | Loss: 3.1970 | LR: 4.65e-04 | 1275.0ms/step | 15.7 samples/s +Step 8140/44000 | Loss: 3.2319 | LR: 4.65e-04 | 1274.8ms/step | 15.7 samples/s +Step 8150/44000 | Loss: 3.1948 | LR: 4.65e-04 | 1276.0ms/step | 15.7 samples/s +Step 8160/44000 | Loss: 3.1868 | LR: 4.65e-04 | 1276.0ms/step | 15.7 samples/s +Step 8170/44000 | Loss: 3.2118 | LR: 4.65e-04 | 1275.2ms/step | 15.7 samples/s +Step 8180/44000 | Loss: 3.2004 | LR: 4.64e-04 | 1278.0ms/step | 15.6 samples/s +Step 8190/44000 | Loss: 3.1934 | LR: 4.64e-04 | 1276.3ms/step | 15.7 samples/s +Step 8200/44000 | Loss: 3.2193 | LR: 4.64e-04 | 1274.2ms/step | 15.7 samples/s +Step 8210/44000 | Loss: 3.2196 | LR: 4.64e-04 | 1273.5ms/step | 15.7 samples/s +Step 8220/44000 | Loss: 3.2243 | LR: 4.64e-04 | 1275.7ms/step | 15.7 samples/s +Step 8230/44000 | Loss: 3.1857 | LR: 4.64e-04 | 1273.6ms/step | 15.7 samples/s +Step 8240/44000 | Loss: 3.1978 | LR: 4.64e-04 | 1273.4ms/step | 15.7 samples/s +Step 8250/44000 | Loss: 3.1953 | LR: 4.64e-04 | 1275.2ms/step | 15.7 samples/s +Validation at step 8250: Loss = 3.2116, Perplexity = 24.82 +Step 8260/44000 | Loss: 3.1808 | LR: 4.64e-04 | 3419.2ms/step | 5.8 samples/s +Step 8270/44000 | Loss: 3.2239 | LR: 4.64e-04 | 1270.9ms/step | 15.7 samples/s +Step 8280/44000 | Loss: 3.2006 | LR: 4.64e-04 | 1272.5ms/step | 15.7 samples/s +Step 8290/44000 | Loss: 3.2050 | LR: 4.63e-04 | 1273.4ms/step | 15.7 samples/s +Step 8300/44000 | Loss: 3.1601 | LR: 4.63e-04 | 1274.1ms/step | 15.7 samples/s +Step 8310/44000 | Loss: 3.2120 | LR: 4.63e-04 | 1274.6ms/step | 15.7 samples/s +Step 8320/44000 | Loss: 3.1870 | LR: 4.63e-04 | 1275.3ms/step | 15.7 samples/s +Step 8330/44000 | Loss: 3.1940 | LR: 4.63e-04 | 1275.7ms/step | 15.7 samples/s +Step 8340/44000 | Loss: 3.2391 | LR: 4.63e-04 | 1275.1ms/step | 15.7 samples/s +Step 8350/44000 | Loss: 3.1875 | LR: 4.63e-04 | 1275.0ms/step | 15.7 samples/s +Step 8360/44000 | Loss: 3.2131 | LR: 4.63e-04 | 1276.0ms/step | 15.7 samples/s +Step 8370/44000 | Loss: 3.1979 | LR: 4.63e-04 | 1275.5ms/step | 15.7 samples/s +Step 8380/44000 | Loss: 3.2362 | LR: 4.63e-04 | 1274.1ms/step | 15.7 samples/s +Step 8390/44000 | Loss: 3.1700 | LR: 4.62e-04 | 1274.4ms/step | 15.7 samples/s +Step 8400/44000 | Loss: 3.1865 | LR: 4.62e-04 | 1274.8ms/step | 15.7 samples/s +Step 8410/44000 | Loss: 3.1837 | LR: 4.62e-04 | 1274.9ms/step | 15.7 samples/s +Step 8420/44000 | Loss: 3.2241 | LR: 4.62e-04 | 1275.7ms/step | 15.7 samples/s +Step 8430/44000 | Loss: 3.1888 | LR: 4.62e-04 | 1275.1ms/step | 15.7 samples/s +Step 8440/44000 | Loss: 3.2147 | LR: 4.62e-04 | 1275.6ms/step | 15.7 samples/s +Step 8450/44000 | Loss: 3.1669 | LR: 4.62e-04 | 1275.6ms/step | 15.7 samples/s +Step 8460/44000 | Loss: 3.2163 | LR: 4.62e-04 | 1275.3ms/step | 15.7 samples/s +Step 8470/44000 | Loss: 3.1627 | LR: 4.62e-04 | 1275.8ms/step | 15.7 samples/s +Step 8480/44000 | Loss: 3.1876 | LR: 4.62e-04 | 1275.6ms/step | 15.7 samples/s +Step 8490/44000 | Loss: 3.2084 | LR: 4.62e-04 | 1275.7ms/step | 15.7 samples/s +Step 8500/44000 | Loss: 3.2162 | LR: 4.61e-04 | 1273.5ms/step | 15.7 samples/s +Validation at step 8500: Loss = 3.2045, Perplexity = 24.64 +Step 8510/44000 | Loss: 3.2150 | LR: 4.61e-04 | 3421.4ms/step | 5.8 samples/s +Step 8520/44000 | Loss: 3.1873 | LR: 4.61e-04 | 1275.7ms/step | 15.7 samples/s +Step 8530/44000 | Loss: 3.2141 | LR: 4.61e-04 | 1274.0ms/step | 15.7 samples/s +Step 8540/44000 | Loss: 3.1723 | LR: 4.61e-04 | 1276.1ms/step | 15.7 samples/s +Step 8550/44000 | Loss: 3.1911 | LR: 4.61e-04 | 1275.5ms/step | 15.7 samples/s +Step 8560/44000 | Loss: 3.1980 | LR: 4.61e-04 | 1275.7ms/step | 15.7 samples/s +Step 8570/44000 | Loss: 3.1663 | LR: 4.61e-04 | 1276.1ms/step | 15.7 samples/s +Step 8580/44000 | Loss: 3.1971 | LR: 4.61e-04 | 1273.5ms/step | 15.7 samples/s +Step 8590/44000 | Loss: 3.1958 | LR: 4.61e-04 | 1273.1ms/step | 15.7 samples/s +Step 8600/44000 | Loss: 3.1850 | LR: 4.60e-04 | 1273.5ms/step | 15.7 samples/s +Step 8610/44000 | Loss: 3.1645 | LR: 4.60e-04 | 1273.7ms/step | 15.7 samples/s +Step 8620/44000 | Loss: 3.1946 | LR: 4.60e-04 | 1274.7ms/step | 15.7 samples/s +Step 8630/44000 | Loss: 3.1828 | LR: 4.60e-04 | 1273.2ms/step | 15.7 samples/s +Step 8640/44000 | Loss: 3.1902 | LR: 4.60e-04 | 1273.3ms/step | 15.7 samples/s +Step 8650/44000 | Loss: 3.1940 | LR: 4.60e-04 | 1274.4ms/step | 15.7 samples/s +Step 8660/44000 | Loss: 3.1563 | LR: 4.60e-04 | 1274.4ms/step | 15.7 samples/s +Step 8670/44000 | Loss: 3.1970 | LR: 4.60e-04 | 1274.0ms/step | 15.7 samples/s +Step 8680/44000 | Loss: 3.1970 | LR: 4.60e-04 | 1273.5ms/step | 15.7 samples/s +Step 8690/44000 | Loss: 3.1762 | LR: 4.60e-04 | 1275.0ms/step | 15.7 samples/s +Step 8700/44000 | Loss: 3.1528 | LR: 4.59e-04 | 1275.6ms/step | 15.7 samples/s +Step 8710/44000 | Loss: 3.1846 | LR: 4.59e-04 | 1273.7ms/step | 15.7 samples/s +Step 8720/44000 | Loss: 3.1892 | LR: 4.59e-04 | 1273.8ms/step | 15.7 samples/s +Step 8730/44000 | Loss: 3.1926 | LR: 4.59e-04 | 1274.7ms/step | 15.7 samples/s +Step 8740/44000 | Loss: 3.1846 | LR: 4.59e-04 | 1274.7ms/step | 15.7 samples/s +Step 8750/44000 | Loss: 3.1586 | LR: 4.59e-04 | 1275.0ms/step | 15.7 samples/s +Validation at step 8750: Loss = 3.1984, Perplexity = 24.49 +Step 8760/44000 | Loss: 3.1723 | LR: 4.59e-04 | 3422.7ms/step | 5.8 samples/s +Step 8770/44000 | Loss: 3.2125 | LR: 4.59e-04 | 1273.9ms/step | 15.7 samples/s +Step 8780/44000 | Loss: 3.1771 | LR: 4.59e-04 | 1273.7ms/step | 15.7 samples/s +Step 8790/44000 | Loss: 3.1648 | LR: 4.59e-04 | 1272.8ms/step | 15.7 samples/s +Step 8800/44000 | Loss: 3.1593 | LR: 4.58e-04 | 1273.3ms/step | 15.7 samples/s +Step 8810/44000 | Loss: 3.1812 | LR: 4.58e-04 | 1274.0ms/step | 15.7 samples/s +Step 8820/44000 | Loss: 3.1651 | LR: 4.58e-04 | 1273.6ms/step | 15.7 samples/s +Step 8830/44000 | Loss: 3.1698 | LR: 4.58e-04 | 1275.7ms/step | 15.7 samples/s +Step 8840/44000 | Loss: 3.1789 | LR: 4.58e-04 | 1275.6ms/step | 15.7 samples/s +Step 8850/44000 | Loss: 3.1810 | LR: 4.58e-04 | 1276.6ms/step | 15.7 samples/s +Step 8860/44000 | Loss: 3.2002 | LR: 4.58e-04 | 1277.5ms/step | 15.7 samples/s +Step 8870/44000 | Loss: 3.1815 | LR: 4.58e-04 | 1275.2ms/step | 15.7 samples/s +Step 8880/44000 | Loss: 3.1612 | LR: 4.58e-04 | 1273.8ms/step | 15.7 samples/s +Step 8890/44000 | Loss: 3.2071 | LR: 4.58e-04 | 1274.6ms/step | 15.7 samples/s +Step 8900/44000 | Loss: 3.1691 | LR: 4.57e-04 | 1273.8ms/step | 15.7 samples/s +Step 8910/44000 | Loss: 3.2029 | LR: 4.57e-04 | 1270.3ms/step | 15.7 samples/s +Step 8920/44000 | Loss: 3.1828 | LR: 4.57e-04 | 1271.1ms/step | 15.7 samples/s +Step 8930/44000 | Loss: 3.1808 | LR: 4.57e-04 | 1273.3ms/step | 15.7 samples/s +Step 8940/44000 | Loss: 3.1735 | LR: 4.57e-04 | 1273.6ms/step | 15.7 samples/s +Step 8950/44000 | Loss: 3.1797 | LR: 4.57e-04 | 1273.2ms/step | 15.7 samples/s +Step 8960/44000 | Loss: 3.1624 | LR: 4.57e-04 | 1273.5ms/step | 15.7 samples/s +Step 8970/44000 | Loss: 3.1841 | LR: 4.57e-04 | 1274.2ms/step | 15.7 samples/s +Step 8980/44000 | Loss: 3.1842 | LR: 4.57e-04 | 1275.2ms/step | 15.7 samples/s +Step 8990/44000 | Loss: 3.1700 | LR: 4.57e-04 | 1274.4ms/step | 15.7 samples/s +Step 9000/44000 | Loss: 3.1699 | LR: 4.56e-04 | 1273.9ms/step | 15.7 samples/s +Validation at step 9000: Loss = 3.1910, Perplexity = 24.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-9000 +Step 9010/44000 | Loss: 3.1709 | LR: 4.56e-04 | 3704.5ms/step | 5.4 samples/s +Step 9020/44000 | Loss: 3.1875 | LR: 4.56e-04 | 1270.0ms/step | 15.7 samples/s +Step 9030/44000 | Loss: 3.1622 | LR: 4.56e-04 | 1271.9ms/step | 15.7 samples/s +Step 9040/44000 | Loss: 3.1655 | LR: 4.56e-04 | 1272.9ms/step | 15.7 samples/s +Step 9050/44000 | Loss: 3.1680 | LR: 4.56e-04 | 1272.2ms/step | 15.7 samples/s +Step 9060/44000 | Loss: 3.2038 | LR: 4.56e-04 | 1272.7ms/step | 15.7 samples/s +Step 9070/44000 | Loss: 3.1984 | LR: 4.56e-04 | 1273.5ms/step | 15.7 samples/s +Step 9080/44000 | Loss: 3.2201 | LR: 4.56e-04 | 1273.9ms/step | 15.7 samples/s +Step 9090/44000 | Loss: 3.1494 | LR: 4.56e-04 | 1274.9ms/step | 15.7 samples/s +Step 9100/44000 | Loss: 3.1560 | LR: 4.55e-04 | 1275.1ms/step | 15.7 samples/s +Step 9110/44000 | Loss: 3.1530 | LR: 4.55e-04 | 1274.5ms/step | 15.7 samples/s +Step 9120/44000 | Loss: 3.1551 | LR: 4.55e-04 | 1275.4ms/step | 15.7 samples/s +Step 9130/44000 | Loss: 3.2008 | LR: 4.55e-04 | 1274.5ms/step | 15.7 samples/s +Step 9140/44000 | Loss: 3.1494 | LR: 4.55e-04 | 1275.3ms/step | 15.7 samples/s +Step 9150/44000 | Loss: 3.1772 | LR: 4.55e-04 | 1276.4ms/step | 15.7 samples/s +Step 9160/44000 | Loss: 3.1512 | LR: 4.55e-04 | 1275.3ms/step | 15.7 samples/s +Step 9170/44000 | Loss: 3.1328 | LR: 4.55e-04 | 1274.4ms/step | 15.7 samples/s +Step 9180/44000 | Loss: 3.1818 | LR: 4.55e-04 | 1273.0ms/step | 15.7 samples/s +Step 9190/44000 | Loss: 3.1573 | LR: 4.54e-04 | 1272.3ms/step | 15.7 samples/s +Step 9200/44000 | Loss: 3.1851 | LR: 4.54e-04 | 1273.2ms/step | 15.7 samples/s +Step 9210/44000 | Loss: 3.1786 | LR: 4.54e-04 | 1273.3ms/step | 15.7 samples/s +Step 9220/44000 | Loss: 3.1687 | LR: 4.54e-04 | 1273.6ms/step | 15.7 samples/s +Step 9230/44000 | Loss: 3.1692 | LR: 4.54e-04 | 1274.3ms/step | 15.7 samples/s +Step 9240/44000 | Loss: 3.2201 | LR: 4.54e-04 | 1274.3ms/step | 15.7 samples/s +Step 9250/44000 | Loss: 3.1738 | LR: 4.54e-04 | 1274.5ms/step | 15.7 samples/s +Validation at step 9250: Loss = 3.1848, Perplexity = 24.16 +Step 9260/44000 | Loss: 3.1800 | LR: 4.54e-04 | 3424.4ms/step | 5.8 samples/s +Step 9270/44000 | Loss: 3.1759 | LR: 4.54e-04 | 1275.6ms/step | 15.7 samples/s +Step 9280/44000 | Loss: 3.1755 | LR: 4.54e-04 | 1275.8ms/step | 15.7 samples/s +Step 9290/44000 | Loss: 3.1880 | LR: 4.53e-04 | 1275.2ms/step | 15.7 samples/s +Step 9300/44000 | Loss: 3.1474 | LR: 4.53e-04 | 1274.5ms/step | 15.7 samples/s +Step 9310/44000 | Loss: 3.1678 | LR: 4.53e-04 | 1273.0ms/step | 15.7 samples/s +Step 9320/44000 | Loss: 3.1519 | LR: 4.53e-04 | 1274.5ms/step | 15.7 samples/s +Step 9330/44000 | Loss: 3.1729 | LR: 4.53e-04 | 1274.1ms/step | 15.7 samples/s +Step 9340/44000 | Loss: 3.1554 | LR: 4.53e-04 | 1273.2ms/step | 15.7 samples/s +Step 9350/44000 | Loss: 3.1460 | LR: 4.53e-04 | 1274.2ms/step | 15.7 samples/s +Step 9360/44000 | Loss: 3.1726 | LR: 4.53e-04 | 1274.0ms/step | 15.7 samples/s +Step 9370/44000 | Loss: 3.1739 | LR: 4.53e-04 | 1273.0ms/step | 15.7 samples/s +Step 9380/44000 | Loss: 3.1708 | LR: 4.52e-04 | 1274.3ms/step | 15.7 samples/s +Step 9390/44000 | Loss: 3.1827 | LR: 4.52e-04 | 1274.7ms/step | 15.7 samples/s +Step 9400/44000 | Loss: 3.1600 | LR: 4.52e-04 | 1274.0ms/step | 15.7 samples/s +Step 9410/44000 | Loss: 3.1648 | LR: 4.52e-04 | 1275.2ms/step | 15.7 samples/s +Step 9420/44000 | Loss: 3.1769 | LR: 4.52e-04 | 1275.5ms/step | 15.7 samples/s +Step 9430/44000 | Loss: 3.1604 | LR: 4.52e-04 | 1274.7ms/step | 15.7 samples/s +Step 9440/44000 | Loss: 3.1397 | LR: 4.52e-04 | 1273.5ms/step | 15.7 samples/s +Step 9450/44000 | Loss: 3.1638 | LR: 4.52e-04 | 1273.4ms/step | 15.7 samples/s +Step 9460/44000 | Loss: 3.1766 | LR: 4.52e-04 | 1274.4ms/step | 15.7 samples/s +Step 9470/44000 | Loss: 3.1568 | LR: 4.52e-04 | 1275.2ms/step | 15.7 samples/s +Step 9480/44000 | Loss: 3.1535 | LR: 4.51e-04 | 1274.1ms/step | 15.7 samples/s +Step 9490/44000 | Loss: 3.1672 | LR: 4.51e-04 | 1273.9ms/step | 15.7 samples/s +Step 9500/44000 | Loss: 3.1697 | LR: 4.51e-04 | 1273.4ms/step | 15.7 samples/s +Validation at step 9500: Loss = 3.1773, Perplexity = 23.98 +Step 9510/44000 | Loss: 3.2008 | LR: 4.51e-04 | 3418.2ms/step | 5.9 samples/s +Step 9520/44000 | Loss: 3.1536 | LR: 4.51e-04 | 1274.3ms/step | 15.7 samples/s +Step 9530/44000 | Loss: 3.1933 | LR: 4.51e-04 | 1273.5ms/step | 15.7 samples/s +Step 9540/44000 | Loss: 3.1907 | LR: 4.51e-04 | 1273.3ms/step | 15.7 samples/s +Step 9550/44000 | Loss: 3.1774 | LR: 4.51e-04 | 1274.0ms/step | 15.7 samples/s +Step 9560/44000 | Loss: 3.1564 | LR: 4.51e-04 | 1272.7ms/step | 15.7 samples/s +Step 9570/44000 | Loss: 3.1531 | LR: 4.50e-04 | 1273.3ms/step | 15.7 samples/s +Step 9580/44000 | Loss: 3.1838 | LR: 4.50e-04 | 1274.5ms/step | 15.7 samples/s +Step 9590/44000 | Loss: 3.1485 | LR: 4.50e-04 | 1275.7ms/step | 15.7 samples/s +Step 9600/44000 | Loss: 3.1509 | LR: 4.50e-04 | 1274.8ms/step | 15.7 samples/s +Step 9610/44000 | Loss: 3.1518 | LR: 4.50e-04 | 1275.1ms/step | 15.7 samples/s +Step 9620/44000 | Loss: 3.1920 | LR: 4.50e-04 | 1275.9ms/step | 15.7 samples/s +Step 9630/44000 | Loss: 3.1681 | LR: 4.50e-04 | 1276.1ms/step | 15.7 samples/s +Step 9640/44000 | Loss: 3.1652 | LR: 4.50e-04 | 1274.6ms/step | 15.7 samples/s +Step 9650/44000 | Loss: 3.1677 | LR: 4.50e-04 | 1275.6ms/step | 15.7 samples/s +Step 9660/44000 | Loss: 3.1425 | LR: 4.49e-04 | 1275.1ms/step | 15.7 samples/s +Step 9670/44000 | Loss: 3.1655 | LR: 4.49e-04 | 1274.8ms/step | 15.7 samples/s +Step 9680/44000 | Loss: 3.1706 | LR: 4.49e-04 | 1274.7ms/step | 15.7 samples/s +Step 9690/44000 | Loss: 3.1674 | LR: 4.49e-04 | 1274.3ms/step | 15.7 samples/s +Step 9700/44000 | Loss: 3.1713 | LR: 4.49e-04 | 1275.4ms/step | 15.7 samples/s +Step 9710/44000 | Loss: 3.1700 | LR: 4.49e-04 | 1275.7ms/step | 15.7 samples/s +Step 9720/44000 | Loss: 3.1566 | LR: 4.49e-04 | 1275.7ms/step | 15.7 samples/s +Step 9730/44000 | Loss: 3.1723 | LR: 4.49e-04 | 1274.6ms/step | 15.7 samples/s +Step 9740/44000 | Loss: 3.1594 | LR: 4.49e-04 | 1276.1ms/step | 15.7 samples/s +Step 9750/44000 | Loss: 3.1530 | LR: 4.48e-04 | 1275.4ms/step | 15.7 samples/s +Validation at step 9750: Loss = 3.1724, Perplexity = 23.86 +Step 9760/44000 | Loss: 3.1312 | LR: 4.48e-04 | 3419.8ms/step | 5.8 samples/s +Step 9770/44000 | Loss: 3.1344 | LR: 4.48e-04 | 1275.3ms/step | 15.7 samples/s +Step 9780/44000 | Loss: 3.1596 | LR: 4.48e-04 | 1275.2ms/step | 15.7 samples/s +Step 9790/44000 | Loss: 3.1747 | LR: 4.48e-04 | 1274.0ms/step | 15.7 samples/s +Step 9800/44000 | Loss: 3.1592 | LR: 4.48e-04 | 1275.3ms/step | 15.7 samples/s +Step 9810/44000 | Loss: 3.1768 | LR: 4.48e-04 | 1273.9ms/step | 15.7 samples/s +Step 9820/44000 | Loss: 3.1773 | LR: 4.48e-04 | 1274.7ms/step | 15.7 samples/s +Step 9830/44000 | Loss: 3.2052 | LR: 4.48e-04 | 1273.9ms/step | 15.7 samples/s +Step 9840/44000 | Loss: 3.1493 | LR: 4.47e-04 | 1275.0ms/step | 15.7 samples/s +Step 9850/44000 | Loss: 3.1703 | LR: 4.47e-04 | 1274.7ms/step | 15.7 samples/s +Step 9860/44000 | Loss: 3.1305 | LR: 4.47e-04 | 1275.1ms/step | 15.7 samples/s +Step 9870/44000 | Loss: 3.1546 | LR: 4.47e-04 | 1274.9ms/step | 15.7 samples/s +Step 9880/44000 | Loss: 3.1581 | LR: 4.47e-04 | 1274.7ms/step | 15.7 samples/s +Step 9890/44000 | Loss: 3.1389 | LR: 4.47e-04 | 1271.2ms/step | 15.7 samples/s +Step 9900/44000 | Loss: 3.1641 | LR: 4.47e-04 | 1269.6ms/step | 15.8 samples/s +Step 9910/44000 | Loss: 3.1778 | LR: 4.47e-04 | 1271.2ms/step | 15.7 samples/s +Step 9920/44000 | Loss: 3.1431 | LR: 4.47e-04 | 1271.6ms/step | 15.7 samples/s +Step 9930/44000 | Loss: 3.1381 | LR: 4.46e-04 | 1272.7ms/step | 15.7 samples/s +Step 9940/44000 | Loss: 3.1585 | LR: 4.46e-04 | 1273.3ms/step | 15.7 samples/s +Step 9950/44000 | Loss: 3.1421 | LR: 4.46e-04 | 1273.6ms/step | 15.7 samples/s +Step 9960/44000 | Loss: 3.1557 | LR: 4.46e-04 | 1274.4ms/step | 15.7 samples/s +Step 9970/44000 | Loss: 3.1996 | LR: 4.46e-04 | 1274.8ms/step | 15.7 samples/s +Step 9980/44000 | Loss: 3.1498 | LR: 4.46e-04 | 1273.9ms/step | 15.7 samples/s +Step 9990/44000 | Loss: 3.1591 | LR: 4.46e-04 | 1275.6ms/step | 15.7 samples/s +Step 10000/44000 | Loss: 3.1546 | LR: 4.46e-04 | 1275.4ms/step | 15.7 samples/s +Validation at step 10000: Loss = 3.1660, Perplexity = 23.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-10000 +Step 10010/44000 | Loss: 3.1493 | LR: 4.46e-04 | 3706.6ms/step | 5.4 samples/s +Step 10020/44000 | Loss: 3.1789 | LR: 4.45e-04 | 1270.8ms/step | 15.7 samples/s +Step 10030/44000 | Loss: 3.1591 | LR: 4.45e-04 | 1270.4ms/step | 15.7 samples/s +Step 10040/44000 | Loss: 3.1281 | LR: 4.45e-04 | 1270.0ms/step | 15.7 samples/s +Step 10050/44000 | Loss: 3.1372 | LR: 4.45e-04 | 1272.8ms/step | 15.7 samples/s +Step 10060/44000 | Loss: 3.1376 | LR: 4.45e-04 | 1273.8ms/step | 15.7 samples/s +Step 10070/44000 | Loss: 3.1403 | LR: 4.45e-04 | 1274.1ms/step | 15.7 samples/s +Step 10080/44000 | Loss: 3.1499 | LR: 4.45e-04 | 1273.1ms/step | 15.7 samples/s +Step 10090/44000 | Loss: 3.1467 | LR: 4.45e-04 | 1273.7ms/step | 15.7 samples/s +Step 10100/44000 | Loss: 3.1637 | LR: 4.45e-04 | 1273.7ms/step | 15.7 samples/s +Step 10110/44000 | Loss: 3.1726 | LR: 4.44e-04 | 1274.7ms/step | 15.7 samples/s +Step 10120/44000 | Loss: 3.1947 | LR: 4.44e-04 | 1274.5ms/step | 15.7 samples/s +Step 10130/44000 | Loss: 3.1417 | LR: 4.44e-04 | 1275.6ms/step | 15.7 samples/s +Step 10140/44000 | Loss: 3.2019 | LR: 4.44e-04 | 1273.3ms/step | 15.7 samples/s +Step 10150/44000 | Loss: 3.1602 | LR: 4.44e-04 | 1274.4ms/step | 15.7 samples/s +Step 10160/44000 | Loss: 3.1598 | LR: 4.44e-04 | 1273.9ms/step | 15.7 samples/s +Step 10170/44000 | Loss: 3.1305 | LR: 4.44e-04 | 1274.3ms/step | 15.7 samples/s +Step 10180/44000 | Loss: 3.1649 | LR: 4.44e-04 | 1274.7ms/step | 15.7 samples/s +Step 10190/44000 | Loss: 3.1350 | LR: 4.43e-04 | 1274.2ms/step | 15.7 samples/s +Step 10200/44000 | Loss: 3.1369 | LR: 4.43e-04 | 1273.5ms/step | 15.7 samples/s +Step 10210/44000 | Loss: 3.1539 | LR: 4.43e-04 | 1273.6ms/step | 15.7 samples/s +Step 10220/44000 | Loss: 3.1403 | LR: 4.43e-04 | 1273.1ms/step | 15.7 samples/s +Step 10230/44000 | Loss: 3.1628 | LR: 4.43e-04 | 1273.4ms/step | 15.7 samples/s +Step 10240/44000 | Loss: 3.1402 | LR: 4.43e-04 | 1275.8ms/step | 15.7 samples/s +Step 10250/44000 | Loss: 3.1605 | LR: 4.43e-04 | 1274.2ms/step | 15.7 samples/s +Validation at step 10250: Loss = 3.1597, Perplexity = 23.56 +Step 10260/44000 | Loss: 3.1209 | LR: 4.43e-04 | 3411.9ms/step | 5.9 samples/s +Step 10270/44000 | Loss: 3.1076 | LR: 4.43e-04 | 1273.4ms/step | 15.7 samples/s +Step 10280/44000 | Loss: 3.1110 | LR: 4.42e-04 | 1272.8ms/step | 15.7 samples/s +Step 10290/44000 | Loss: 3.1044 | LR: 4.42e-04 | 1273.0ms/step | 15.7 samples/s +Step 10300/44000 | Loss: 3.1376 | LR: 4.42e-04 | 1273.4ms/step | 15.7 samples/s +Step 10310/44000 | Loss: 3.1622 | LR: 4.42e-04 | 1273.7ms/step | 15.7 samples/s +Step 10320/44000 | Loss: 3.1277 | LR: 4.42e-04 | 1272.1ms/step | 15.7 samples/s +Step 10330/44000 | Loss: 3.1704 | LR: 4.42e-04 | 1273.5ms/step | 15.7 samples/s +Step 10340/44000 | Loss: 3.1438 | LR: 4.42e-04 | 1273.2ms/step | 15.7 samples/s +Step 10350/44000 | Loss: 3.1565 | LR: 4.42e-04 | 1274.7ms/step | 15.7 samples/s +Step 10360/44000 | Loss: 3.1288 | LR: 4.42e-04 | 1274.9ms/step | 15.7 samples/s +Step 10370/44000 | Loss: 3.1553 | LR: 4.41e-04 | 1273.3ms/step | 15.7 samples/s +Step 10380/44000 | Loss: 3.1672 | LR: 4.41e-04 | 1273.0ms/step | 15.7 samples/s +Step 10390/44000 | Loss: 3.1378 | LR: 4.41e-04 | 1274.1ms/step | 15.7 samples/s +Step 10400/44000 | Loss: 3.1682 | LR: 4.41e-04 | 1275.4ms/step | 15.7 samples/s +Step 10410/44000 | Loss: 3.1744 | LR: 4.41e-04 | 1275.1ms/step | 15.7 samples/s +Step 10420/44000 | Loss: 3.1684 | LR: 4.41e-04 | 1273.8ms/step | 15.7 samples/s +Step 10430/44000 | Loss: 3.1594 | LR: 4.41e-04 | 1274.3ms/step | 15.7 samples/s +Step 10440/44000 | Loss: 3.1304 | LR: 4.41e-04 | 1275.1ms/step | 15.7 samples/s +Step 10450/44000 | Loss: 3.1412 | LR: 4.40e-04 | 1276.0ms/step | 15.7 samples/s +Step 10460/44000 | Loss: 3.1361 | LR: 4.40e-04 | 1275.1ms/step | 15.7 samples/s +Step 10470/44000 | Loss: 3.1665 | LR: 4.40e-04 | 1274.4ms/step | 15.7 samples/s +Step 10480/44000 | Loss: 3.1349 | LR: 4.40e-04 | 1273.9ms/step | 15.7 samples/s +Step 10490/44000 | Loss: 3.1151 | LR: 4.40e-04 | 1274.3ms/step | 15.7 samples/s +Step 10500/44000 | Loss: 3.1689 | LR: 4.40e-04 | 1274.5ms/step | 15.7 samples/s +Validation at step 10500: Loss = 3.1546, Perplexity = 23.44 +Step 10510/44000 | Loss: 3.1782 | LR: 4.40e-04 | 3417.1ms/step | 5.9 samples/s +Step 10520/44000 | Loss: 3.1183 | LR: 4.40e-04 | 1275.0ms/step | 15.7 samples/s +Step 10530/44000 | Loss: 3.1360 | LR: 4.40e-04 | 1274.8ms/step | 15.7 samples/s +Step 10540/44000 | Loss: 3.1469 | LR: 4.39e-04 | 1275.8ms/step | 15.7 samples/s +Step 10550/44000 | Loss: 3.1381 | LR: 4.39e-04 | 1273.8ms/step | 15.7 samples/s +Step 10560/44000 | Loss: 3.1255 | LR: 4.39e-04 | 1274.6ms/step | 15.7 samples/s +Step 10570/44000 | Loss: 3.1403 | LR: 4.39e-04 | 1274.3ms/step | 15.7 samples/s +Step 10580/44000 | Loss: 3.1564 | LR: 4.39e-04 | 1274.7ms/step | 15.7 samples/s +Step 10590/44000 | Loss: 3.1078 | LR: 4.39e-04 | 1274.8ms/step | 15.7 samples/s +Step 10600/44000 | Loss: 3.1276 | LR: 4.39e-04 | 1275.4ms/step | 15.7 samples/s +Step 10610/44000 | Loss: 3.1175 | LR: 4.39e-04 | 1274.9ms/step | 15.7 samples/s +Step 10620/44000 | Loss: 3.1626 | LR: 4.38e-04 | 1275.3ms/step | 15.7 samples/s +Step 10630/44000 | Loss: 3.1524 | LR: 4.38e-04 | 1274.8ms/step | 15.7 samples/s +Step 10640/44000 | Loss: 3.1325 | LR: 4.38e-04 | 1277.3ms/step | 15.7 samples/s +Step 10650/44000 | Loss: 3.1464 | LR: 4.38e-04 | 1275.6ms/step | 15.7 samples/s +Step 10660/44000 | Loss: 3.1332 | LR: 4.38e-04 | 1275.8ms/step | 15.7 samples/s +Step 10670/44000 | Loss: 3.1377 | LR: 4.38e-04 | 1275.2ms/step | 15.7 samples/s +Step 10680/44000 | Loss: 3.1554 | LR: 4.38e-04 | 1274.1ms/step | 15.7 samples/s +Step 10690/44000 | Loss: 3.1059 | LR: 4.38e-04 | 1275.3ms/step | 15.7 samples/s +Step 10700/44000 | Loss: 3.0788 | LR: 4.37e-04 | 1274.8ms/step | 15.7 samples/s +Step 10710/44000 | Loss: 3.1127 | LR: 4.37e-04 | 1274.1ms/step | 15.7 samples/s +Step 10720/44000 | Loss: 3.1009 | LR: 4.37e-04 | 1276.1ms/step | 15.7 samples/s +Step 10730/44000 | Loss: 3.1125 | LR: 4.37e-04 | 1273.2ms/step | 15.7 samples/s +Step 10740/44000 | Loss: 3.1242 | LR: 4.37e-04 | 1273.0ms/step | 15.7 samples/s +Step 10750/44000 | Loss: 3.1160 | LR: 4.37e-04 | 1273.7ms/step | 15.7 samples/s +Validation at step 10750: Loss = 3.1500, Perplexity = 23.34 +Step 10760/44000 | Loss: 3.1533 | LR: 4.37e-04 | 3412.8ms/step | 5.9 samples/s +Step 10770/44000 | Loss: 3.1540 | LR: 4.37e-04 | 1273.3ms/step | 15.7 samples/s +Step 10780/44000 | Loss: 3.1270 | LR: 4.37e-04 | 1272.7ms/step | 15.7 samples/s +Step 10790/44000 | Loss: 3.1354 | LR: 4.36e-04 | 1272.1ms/step | 15.7 samples/s +Step 10800/44000 | Loss: 3.1227 | LR: 4.36e-04 | 1272.4ms/step | 15.7 samples/s +Step 10810/44000 | Loss: 3.1379 | LR: 4.36e-04 | 1273.0ms/step | 15.7 samples/s +Step 10820/44000 | Loss: 3.1471 | LR: 4.36e-04 | 1273.0ms/step | 15.7 samples/s +Step 10830/44000 | Loss: 3.1304 | LR: 4.36e-04 | 1272.6ms/step | 15.7 samples/s +Step 10840/44000 | Loss: 3.1550 | LR: 4.36e-04 | 1273.1ms/step | 15.7 samples/s +Step 10850/44000 | Loss: 3.1488 | LR: 4.36e-04 | 1273.5ms/step | 15.7 samples/s +Step 10860/44000 | Loss: 3.1691 | LR: 4.36e-04 | 1272.9ms/step | 15.7 samples/s +Step 10870/44000 | Loss: 3.1325 | LR: 4.35e-04 | 1272.0ms/step | 15.7 samples/s +Step 10880/44000 | Loss: 3.1284 | LR: 4.35e-04 | 1270.0ms/step | 15.7 samples/s +Step 10890/44000 | Loss: 3.1356 | LR: 4.35e-04 | 1271.2ms/step | 15.7 samples/s +Step 10900/44000 | Loss: 3.1387 | LR: 4.35e-04 | 1272.6ms/step | 15.7 samples/s +Step 10910/44000 | Loss: 3.1153 | LR: 4.35e-04 | 1272.9ms/step | 15.7 samples/s +Step 10920/44000 | Loss: 3.1008 | LR: 4.35e-04 | 1273.4ms/step | 15.7 samples/s +Step 10930/44000 | Loss: 3.1675 | LR: 4.35e-04 | 1273.5ms/step | 15.7 samples/s +Step 10940/44000 | Loss: 3.1357 | LR: 4.35e-04 | 1273.1ms/step | 15.7 samples/s +Step 10950/44000 | Loss: 3.1170 | LR: 4.34e-04 | 1273.4ms/step | 15.7 samples/s +Step 10960/44000 | Loss: 3.1462 | LR: 4.34e-04 | 1273.2ms/step | 15.7 samples/s +Step 10970/44000 | Loss: 3.1300 | LR: 4.34e-04 | 1273.4ms/step | 15.7 samples/s +Step 10980/44000 | Loss: 3.1223 | LR: 4.34e-04 | 1273.1ms/step | 15.7 samples/s +Step 10990/44000 | Loss: 3.1374 | LR: 4.34e-04 | 1272.8ms/step | 15.7 samples/s +Step 11000/44000 | Loss: 3.1290 | LR: 4.34e-04 | 1274.3ms/step | 15.7 samples/s +Validation at step 11000: Loss = 3.1461, Perplexity = 23.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-11000 +Step 11010/44000 | Loss: 3.1447 | LR: 4.34e-04 | 3703.0ms/step | 5.4 samples/s +Step 11020/44000 | Loss: 3.1528 | LR: 4.34e-04 | 1270.5ms/step | 15.7 samples/s +Step 11030/44000 | Loss: 3.0989 | LR: 4.33e-04 | 1271.7ms/step | 15.7 samples/s +Step 11040/44000 | Loss: 3.1417 | LR: 4.33e-04 | 1271.3ms/step | 15.7 samples/s +Step 11050/44000 | Loss: 3.1221 | LR: 4.33e-04 | 1275.0ms/step | 15.7 samples/s +Step 11060/44000 | Loss: 3.1355 | LR: 4.33e-04 | 1274.2ms/step | 15.7 samples/s +Step 11070/44000 | Loss: 3.1301 | LR: 4.33e-04 | 1274.7ms/step | 15.7 samples/s +Step 11080/44000 | Loss: 3.0944 | LR: 4.33e-04 | 1275.0ms/step | 15.7 samples/s +Step 11090/44000 | Loss: 3.1542 | LR: 4.33e-04 | 1273.6ms/step | 15.7 samples/s +Step 11100/44000 | Loss: 3.1289 | LR: 4.33e-04 | 1273.6ms/step | 15.7 samples/s +Step 11110/44000 | Loss: 3.0925 | LR: 4.32e-04 | 1274.3ms/step | 15.7 samples/s +Step 11120/44000 | Loss: 3.1208 | LR: 4.32e-04 | 1273.4ms/step | 15.7 samples/s +Step 11130/44000 | Loss: 3.1176 | LR: 4.32e-04 | 1274.5ms/step | 15.7 samples/s +Step 11140/44000 | Loss: 3.1213 | LR: 4.32e-04 | 1274.2ms/step | 15.7 samples/s +Step 11150/44000 | Loss: 3.1417 | LR: 4.32e-04 | 1274.7ms/step | 15.7 samples/s +Step 11160/44000 | Loss: 3.1250 | LR: 4.32e-04 | 1274.3ms/step | 15.7 samples/s +Step 11170/44000 | Loss: 3.1222 | LR: 4.32e-04 | 1275.2ms/step | 15.7 samples/s +Step 11180/44000 | Loss: 3.1308 | LR: 4.32e-04 | 1277.1ms/step | 15.7 samples/s +Step 11190/44000 | Loss: 3.1271 | LR: 4.31e-04 | 1275.0ms/step | 15.7 samples/s +Step 11200/44000 | Loss: 3.1303 | LR: 4.31e-04 | 1273.5ms/step | 15.7 samples/s +Step 11210/44000 | Loss: 3.1316 | LR: 4.31e-04 | 1272.8ms/step | 15.7 samples/s +Step 11220/44000 | Loss: 3.1065 | LR: 4.31e-04 | 1273.0ms/step | 15.7 samples/s +Step 11230/44000 | Loss: 3.1294 | LR: 4.31e-04 | 1273.3ms/step | 15.7 samples/s +Step 11240/44000 | Loss: 3.1190 | LR: 4.31e-04 | 1272.8ms/step | 15.7 samples/s +Step 11250/44000 | Loss: 3.1251 | LR: 4.31e-04 | 1272.9ms/step | 15.7 samples/s +Validation at step 11250: Loss = 3.1396, Perplexity = 23.09 +Step 11260/44000 | Loss: 3.0998 | LR: 4.31e-04 | 3414.8ms/step | 5.9 samples/s +Step 11270/44000 | Loss: 3.1417 | LR: 4.30e-04 | 1274.4ms/step | 15.7 samples/s +Step 11280/44000 | Loss: 3.1174 | LR: 4.30e-04 | 1274.3ms/step | 15.7 samples/s +Step 11290/44000 | Loss: 3.1280 | LR: 4.30e-04 | 1274.7ms/step | 15.7 samples/s +Step 11300/44000 | Loss: 3.1454 | LR: 4.30e-04 | 1274.2ms/step | 15.7 samples/s +Step 11310/44000 | Loss: 3.1119 | LR: 4.30e-04 | 1274.9ms/step | 15.7 samples/s +Step 11320/44000 | Loss: 3.1261 | LR: 4.30e-04 | 1273.8ms/step | 15.7 samples/s +Step 11330/44000 | Loss: 3.1260 | LR: 4.30e-04 | 1274.8ms/step | 15.7 samples/s +Step 11340/44000 | Loss: 3.0964 | LR: 4.30e-04 | 1274.6ms/step | 15.7 samples/s +Step 11350/44000 | Loss: 3.1244 | LR: 4.29e-04 | 1274.2ms/step | 15.7 samples/s +Step 11360/44000 | Loss: 3.1079 | LR: 4.29e-04 | 1274.4ms/step | 15.7 samples/s +Step 11370/44000 | Loss: 3.1378 | LR: 4.29e-04 | 1273.4ms/step | 15.7 samples/s +Step 11380/44000 | Loss: 3.1363 | LR: 4.29e-04 | 1274.1ms/step | 15.7 samples/s +Step 11390/44000 | Loss: 3.1528 | LR: 4.29e-04 | 1274.8ms/step | 15.7 samples/s +Step 11400/44000 | Loss: 3.1185 | LR: 4.29e-04 | 1275.0ms/step | 15.7 samples/s +Step 11410/44000 | Loss: 3.0869 | LR: 4.29e-04 | 1273.6ms/step | 15.7 samples/s +Step 11420/44000 | Loss: 3.1362 | LR: 4.29e-04 | 1274.4ms/step | 15.7 samples/s +Step 11430/44000 | Loss: 3.1173 | LR: 4.28e-04 | 1274.2ms/step | 15.7 samples/s +Step 11440/44000 | Loss: 3.1421 | LR: 4.28e-04 | 1275.1ms/step | 15.7 samples/s +Step 11450/44000 | Loss: 3.1180 | LR: 4.28e-04 | 1273.6ms/step | 15.7 samples/s +Step 11460/44000 | Loss: 3.1532 | LR: 4.28e-04 | 1273.1ms/step | 15.7 samples/s +Step 11470/44000 | Loss: 3.1274 | LR: 4.28e-04 | 1274.2ms/step | 15.7 samples/s +Step 11480/44000 | Loss: 3.1159 | LR: 4.28e-04 | 1273.9ms/step | 15.7 samples/s +Step 11490/44000 | Loss: 3.1180 | LR: 4.28e-04 | 1275.1ms/step | 15.7 samples/s +Step 11500/44000 | Loss: 3.1199 | LR: 4.28e-04 | 1274.7ms/step | 15.7 samples/s +Validation at step 11500: Loss = 3.1348, Perplexity = 22.98 +Step 11510/44000 | Loss: 3.1353 | LR: 4.27e-04 | 3418.8ms/step | 5.9 samples/s +Step 11520/44000 | Loss: 3.0931 | LR: 4.27e-04 | 1274.1ms/step | 15.7 samples/s +Step 11530/44000 | Loss: 3.1227 | LR: 4.27e-04 | 1275.0ms/step | 15.7 samples/s +Step 11540/44000 | Loss: 3.1217 | LR: 4.27e-04 | 1275.7ms/step | 15.7 samples/s +Step 11550/44000 | Loss: 3.1056 | LR: 4.27e-04 | 1275.3ms/step | 15.7 samples/s +Step 11560/44000 | Loss: 3.1238 | LR: 4.27e-04 | 1275.6ms/step | 15.7 samples/s +Step 11570/44000 | Loss: 3.1236 | LR: 4.27e-04 | 1275.3ms/step | 15.7 samples/s +Step 11580/44000 | Loss: 3.1224 | LR: 4.27e-04 | 1275.0ms/step | 15.7 samples/s +Step 11590/44000 | Loss: 3.1307 | LR: 4.26e-04 | 1275.1ms/step | 15.7 samples/s +Step 11600/44000 | Loss: 3.1167 | LR: 4.26e-04 | 1275.8ms/step | 15.7 samples/s +Step 11610/44000 | Loss: 3.1246 | LR: 4.26e-04 | 1274.9ms/step | 15.7 samples/s +Step 11620/44000 | Loss: 3.1186 | LR: 4.26e-04 | 1274.0ms/step | 15.7 samples/s +Step 11630/44000 | Loss: 3.0975 | LR: 4.26e-04 | 1273.6ms/step | 15.7 samples/s +Step 11640/44000 | Loss: 3.1220 | LR: 4.26e-04 | 1273.5ms/step | 15.7 samples/s +Step 11650/44000 | Loss: 3.1242 | LR: 4.26e-04 | 1273.5ms/step | 15.7 samples/s +Step 11660/44000 | Loss: 3.0933 | LR: 4.26e-04 | 1273.4ms/step | 15.7 samples/s +Step 11670/44000 | Loss: 3.1175 | LR: 4.25e-04 | 1274.0ms/step | 15.7 samples/s +Step 11680/44000 | Loss: 3.0939 | LR: 4.25e-04 | 1274.5ms/step | 15.7 samples/s +Step 11690/44000 | Loss: 3.1492 | LR: 4.25e-04 | 1275.7ms/step | 15.7 samples/s +Step 11700/44000 | Loss: 3.0872 | LR: 4.25e-04 | 1274.6ms/step | 15.7 samples/s +Step 11710/44000 | Loss: 3.1383 | LR: 4.25e-04 | 1273.0ms/step | 15.7 samples/s +Step 11720/44000 | Loss: 3.1304 | LR: 4.25e-04 | 1273.2ms/step | 15.7 samples/s +Step 11730/44000 | Loss: 3.1283 | LR: 4.25e-04 | 1272.0ms/step | 15.7 samples/s +Step 11740/44000 | Loss: 3.1535 | LR: 4.24e-04 | 1272.3ms/step | 15.7 samples/s +Step 11750/44000 | Loss: 3.1152 | LR: 4.24e-04 | 1274.5ms/step | 15.7 samples/s +Validation at step 11750: Loss = 3.1309, Perplexity = 22.89 +Step 11760/44000 | Loss: 3.1495 | LR: 4.24e-04 | 3419.2ms/step | 5.8 samples/s +Step 11770/44000 | Loss: 3.1424 | LR: 4.24e-04 | 1273.2ms/step | 15.7 samples/s +Step 11780/44000 | Loss: 3.1480 | LR: 4.24e-04 | 1274.4ms/step | 15.7 samples/s +Step 11790/44000 | Loss: 3.1115 | LR: 4.24e-04 | 1276.2ms/step | 15.7 samples/s +Step 11800/44000 | Loss: 3.1045 | LR: 4.24e-04 | 1275.6ms/step | 15.7 samples/s +Step 11810/44000 | Loss: 3.0972 | LR: 4.24e-04 | 1274.1ms/step | 15.7 samples/s +Step 11820/44000 | Loss: 3.1251 | LR: 4.23e-04 | 1273.8ms/step | 15.7 samples/s +Step 11830/44000 | Loss: 3.1483 | LR: 4.23e-04 | 1274.2ms/step | 15.7 samples/s +Step 11840/44000 | Loss: 3.0901 | LR: 4.23e-04 | 1275.0ms/step | 15.7 samples/s +Step 11850/44000 | Loss: 3.1297 | LR: 4.23e-04 | 1274.9ms/step | 15.7 samples/s +Step 11860/44000 | Loss: 3.1050 | LR: 4.23e-04 | 1270.7ms/step | 15.7 samples/s +Step 11870/44000 | Loss: 3.1305 | LR: 4.23e-04 | 1268.9ms/step | 15.8 samples/s +Step 11880/44000 | Loss: 3.1143 | LR: 4.23e-04 | 1270.3ms/step | 15.7 samples/s +Step 11890/44000 | Loss: 3.0895 | LR: 4.23e-04 | 1272.6ms/step | 15.7 samples/s +Step 11900/44000 | Loss: 3.1480 | LR: 4.22e-04 | 1273.4ms/step | 15.7 samples/s +Step 11910/44000 | Loss: 3.1182 | LR: 4.22e-04 | 1273.2ms/step | 15.7 samples/s +Step 11920/44000 | Loss: 3.1153 | LR: 4.22e-04 | 1273.5ms/step | 15.7 samples/s +Step 11930/44000 | Loss: 3.1189 | LR: 4.22e-04 | 1274.4ms/step | 15.7 samples/s +Step 11940/44000 | Loss: 3.1060 | LR: 4.22e-04 | 1274.2ms/step | 15.7 samples/s +Step 11950/44000 | Loss: 3.0855 | LR: 4.22e-04 | 1273.9ms/step | 15.7 samples/s +Step 11960/44000 | Loss: 3.1043 | LR: 4.22e-04 | 1275.0ms/step | 15.7 samples/s +Step 11970/44000 | Loss: 3.1343 | LR: 4.21e-04 | 1275.3ms/step | 15.7 samples/s +Step 11980/44000 | Loss: 3.1064 | LR: 4.21e-04 | 1275.4ms/step | 15.7 samples/s +Step 11990/44000 | Loss: 3.1045 | LR: 4.21e-04 | 1275.5ms/step | 15.7 samples/s +Step 12000/44000 | Loss: 3.1210 | LR: 4.21e-04 | 1275.9ms/step | 15.7 samples/s +Validation at step 12000: Loss = 3.1255, Perplexity = 22.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-12000 +Step 12010/44000 | Loss: 3.1055 | LR: 4.21e-04 | 3697.2ms/step | 5.4 samples/s +Step 12020/44000 | Loss: 3.0876 | LR: 4.21e-04 | 1273.7ms/step | 15.7 samples/s +Step 12030/44000 | Loss: 3.1235 | LR: 4.21e-04 | 1274.5ms/step | 15.7 samples/s +Step 12040/44000 | Loss: 3.1266 | LR: 4.21e-04 | 1274.3ms/step | 15.7 samples/s +Step 12050/44000 | Loss: 3.1115 | LR: 4.20e-04 | 1274.7ms/step | 15.7 samples/s +Step 12060/44000 | Loss: 3.1053 | LR: 4.20e-04 | 1276.1ms/step | 15.7 samples/s +Step 12070/44000 | Loss: 3.1450 | LR: 4.20e-04 | 1275.5ms/step | 15.7 samples/s +Step 12080/44000 | Loss: 3.1180 | LR: 4.20e-04 | 1274.1ms/step | 15.7 samples/s +Step 12090/44000 | Loss: 3.1318 | LR: 4.20e-04 | 1274.9ms/step | 15.7 samples/s +Step 12100/44000 | Loss: 3.1234 | LR: 4.20e-04 | 1274.5ms/step | 15.7 samples/s +Step 12110/44000 | Loss: 3.1208 | LR: 4.20e-04 | 1274.6ms/step | 15.7 samples/s +Step 12120/44000 | Loss: 3.1214 | LR: 4.19e-04 | 1274.7ms/step | 15.7 samples/s +Step 12130/44000 | Loss: 3.1353 | LR: 4.19e-04 | 1274.7ms/step | 15.7 samples/s +Step 12140/44000 | Loss: 3.0855 | LR: 4.19e-04 | 1274.6ms/step | 15.7 samples/s +Step 12150/44000 | Loss: 3.1007 | LR: 4.19e-04 | 1274.4ms/step | 15.7 samples/s +Step 12160/44000 | Loss: 3.1327 | LR: 4.19e-04 | 1275.8ms/step | 15.7 samples/s +Step 12170/44000 | Loss: 3.1196 | LR: 4.19e-04 | 1276.5ms/step | 15.7 samples/s +Step 12180/44000 | Loss: 3.0970 | LR: 4.19e-04 | 1275.2ms/step | 15.7 samples/s +Step 12190/44000 | Loss: 3.1119 | LR: 4.19e-04 | 1275.5ms/step | 15.7 samples/s +Step 12200/44000 | Loss: 3.0779 | LR: 4.18e-04 | 1276.0ms/step | 15.7 samples/s +Step 12210/44000 | Loss: 3.1083 | LR: 4.18e-04 | 1275.2ms/step | 15.7 samples/s +Step 12220/44000 | Loss: 3.0839 | LR: 4.18e-04 | 1275.4ms/step | 15.7 samples/s +Step 12230/44000 | Loss: 3.0834 | LR: 4.18e-04 | 1276.2ms/step | 15.7 samples/s +Step 12240/44000 | Loss: 3.1364 | LR: 4.18e-04 | 1275.2ms/step | 15.7 samples/s +Step 12250/44000 | Loss: 3.1185 | LR: 4.18e-04 | 1273.2ms/step | 15.7 samples/s +Validation at step 12250: Loss = 3.1228, Perplexity = 22.71 +Step 12260/44000 | Loss: 3.0953 | LR: 4.18e-04 | 3415.1ms/step | 5.9 samples/s +Step 12270/44000 | Loss: 3.1231 | LR: 4.17e-04 | 1276.4ms/step | 15.7 samples/s +Step 12280/44000 | Loss: 3.1006 | LR: 4.17e-04 | 1275.5ms/step | 15.7 samples/s +Step 12290/44000 | Loss: 3.1191 | LR: 4.17e-04 | 1274.6ms/step | 15.7 samples/s +Step 12300/44000 | Loss: 3.1187 | LR: 4.17e-04 | 1273.6ms/step | 15.7 samples/s +Step 12310/44000 | Loss: 3.1248 | LR: 4.17e-04 | 1274.2ms/step | 15.7 samples/s +Step 12320/44000 | Loss: 3.1035 | LR: 4.17e-04 | 1274.5ms/step | 15.7 samples/s +Step 12330/44000 | Loss: 3.1310 | LR: 4.17e-04 | 1276.1ms/step | 15.7 samples/s +Step 12340/44000 | Loss: 3.1127 | LR: 4.17e-04 | 1276.6ms/step | 15.7 samples/s +Step 12350/44000 | Loss: 3.1296 | LR: 4.16e-04 | 1275.1ms/step | 15.7 samples/s +Step 12360/44000 | Loss: 3.1115 | LR: 4.16e-04 | 1273.4ms/step | 15.7 samples/s +Step 12370/44000 | Loss: 3.1063 | LR: 4.16e-04 | 1271.9ms/step | 15.7 samples/s +Step 12380/44000 | Loss: 3.0775 | LR: 4.16e-04 | 1272.0ms/step | 15.7 samples/s +Step 12390/44000 | Loss: 3.1123 | LR: 4.16e-04 | 1273.5ms/step | 15.7 samples/s +Step 12400/44000 | Loss: 3.0788 | LR: 4.16e-04 | 1273.8ms/step | 15.7 samples/s +Step 12410/44000 | Loss: 3.1020 | LR: 4.16e-04 | 1273.9ms/step | 15.7 samples/s +Step 12420/44000 | Loss: 3.1155 | LR: 4.15e-04 | 1274.9ms/step | 15.7 samples/s +Step 12430/44000 | Loss: 3.0978 | LR: 4.15e-04 | 1274.2ms/step | 15.7 samples/s +Step 12440/44000 | Loss: 3.1104 | LR: 4.15e-04 | 1275.0ms/step | 15.7 samples/s +Step 12450/44000 | Loss: 3.1054 | LR: 4.15e-04 | 1274.6ms/step | 15.7 samples/s +Step 12460/44000 | Loss: 3.1129 | LR: 4.15e-04 | 1275.4ms/step | 15.7 samples/s +Step 12470/44000 | Loss: 3.1040 | LR: 4.15e-04 | 1275.2ms/step | 15.7 samples/s +Step 12480/44000 | Loss: 3.0966 | LR: 4.15e-04 | 1275.7ms/step | 15.7 samples/s +Step 12490/44000 | Loss: 3.0782 | LR: 4.14e-04 | 1277.3ms/step | 15.7 samples/s +Step 12500/44000 | Loss: 3.0730 | LR: 4.14e-04 | 1276.5ms/step | 15.7 samples/s +Validation at step 12500: Loss = 3.1178, Perplexity = 22.60 +Step 12510/44000 | Loss: 3.1281 | LR: 4.14e-04 | 3417.2ms/step | 5.9 samples/s +Step 12520/44000 | Loss: 3.1104 | LR: 4.14e-04 | 1273.9ms/step | 15.7 samples/s +Step 12530/44000 | Loss: 3.1052 | LR: 4.14e-04 | 1273.5ms/step | 15.7 samples/s +Step 12540/44000 | Loss: 3.1052 | LR: 4.14e-04 | 1273.3ms/step | 15.7 samples/s +Step 12550/44000 | Loss: 3.1200 | LR: 4.14e-04 | 1273.5ms/step | 15.7 samples/s +Step 12560/44000 | Loss: 3.0837 | LR: 4.14e-04 | 1273.3ms/step | 15.7 samples/s +Step 12570/44000 | Loss: 3.0960 | LR: 4.13e-04 | 1274.8ms/step | 15.7 samples/s +Step 12580/44000 | Loss: 3.1402 | LR: 4.13e-04 | 1274.0ms/step | 15.7 samples/s +Step 12590/44000 | Loss: 3.0988 | LR: 4.13e-04 | 1274.3ms/step | 15.7 samples/s +Step 12600/44000 | Loss: 3.1051 | LR: 4.13e-04 | 1274.1ms/step | 15.7 samples/s +Step 12610/44000 | Loss: 3.1174 | LR: 4.13e-04 | 1275.0ms/step | 15.7 samples/s +Step 12620/44000 | Loss: 3.1497 | LR: 4.13e-04 | 1275.2ms/step | 15.7 samples/s +Step 12630/44000 | Loss: 3.1111 | LR: 4.13e-04 | 1275.0ms/step | 15.7 samples/s +Step 12640/44000 | Loss: 3.1015 | LR: 4.12e-04 | 1274.9ms/step | 15.7 samples/s +Step 12650/44000 | Loss: 3.1117 | LR: 4.12e-04 | 1274.8ms/step | 15.7 samples/s +Step 12660/44000 | Loss: 3.1002 | LR: 4.12e-04 | 1274.6ms/step | 15.7 samples/s +Step 12670/44000 | Loss: 3.1157 | LR: 4.12e-04 | 1275.0ms/step | 15.7 samples/s +Step 12680/44000 | Loss: 3.1173 | LR: 4.12e-04 | 1274.7ms/step | 15.7 samples/s +Step 12690/44000 | Loss: 3.1064 | LR: 4.12e-04 | 1274.9ms/step | 15.7 samples/s +Step 12700/44000 | Loss: 3.1015 | LR: 4.12e-04 | 1275.6ms/step | 15.7 samples/s +Step 12710/44000 | Loss: 3.0674 | LR: 4.11e-04 | 1274.1ms/step | 15.7 samples/s +Step 12720/44000 | Loss: 3.0670 | LR: 4.11e-04 | 1273.7ms/step | 15.7 samples/s +Step 12730/44000 | Loss: 3.0926 | LR: 4.11e-04 | 1274.8ms/step | 15.7 samples/s +Step 12740/44000 | Loss: 3.1108 | LR: 4.11e-04 | 1275.8ms/step | 15.7 samples/s +Step 12750/44000 | Loss: 3.0726 | LR: 4.11e-04 | 1274.7ms/step | 15.7 samples/s +Validation at step 12750: Loss = 3.1130, Perplexity = 22.49 +Step 12760/44000 | Loss: 3.0988 | LR: 4.11e-04 | 3419.6ms/step | 5.8 samples/s +Step 12770/44000 | Loss: 3.1082 | LR: 4.11e-04 | 1274.3ms/step | 15.7 samples/s +Step 12780/44000 | Loss: 3.0890 | LR: 4.10e-04 | 1274.1ms/step | 15.7 samples/s +Step 12790/44000 | Loss: 3.0927 | LR: 4.10e-04 | 1273.3ms/step | 15.7 samples/s +Step 12800/44000 | Loss: 3.1134 | LR: 4.10e-04 | 1274.4ms/step | 15.7 samples/s +Step 12810/44000 | Loss: 3.1126 | LR: 4.10e-04 | 1274.6ms/step | 15.7 samples/s +Step 12820/44000 | Loss: 3.0906 | LR: 4.10e-04 | 1274.5ms/step | 15.7 samples/s +Step 12830/44000 | Loss: 3.1021 | LR: 4.10e-04 | 1276.5ms/step | 15.7 samples/s +Step 12840/44000 | Loss: 3.0813 | LR: 4.10e-04 | 1273.6ms/step | 15.7 samples/s +Step 12850/44000 | Loss: 3.1013 | LR: 4.09e-04 | 1272.5ms/step | 15.7 samples/s +Step 12860/44000 | Loss: 3.0775 | LR: 4.09e-04 | 1273.4ms/step | 15.7 samples/s +Step 12870/44000 | Loss: 3.1060 | LR: 4.09e-04 | 1273.3ms/step | 15.7 samples/s +Step 12880/44000 | Loss: 3.0679 | LR: 4.09e-04 | 1274.7ms/step | 15.7 samples/s +Step 12890/44000 | Loss: 3.1224 | LR: 4.09e-04 | 1273.1ms/step | 15.7 samples/s +Step 12900/44000 | Loss: 3.0835 | LR: 4.09e-04 | 1273.5ms/step | 15.7 samples/s +Step 12910/44000 | Loss: 3.0992 | LR: 4.09e-04 | 1274.5ms/step | 15.7 samples/s +Step 12920/44000 | Loss: 3.1000 | LR: 4.09e-04 | 1275.3ms/step | 15.7 samples/s +Step 12930/44000 | Loss: 3.1020 | LR: 4.08e-04 | 1273.8ms/step | 15.7 samples/s +Step 12940/44000 | Loss: 3.1152 | LR: 4.08e-04 | 1273.1ms/step | 15.7 samples/s +Step 12950/44000 | Loss: 3.1002 | LR: 4.08e-04 | 1273.9ms/step | 15.7 samples/s +Step 12960/44000 | Loss: 3.1071 | LR: 4.08e-04 | 1273.4ms/step | 15.7 samples/s +Step 12970/44000 | Loss: 3.1143 | LR: 4.08e-04 | 1273.4ms/step | 15.7 samples/s +Step 12980/44000 | Loss: 3.1094 | LR: 4.08e-04 | 1273.9ms/step | 15.7 samples/s +Step 12990/44000 | Loss: 3.1043 | LR: 4.08e-04 | 1275.0ms/step | 15.7 samples/s +Step 13000/44000 | Loss: 3.0904 | LR: 4.07e-04 | 1274.8ms/step | 15.7 samples/s +Validation at step 13000: Loss = 3.1117, Perplexity = 22.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-13000 +Step 13010/44000 | Loss: 3.0967 | LR: 4.07e-04 | 3705.1ms/step | 5.4 samples/s +Step 13020/44000 | Loss: 3.0977 | LR: 4.07e-04 | 1274.5ms/step | 15.7 samples/s +Step 13030/44000 | Loss: 3.1030 | LR: 4.07e-04 | 1273.2ms/step | 15.7 samples/s +Step 13040/44000 | Loss: 3.0774 | LR: 4.07e-04 | 1270.5ms/step | 15.7 samples/s +Step 13050/44000 | Loss: 3.0629 | LR: 4.07e-04 | 1272.5ms/step | 15.7 samples/s +Step 13060/44000 | Loss: 3.1094 | LR: 4.07e-04 | 1273.6ms/step | 15.7 samples/s +Step 13070/44000 | Loss: 3.1015 | LR: 4.06e-04 | 1275.2ms/step | 15.7 samples/s +Step 13080/44000 | Loss: 3.1183 | LR: 4.06e-04 | 1274.7ms/step | 15.7 samples/s +Step 13090/44000 | Loss: 3.0869 | LR: 4.06e-04 | 1275.0ms/step | 15.7 samples/s +Step 13100/44000 | Loss: 3.0902 | LR: 4.06e-04 | 1274.9ms/step | 15.7 samples/s +Step 13110/44000 | Loss: 3.1016 | LR: 4.06e-04 | 1274.6ms/step | 15.7 samples/s +Step 13120/44000 | Loss: 3.1124 | LR: 4.06e-04 | 1274.8ms/step | 15.7 samples/s +Step 13130/44000 | Loss: 3.1128 | LR: 4.06e-04 | 1274.4ms/step | 15.7 samples/s +Step 13140/44000 | Loss: 3.1138 | LR: 4.05e-04 | 1275.0ms/step | 15.7 samples/s +Step 13150/44000 | Loss: 3.0949 | LR: 4.05e-04 | 1274.6ms/step | 15.7 samples/s +Step 13160/44000 | Loss: 3.0871 | LR: 4.05e-04 | 1274.5ms/step | 15.7 samples/s +Step 13170/44000 | Loss: 3.0920 | LR: 4.05e-04 | 1275.7ms/step | 15.7 samples/s +Step 13180/44000 | Loss: 3.1142 | LR: 4.05e-04 | 1274.4ms/step | 15.7 samples/s +Step 13190/44000 | Loss: 3.0894 | LR: 4.05e-04 | 1275.9ms/step | 15.7 samples/s +Step 13200/44000 | Loss: 3.1232 | LR: 4.05e-04 | 1275.5ms/step | 15.7 samples/s +Step 13210/44000 | Loss: 3.0773 | LR: 4.04e-04 | 1274.7ms/step | 15.7 samples/s +Step 13220/44000 | Loss: 3.0800 | LR: 4.04e-04 | 1274.0ms/step | 15.7 samples/s +Step 13230/44000 | Loss: 3.1097 | LR: 4.04e-04 | 1274.9ms/step | 15.7 samples/s +Step 13240/44000 | Loss: 3.0901 | LR: 4.04e-04 | 1273.2ms/step | 15.7 samples/s +Step 13250/44000 | Loss: 3.1075 | LR: 4.04e-04 | 1274.5ms/step | 15.7 samples/s +Validation at step 13250: Loss = 3.1056, Perplexity = 22.32 +Step 13260/44000 | Loss: 3.0986 | LR: 4.04e-04 | 3418.8ms/step | 5.8 samples/s +Step 13270/44000 | Loss: 3.0882 | LR: 4.04e-04 | 1275.8ms/step | 15.7 samples/s +Step 13280/44000 | Loss: 3.1154 | LR: 4.03e-04 | 1275.4ms/step | 15.7 samples/s +Step 13290/44000 | Loss: 3.1003 | LR: 4.03e-04 | 1274.6ms/step | 15.7 samples/s +Step 13300/44000 | Loss: 3.0624 | LR: 4.03e-04 | 1274.6ms/step | 15.7 samples/s +Step 13310/44000 | Loss: 3.0943 | LR: 4.03e-04 | 1275.4ms/step | 15.7 samples/s +Step 13320/44000 | Loss: 3.0759 | LR: 4.03e-04 | 1275.6ms/step | 15.7 samples/s +Step 13330/44000 | Loss: 3.0833 | LR: 4.03e-04 | 1276.0ms/step | 15.7 samples/s +Step 13340/44000 | Loss: 3.0909 | LR: 4.03e-04 | 1276.1ms/step | 15.7 samples/s +Step 13350/44000 | Loss: 3.0779 | LR: 4.02e-04 | 1274.4ms/step | 15.7 samples/s +Step 13360/44000 | Loss: 3.0920 | LR: 4.02e-04 | 1273.8ms/step | 15.7 samples/s +Step 13370/44000 | Loss: 3.1221 | LR: 4.02e-04 | 1274.0ms/step | 15.7 samples/s +Step 13380/44000 | Loss: 3.1140 | LR: 4.02e-04 | 1274.7ms/step | 15.7 samples/s +Step 13390/44000 | Loss: 3.0828 | LR: 4.02e-04 | 1274.2ms/step | 15.7 samples/s +Step 13400/44000 | Loss: 3.0837 | LR: 4.02e-04 | 1274.5ms/step | 15.7 samples/s +Step 13410/44000 | Loss: 3.0758 | LR: 4.02e-04 | 1273.4ms/step | 15.7 samples/s +Step 13420/44000 | Loss: 3.0471 | LR: 4.01e-04 | 1274.8ms/step | 15.7 samples/s +Step 13430/44000 | Loss: 3.1026 | LR: 4.01e-04 | 1274.5ms/step | 15.7 samples/s +Step 13440/44000 | Loss: 3.0927 | LR: 4.01e-04 | 1273.3ms/step | 15.7 samples/s +Step 13450/44000 | Loss: 3.1028 | LR: 4.01e-04 | 1274.3ms/step | 15.7 samples/s +Step 13460/44000 | Loss: 3.0815 | LR: 4.01e-04 | 1274.3ms/step | 15.7 samples/s +Step 13470/44000 | Loss: 3.0925 | LR: 4.01e-04 | 1273.7ms/step | 15.7 samples/s +Step 13480/44000 | Loss: 3.0686 | LR: 4.01e-04 | 1273.7ms/step | 15.7 samples/s +Step 13490/44000 | Loss: 3.0945 | LR: 4.00e-04 | 1273.0ms/step | 15.7 samples/s +Step 13500/44000 | Loss: 3.0962 | LR: 4.00e-04 | 1275.2ms/step | 15.7 samples/s +Validation at step 13500: Loss = 3.1009, Perplexity = 22.22 +Step 13510/44000 | Loss: 3.0819 | LR: 4.00e-04 | 3416.8ms/step | 5.9 samples/s +Step 13520/44000 | Loss: 3.0791 | LR: 4.00e-04 | 1271.9ms/step | 15.7 samples/s +Step 13530/44000 | Loss: 3.0766 | LR: 4.00e-04 | 1272.1ms/step | 15.7 samples/s +Step 13540/44000 | Loss: 3.0681 | LR: 4.00e-04 | 1271.8ms/step | 15.7 samples/s +Step 13550/44000 | Loss: 3.0865 | LR: 4.00e-04 | 1274.0ms/step | 15.7 samples/s +Step 13560/44000 | Loss: 3.0781 | LR: 3.99e-04 | 1273.7ms/step | 15.7 samples/s +Step 13570/44000 | Loss: 3.1145 | LR: 3.99e-04 | 1273.2ms/step | 15.7 samples/s +Step 13580/44000 | Loss: 3.0933 | LR: 3.99e-04 | 1273.9ms/step | 15.7 samples/s +Step 13590/44000 | Loss: 3.0709 | LR: 3.99e-04 | 1274.7ms/step | 15.7 samples/s +Step 13600/44000 | Loss: 3.0826 | LR: 3.99e-04 | 1273.1ms/step | 15.7 samples/s +Step 13610/44000 | Loss: 3.0726 | LR: 3.99e-04 | 1272.7ms/step | 15.7 samples/s +Step 13620/44000 | Loss: 3.1050 | LR: 3.98e-04 | 1274.7ms/step | 15.7 samples/s +Step 13630/44000 | Loss: 3.0947 | LR: 3.98e-04 | 1273.5ms/step | 15.7 samples/s +Step 13640/44000 | Loss: 3.0722 | LR: 3.98e-04 | 1273.5ms/step | 15.7 samples/s +Step 13650/44000 | Loss: 3.0896 | LR: 3.98e-04 | 1272.9ms/step | 15.7 samples/s +Step 13660/44000 | Loss: 3.0763 | LR: 3.98e-04 | 1273.3ms/step | 15.7 samples/s +Step 13670/44000 | Loss: 3.0620 | LR: 3.98e-04 | 1273.7ms/step | 15.7 samples/s +Step 13680/44000 | Loss: 3.0776 | LR: 3.98e-04 | 1271.4ms/step | 15.7 samples/s +Step 13690/44000 | Loss: 3.0951 | LR: 3.97e-04 | 1273.1ms/step | 15.7 samples/s +Step 13700/44000 | Loss: 3.0803 | LR: 3.97e-04 | 1273.3ms/step | 15.7 samples/s +Step 13710/44000 | Loss: 3.0895 | LR: 3.97e-04 | 1274.9ms/step | 15.7 samples/s +Step 13720/44000 | Loss: 3.0724 | LR: 3.97e-04 | 1274.7ms/step | 15.7 samples/s +Step 13730/44000 | Loss: 3.0894 | LR: 3.97e-04 | 1273.7ms/step | 15.7 samples/s +Step 13740/44000 | Loss: 3.0960 | LR: 3.97e-04 | 1274.6ms/step | 15.7 samples/s +Step 13750/44000 | Loss: 3.1267 | LR: 3.97e-04 | 1274.6ms/step | 15.7 samples/s +Validation at step 13750: Loss = 3.0984, Perplexity = 22.16 +Step 13760/44000 | Loss: 3.1157 | LR: 3.96e-04 | 3418.5ms/step | 5.9 samples/s +Step 13770/44000 | Loss: 3.0860 | LR: 3.96e-04 | 1274.4ms/step | 15.7 samples/s +Step 13780/44000 | Loss: 3.0766 | LR: 3.96e-04 | 1274.1ms/step | 15.7 samples/s +Step 13790/44000 | Loss: 3.1157 | LR: 3.96e-04 | 1274.9ms/step | 15.7 samples/s +Step 13800/44000 | Loss: 3.0729 | LR: 3.96e-04 | 1275.2ms/step | 15.7 samples/s +Step 13810/44000 | Loss: 3.0681 | LR: 3.96e-04 | 1274.9ms/step | 15.7 samples/s +Step 13820/44000 | Loss: 3.0830 | LR: 3.96e-04 | 1274.8ms/step | 15.7 samples/s +Step 13830/44000 | Loss: 3.0664 | LR: 3.95e-04 | 1273.8ms/step | 15.7 samples/s +Step 13840/44000 | Loss: 3.0586 | LR: 3.95e-04 | 1273.0ms/step | 15.7 samples/s +Step 13850/44000 | Loss: 3.0779 | LR: 3.95e-04 | 1273.0ms/step | 15.7 samples/s +Step 13860/44000 | Loss: 3.0857 | LR: 3.95e-04 | 1273.2ms/step | 15.7 samples/s +Step 13870/44000 | Loss: 3.0839 | LR: 3.95e-04 | 1273.6ms/step | 15.7 samples/s +Step 13880/44000 | Loss: 3.1011 | LR: 3.95e-04 | 1273.2ms/step | 15.7 samples/s +Step 13890/44000 | Loss: 3.0685 | LR: 3.95e-04 | 1274.0ms/step | 15.7 samples/s +Step 13900/44000 | Loss: 3.1187 | LR: 3.94e-04 | 1274.8ms/step | 15.7 samples/s +Step 13910/44000 | Loss: 3.0911 | LR: 3.94e-04 | 1273.3ms/step | 15.7 samples/s +Step 13920/44000 | Loss: 3.0722 | LR: 3.94e-04 | 1274.4ms/step | 15.7 samples/s +Step 13930/44000 | Loss: 3.0845 | LR: 3.94e-04 | 1273.5ms/step | 15.7 samples/s +Step 13940/44000 | Loss: 3.0652 | LR: 3.94e-04 | 1274.4ms/step | 15.7 samples/s +Step 13950/44000 | Loss: 3.0658 | LR: 3.94e-04 | 1275.4ms/step | 15.7 samples/s +Step 13960/44000 | Loss: 3.0859 | LR: 3.93e-04 | 1275.3ms/step | 15.7 samples/s +Step 13970/44000 | Loss: 3.0699 | LR: 3.93e-04 | 1274.2ms/step | 15.7 samples/s +Step 13980/44000 | Loss: 3.0698 | LR: 3.93e-04 | 1273.8ms/step | 15.7 samples/s +Step 13990/44000 | Loss: 3.0702 | LR: 3.93e-04 | 1274.1ms/step | 15.7 samples/s +Step 14000/44000 | Loss: 3.0900 | LR: 3.93e-04 | 1274.2ms/step | 15.7 samples/s +Validation at step 14000: Loss = 3.0939, Perplexity = 22.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-14000 +Step 14010/44000 | Loss: 3.0929 | LR: 3.93e-04 | 3700.3ms/step | 5.4 samples/s +Step 14020/44000 | Loss: 3.0661 | LR: 3.93e-04 | 1273.6ms/step | 15.7 samples/s +Step 14030/44000 | Loss: 3.0346 | LR: 3.92e-04 | 1273.9ms/step | 15.7 samples/s +Step 14040/44000 | Loss: 3.0523 | LR: 3.92e-04 | 1274.0ms/step | 15.7 samples/s +Step 14050/44000 | Loss: 3.0438 | LR: 3.92e-04 | 1274.1ms/step | 15.7 samples/s +Step 14060/44000 | Loss: 3.1000 | LR: 3.92e-04 | 1274.2ms/step | 15.7 samples/s +Step 14070/44000 | Loss: 3.0920 | LR: 3.92e-04 | 1277.7ms/step | 15.7 samples/s +Step 14080/44000 | Loss: 3.0823 | LR: 3.92e-04 | 1279.9ms/step | 15.6 samples/s +Step 14090/44000 | Loss: 3.0656 | LR: 3.92e-04 | 1279.5ms/step | 15.6 samples/s +Step 14100/44000 | Loss: 3.0936 | LR: 3.91e-04 | 1277.2ms/step | 15.7 samples/s +Step 14110/44000 | Loss: 3.0878 | LR: 3.91e-04 | 1277.3ms/step | 15.7 samples/s +Step 14120/44000 | Loss: 3.0809 | LR: 3.91e-04 | 1279.1ms/step | 15.6 samples/s +Step 14130/44000 | Loss: 3.1041 | LR: 3.91e-04 | 1278.6ms/step | 15.6 samples/s +Step 14140/44000 | Loss: 3.0902 | LR: 3.91e-04 | 1279.5ms/step | 15.6 samples/s +Step 14150/44000 | Loss: 3.0601 | LR: 3.91e-04 | 1278.5ms/step | 15.6 samples/s +Step 14160/44000 | Loss: 3.0731 | LR: 3.90e-04 | 1279.3ms/step | 15.6 samples/s +Step 14170/44000 | Loss: 3.0957 | LR: 3.90e-04 | 1278.0ms/step | 15.6 samples/s +Step 14180/44000 | Loss: 3.0845 | LR: 3.90e-04 | 1277.5ms/step | 15.7 samples/s +Step 14190/44000 | Loss: 3.0788 | LR: 3.90e-04 | 1277.6ms/step | 15.7 samples/s +Step 14200/44000 | Loss: 3.0891 | LR: 3.90e-04 | 1278.3ms/step | 15.6 samples/s +Step 14210/44000 | Loss: 3.0746 | LR: 3.90e-04 | 1277.8ms/step | 15.7 samples/s +Step 14220/44000 | Loss: 3.0742 | LR: 3.90e-04 | 1277.4ms/step | 15.7 samples/s +Step 14230/44000 | Loss: 3.0789 | LR: 3.89e-04 | 1278.0ms/step | 15.6 samples/s +Step 14240/44000 | Loss: 3.0597 | LR: 3.89e-04 | 1277.1ms/step | 15.7 samples/s +Step 14250/44000 | Loss: 3.0742 | LR: 3.89e-04 | 1278.1ms/step | 15.6 samples/s +Validation at step 14250: Loss = 3.0906, Perplexity = 21.99 +Step 14260/44000 | Loss: 3.1054 | LR: 3.89e-04 | 3425.4ms/step | 5.8 samples/s +Step 14270/44000 | Loss: 3.0990 | LR: 3.89e-04 | 1276.9ms/step | 15.7 samples/s +Step 14280/44000 | Loss: 3.0954 | LR: 3.89e-04 | 1278.0ms/step | 15.6 samples/s +Step 14290/44000 | Loss: 3.0817 | LR: 3.89e-04 | 1277.9ms/step | 15.7 samples/s +Step 14300/44000 | Loss: 3.0812 | LR: 3.88e-04 | 1278.3ms/step | 15.6 samples/s +Step 14310/44000 | Loss: 3.0799 | LR: 3.88e-04 | 1278.1ms/step | 15.6 samples/s +Step 14320/44000 | Loss: 3.0778 | LR: 3.88e-04 | 1277.5ms/step | 15.7 samples/s +Step 14330/44000 | Loss: 3.0911 | LR: 3.88e-04 | 1277.7ms/step | 15.7 samples/s +Step 14340/44000 | Loss: 3.0892 | LR: 3.88e-04 | 1276.4ms/step | 15.7 samples/s +Step 14350/44000 | Loss: 3.0730 | LR: 3.88e-04 | 1276.8ms/step | 15.7 samples/s +Step 14360/44000 | Loss: 3.0886 | LR: 3.87e-04 | 1277.6ms/step | 15.7 samples/s +Step 14370/44000 | Loss: 3.0866 | LR: 3.87e-04 | 1278.0ms/step | 15.6 samples/s +Step 14380/44000 | Loss: 3.0679 | LR: 3.87e-04 | 1277.7ms/step | 15.7 samples/s +Step 14390/44000 | Loss: 3.0731 | LR: 3.87e-04 | 1278.1ms/step | 15.6 samples/s +Step 14400/44000 | Loss: 3.0757 | LR: 3.87e-04 | 1278.2ms/step | 15.6 samples/s +Step 14410/44000 | Loss: 3.0554 | LR: 3.87e-04 | 1278.8ms/step | 15.6 samples/s +Step 14420/44000 | Loss: 3.0791 | LR: 3.87e-04 | 1278.7ms/step | 15.6 samples/s +Step 14430/44000 | Loss: 3.0449 | LR: 3.86e-04 | 1277.9ms/step | 15.7 samples/s +Step 14440/44000 | Loss: 3.0857 | LR: 3.86e-04 | 1278.6ms/step | 15.6 samples/s +Step 14450/44000 | Loss: 3.0765 | LR: 3.86e-04 | 1277.3ms/step | 15.7 samples/s +Step 14460/44000 | Loss: 3.0889 | LR: 3.86e-04 | 1276.5ms/step | 15.7 samples/s +Step 14470/44000 | Loss: 3.0849 | LR: 3.86e-04 | 1278.1ms/step | 15.6 samples/s +Step 14480/44000 | Loss: 3.0491 | LR: 3.86e-04 | 1278.4ms/step | 15.6 samples/s +Step 14490/44000 | Loss: 3.0635 | LR: 3.85e-04 | 1276.9ms/step | 15.7 samples/s +Step 14500/44000 | Loss: 3.0904 | LR: 3.85e-04 | 1276.9ms/step | 15.7 samples/s +Validation at step 14500: Loss = 3.0851, Perplexity = 21.87 +Step 14510/44000 | Loss: 3.0588 | LR: 3.85e-04 | 3425.1ms/step | 5.8 samples/s +Step 14520/44000 | Loss: 3.0927 | LR: 3.85e-04 | 1277.7ms/step | 15.7 samples/s +Step 14530/44000 | Loss: 3.0705 | LR: 3.85e-04 | 1278.2ms/step | 15.6 samples/s +Step 14540/44000 | Loss: 3.0788 | LR: 3.85e-04 | 1277.6ms/step | 15.7 samples/s +Step 14550/44000 | Loss: 3.0776 | LR: 3.85e-04 | 1278.8ms/step | 15.6 samples/s +Step 14560/44000 | Loss: 3.0867 | LR: 3.84e-04 | 1279.6ms/step | 15.6 samples/s +Step 14570/44000 | Loss: 3.0865 | LR: 3.84e-04 | 1278.0ms/step | 15.6 samples/s +Step 14580/44000 | Loss: 3.0736 | LR: 3.84e-04 | 1277.7ms/step | 15.7 samples/s +Step 14590/44000 | Loss: 3.0840 | LR: 3.84e-04 | 1278.7ms/step | 15.6 samples/s +Step 14600/44000 | Loss: 3.0782 | LR: 3.84e-04 | 1277.5ms/step | 15.7 samples/s +Step 14610/44000 | Loss: 3.0768 | LR: 3.84e-04 | 1277.6ms/step | 15.7 samples/s +Step 14620/44000 | Loss: 3.0895 | LR: 3.83e-04 | 1276.4ms/step | 15.7 samples/s +Step 14630/44000 | Loss: 3.0650 | LR: 3.83e-04 | 1277.8ms/step | 15.7 samples/s +Step 14640/44000 | Loss: 3.0644 | LR: 3.83e-04 | 1278.6ms/step | 15.6 samples/s +Step 14650/44000 | Loss: 3.0743 | LR: 3.83e-04 | 1278.0ms/step | 15.6 samples/s +Step 14660/44000 | Loss: 3.0891 | LR: 3.83e-04 | 1278.8ms/step | 15.6 samples/s +Step 14670/44000 | Loss: 3.0823 | LR: 3.83e-04 | 1279.1ms/step | 15.6 samples/s +Step 14680/44000 | Loss: 3.0492 | LR: 3.83e-04 | 1276.8ms/step | 15.7 samples/s +Step 14690/44000 | Loss: 3.0681 | LR: 3.82e-04 | 1273.6ms/step | 15.7 samples/s +Step 14700/44000 | Loss: 3.0852 | LR: 3.82e-04 | 1274.3ms/step | 15.7 samples/s +Step 14710/44000 | Loss: 3.0742 | LR: 3.82e-04 | 1274.5ms/step | 15.7 samples/s +Step 14720/44000 | Loss: 3.0641 | LR: 3.82e-04 | 1275.5ms/step | 15.7 samples/s +Step 14730/44000 | Loss: 3.0795 | LR: 3.82e-04 | 1278.2ms/step | 15.6 samples/s +Step 14740/44000 | Loss: 3.0686 | LR: 3.82e-04 | 1279.3ms/step | 15.6 samples/s +Step 14750/44000 | Loss: 3.0961 | LR: 3.81e-04 | 1277.1ms/step | 15.7 samples/s +Validation at step 14750: Loss = 3.0830, Perplexity = 21.82 +Step 14760/44000 | Loss: 3.0685 | LR: 3.81e-04 | 3427.5ms/step | 5.8 samples/s +Step 14770/44000 | Loss: 3.0815 | LR: 3.81e-04 | 1278.6ms/step | 15.6 samples/s +Step 14780/44000 | Loss: 3.0918 | LR: 3.81e-04 | 1279.4ms/step | 15.6 samples/s +Step 14790/44000 | Loss: 3.0744 | LR: 3.81e-04 | 1278.2ms/step | 15.6 samples/s +Step 14800/44000 | Loss: 3.0436 | LR: 3.81e-04 | 1278.2ms/step | 15.6 samples/s +Step 14810/44000 | Loss: 3.0621 | LR: 3.81e-04 | 1276.2ms/step | 15.7 samples/s +Step 14820/44000 | Loss: 3.0727 | LR: 3.80e-04 | 1271.4ms/step | 15.7 samples/s +Step 14830/44000 | Loss: 3.0769 | LR: 3.80e-04 | 1272.2ms/step | 15.7 samples/s +Step 14840/44000 | Loss: 3.0650 | LR: 3.80e-04 | 1273.0ms/step | 15.7 samples/s +Step 14850/44000 | Loss: 3.0754 | LR: 3.80e-04 | 1274.3ms/step | 15.7 samples/s +Step 14860/44000 | Loss: 3.0573 | LR: 3.80e-04 | 1273.1ms/step | 15.7 samples/s +Step 14870/44000 | Loss: 3.0601 | LR: 3.80e-04 | 1273.9ms/step | 15.7 samples/s +Step 14880/44000 | Loss: 3.0567 | LR: 3.79e-04 | 1275.0ms/step | 15.7 samples/s +Step 14890/44000 | Loss: 3.0570 | LR: 3.79e-04 | 1274.8ms/step | 15.7 samples/s +Step 14900/44000 | Loss: 3.0662 | LR: 3.79e-04 | 1274.0ms/step | 15.7 samples/s +Step 14910/44000 | Loss: 3.0857 | LR: 3.79e-04 | 1273.4ms/step | 15.7 samples/s +Step 14920/44000 | Loss: 3.0844 | LR: 3.79e-04 | 1274.5ms/step | 15.7 samples/s +Step 14930/44000 | Loss: 3.0888 | LR: 3.79e-04 | 1274.2ms/step | 15.7 samples/s +Step 14940/44000 | Loss: 3.0796 | LR: 3.79e-04 | 1273.6ms/step | 15.7 samples/s +Step 14950/44000 | Loss: 3.1059 | LR: 3.78e-04 | 1274.2ms/step | 15.7 samples/s diff --git a/epfl_final/logs/l8k365m_fixed_s1_resume.log b/epfl_final/logs/l8k365m_fixed_s1_resume.log new file mode 100644 index 0000000000000000000000000000000000000000..956798157a91565a37453a07a65c786b35847688 --- /dev/null +++ b/epfl_final/logs/l8k365m_fixed_s1_resume.log @@ -0,0 +1,3466 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-14000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m fixed s1 8k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "fixed", + "seed1" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_fixed_s1", + "sparse_mask_seed": 1, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-14000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-14000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 14000 (epoch 0, micro_step 140000) +Resuming training from step 14000 +Resuming wandb run dbtl6d2o +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run dbtl6d2o +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_170941-dbtl6d2o +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m fixed s1 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/dbtl6d2o +Starting training... +Validation at step 14000: Loss = 3.0939, Perplexity = 22.06 +wandb: WARNING Tried to log to step 14000 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 14001: sparse_stochastic (configured sparse attention) +Step 14010/44000 | Loss: 3.0717 | LR: 3.93e-04 | 36272.6ms/step | 0.6 samples/s +wandb: WARNING Tried to log to step 14010 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14020/44000 | Loss: 3.0844 | LR: 3.93e-04 | 1272.9ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14020 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14030/44000 | Loss: 3.0550 | LR: 3.92e-04 | 1275.7ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14030 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14040/44000 | Loss: 3.1056 | LR: 3.92e-04 | 1275.3ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14040 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14050/44000 | Loss: 3.0706 | LR: 3.92e-04 | 1277.0ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14050 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14060/44000 | Loss: 3.1171 | LR: 3.92e-04 | 1278.5ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14060 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14070/44000 | Loss: 3.1076 | LR: 3.92e-04 | 1276.0ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14070 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14080/44000 | Loss: 3.0844 | LR: 3.92e-04 | 1277.2ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14080 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14090/44000 | Loss: 3.0508 | LR: 3.92e-04 | 1276.4ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14090 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14100/44000 | Loss: 3.0672 | LR: 3.91e-04 | 1277.1ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14100 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14110/44000 | Loss: 3.0404 | LR: 3.91e-04 | 1276.4ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14110 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14120/44000 | Loss: 3.0622 | LR: 3.91e-04 | 1276.1ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14120 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14130/44000 | Loss: 3.0956 | LR: 3.91e-04 | 1276.7ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14130 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14140/44000 | Loss: 3.0628 | LR: 3.91e-04 | 1278.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14140 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14150/44000 | Loss: 3.0654 | LR: 3.91e-04 | 1278.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14150 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14160/44000 | Loss: 3.0623 | LR: 3.90e-04 | 1275.9ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14160 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14170/44000 | Loss: 3.1063 | LR: 3.90e-04 | 1276.5ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14170 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14180/44000 | Loss: 3.0581 | LR: 3.90e-04 | 1276.6ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14180 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14190/44000 | Loss: 3.0919 | LR: 3.90e-04 | 1276.3ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14190 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14200/44000 | Loss: 3.1001 | LR: 3.90e-04 | 1273.8ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14200 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14210/44000 | Loss: 3.0687 | LR: 3.90e-04 | 1275.4ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14210 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14220/44000 | Loss: 3.0570 | LR: 3.90e-04 | 1275.4ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14220 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14230/44000 | Loss: 3.0769 | LR: 3.89e-04 | 1277.0ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14230 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14240/44000 | Loss: 3.0517 | LR: 3.89e-04 | 1277.7ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14240 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14250/44000 | Loss: 3.0713 | LR: 3.89e-04 | 1276.7ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14250 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 14250: Loss = 3.0902, Perplexity = 21.98 +Step 14260/44000 | Loss: 3.0478 | LR: 3.89e-04 | 3448.4ms/step | 5.8 samples/s +wandb: WARNING Tried to log to step 14260 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14270/44000 | Loss: 3.0643 | LR: 3.89e-04 | 1293.0ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14270 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14280/44000 | Loss: 3.0700 | LR: 3.89e-04 | 1292.1ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14280 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14290/44000 | Loss: 3.0583 | LR: 3.89e-04 | 1291.0ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14290 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14300/44000 | Loss: 3.0548 | LR: 3.88e-04 | 1290.0ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14300 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14310/44000 | Loss: 3.1147 | LR: 3.88e-04 | 1284.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14310 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14320/44000 | Loss: 3.0778 | LR: 3.88e-04 | 1287.2ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14320 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14330/44000 | Loss: 3.0972 | LR: 3.88e-04 | 1285.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14330 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14340/44000 | Loss: 3.0649 | LR: 3.88e-04 | 1287.4ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14340 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14350/44000 | Loss: 3.0928 | LR: 3.88e-04 | 1294.1ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14350 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14360/44000 | Loss: 3.0692 | LR: 3.87e-04 | 1285.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14360 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14370/44000 | Loss: 3.0663 | LR: 3.87e-04 | 1284.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14370 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14380/44000 | Loss: 3.0440 | LR: 3.87e-04 | 1286.7ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14380 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14390/44000 | Loss: 3.0679 | LR: 3.87e-04 | 1294.9ms/step | 15.4 samples/s +wandb: WARNING Tried to log to step 14390 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14400/44000 | Loss: 3.0395 | LR: 3.87e-04 | 1285.5ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14400 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14410/44000 | Loss: 3.0615 | LR: 3.87e-04 | 1288.4ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14410 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14420/44000 | Loss: 3.0514 | LR: 3.87e-04 | 1281.7ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14420 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14430/44000 | Loss: 3.0657 | LR: 3.86e-04 | 1283.7ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14430 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14440/44000 | Loss: 3.0541 | LR: 3.86e-04 | 1282.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14440 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14450/44000 | Loss: 3.0715 | LR: 3.86e-04 | 1286.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14450 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14460/44000 | Loss: 3.0462 | LR: 3.86e-04 | 1286.4ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14460 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14470/44000 | Loss: 3.0909 | LR: 3.86e-04 | 1290.5ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14470 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14480/44000 | Loss: 3.0689 | LR: 3.86e-04 | 1288.5ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14480 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14490/44000 | Loss: 3.0710 | LR: 3.85e-04 | 1286.3ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14490 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14500/44000 | Loss: 3.0812 | LR: 3.85e-04 | 1285.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14500 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 14500: Loss = 3.0864, Perplexity = 21.90 +Step 14510/44000 | Loss: 3.0841 | LR: 3.85e-04 | 3434.0ms/step | 5.8 samples/s +wandb: WARNING Tried to log to step 14510 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14520/44000 | Loss: 3.0567 | LR: 3.85e-04 | 1285.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14520 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14530/44000 | Loss: 3.0839 | LR: 3.85e-04 | 1285.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14530 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14540/44000 | Loss: 3.0747 | LR: 3.85e-04 | 1285.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14540 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14550/44000 | Loss: 3.0472 | LR: 3.85e-04 | 1285.7ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14550 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14560/44000 | Loss: 3.0937 | LR: 3.84e-04 | 1284.2ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14560 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14570/44000 | Loss: 3.0575 | LR: 3.84e-04 | 1283.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14570 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14580/44000 | Loss: 3.0653 | LR: 3.84e-04 | 1283.3ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14580 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14590/44000 | Loss: 3.0659 | LR: 3.84e-04 | 1282.3ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14590 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14600/44000 | Loss: 3.0364 | LR: 3.84e-04 | 1281.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14600 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14610/44000 | Loss: 3.0661 | LR: 3.84e-04 | 1285.5ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14610 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14620/44000 | Loss: 3.1025 | LR: 3.83e-04 | 1286.3ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14620 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14630/44000 | Loss: 3.0533 | LR: 3.83e-04 | 1287.4ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14630 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14640/44000 | Loss: 3.0508 | LR: 3.83e-04 | 1285.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14640 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14650/44000 | Loss: 3.0715 | LR: 3.83e-04 | 1286.5ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14650 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14660/44000 | Loss: 3.0672 | LR: 3.83e-04 | 1284.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14660 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14670/44000 | Loss: 3.0565 | LR: 3.83e-04 | 1283.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14670 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14680/44000 | Loss: 3.0802 | LR: 3.83e-04 | 1284.7ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14680 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14690/44000 | Loss: 3.0857 | LR: 3.82e-04 | 1283.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14690 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14700/44000 | Loss: 3.0545 | LR: 3.82e-04 | 1283.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14700 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14710/44000 | Loss: 3.0851 | LR: 3.82e-04 | 1282.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14710 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14720/44000 | Loss: 3.0832 | LR: 3.82e-04 | 1284.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14720 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14730/44000 | Loss: 3.0736 | LR: 3.82e-04 | 1288.6ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14730 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14740/44000 | Loss: 3.0767 | LR: 3.82e-04 | 1287.0ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14740 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14750/44000 | Loss: 3.0343 | LR: 3.81e-04 | 1283.5ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14750 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 14750: Loss = 3.0829, Perplexity = 21.82 +Step 14760/44000 | Loss: 3.0810 | LR: 3.81e-04 | 3434.1ms/step | 5.8 samples/s +wandb: WARNING Tried to log to step 14760 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14770/44000 | Loss: 3.0461 | LR: 3.81e-04 | 1283.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14770 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14780/44000 | Loss: 3.0425 | LR: 3.81e-04 | 1283.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14780 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14790/44000 | Loss: 3.0605 | LR: 3.81e-04 | 1283.2ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14790 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14800/44000 | Loss: 3.0806 | LR: 3.81e-04 | 1282.2ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14800 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14810/44000 | Loss: 3.0416 | LR: 3.81e-04 | 1279.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14810 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14820/44000 | Loss: 3.0753 | LR: 3.80e-04 | 1281.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14820 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14830/44000 | Loss: 3.0604 | LR: 3.80e-04 | 1275.4ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14830 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14840/44000 | Loss: 3.0660 | LR: 3.80e-04 | 1276.8ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14840 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14850/44000 | Loss: 3.0640 | LR: 3.80e-04 | 1280.7ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14850 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14860/44000 | Loss: 3.0384 | LR: 3.80e-04 | 1286.3ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14860 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14870/44000 | Loss: 3.0396 | LR: 3.80e-04 | 1285.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14870 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14880/44000 | Loss: 3.0364 | LR: 3.79e-04 | 1286.5ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14880 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14890/44000 | Loss: 3.0716 | LR: 3.79e-04 | 1284.4ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14890 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14900/44000 | Loss: 3.0778 | LR: 3.79e-04 | 1282.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14900 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14910/44000 | Loss: 3.0422 | LR: 3.79e-04 | 1283.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14910 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14920/44000 | Loss: 3.0824 | LR: 3.79e-04 | 1282.2ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14920 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14930/44000 | Loss: 3.0637 | LR: 3.79e-04 | 1282.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14930 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14940/44000 | Loss: 3.0736 | LR: 3.79e-04 | 1283.2ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14940 that is less than the current step 14941. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14950/44000 | Loss: 3.0438 | LR: 3.78e-04 | 1283.9ms/step | 15.6 samples/s +Step 14960/44000 | Loss: 3.0051 | LR: 3.78e-04 | 1282.8ms/step | 15.6 samples/s +Step 14970/44000 | Loss: 3.0565 | LR: 3.78e-04 | 1282.5ms/step | 15.6 samples/s +Step 14980/44000 | Loss: 3.0415 | LR: 3.78e-04 | 1284.9ms/step | 15.6 samples/s +Step 14990/44000 | Loss: 3.0745 | LR: 3.78e-04 | 1285.6ms/step | 15.6 samples/s +Step 15000/44000 | Loss: 3.1051 | LR: 3.78e-04 | 1287.8ms/step | 15.5 samples/s +Validation at step 15000: Loss = 3.0799, Perplexity = 21.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-15000 +Step 15010/44000 | Loss: 3.0805 | LR: 3.77e-04 | 3750.3ms/step | 5.3 samples/s +Step 15020/44000 | Loss: 3.0504 | LR: 3.77e-04 | 1281.9ms/step | 15.6 samples/s +Step 15030/44000 | Loss: 3.0684 | LR: 3.77e-04 | 1284.2ms/step | 15.6 samples/s +Step 15040/44000 | Loss: 3.0779 | LR: 3.77e-04 | 1284.6ms/step | 15.6 samples/s +Step 15050/44000 | Loss: 3.0543 | LR: 3.77e-04 | 1281.5ms/step | 15.6 samples/s +Step 15060/44000 | Loss: 3.0881 | LR: 3.77e-04 | 1283.0ms/step | 15.6 samples/s +Step 15070/44000 | Loss: 3.0511 | LR: 3.77e-04 | 1285.0ms/step | 15.6 samples/s +Step 15080/44000 | Loss: 3.0911 | LR: 3.76e-04 | 1284.2ms/step | 15.6 samples/s +Step 15090/44000 | Loss: 3.0651 | LR: 3.76e-04 | 1285.8ms/step | 15.6 samples/s +Step 15100/44000 | Loss: 3.0406 | LR: 3.76e-04 | 1283.2ms/step | 15.6 samples/s +Step 15110/44000 | Loss: 3.0831 | LR: 3.76e-04 | 1284.4ms/step | 15.6 samples/s +Step 15120/44000 | Loss: 3.0865 | LR: 3.76e-04 | 1283.3ms/step | 15.6 samples/s +Step 15130/44000 | Loss: 3.0586 | LR: 3.76e-04 | 1284.0ms/step | 15.6 samples/s +Step 15140/44000 | Loss: 3.0707 | LR: 3.75e-04 | 1286.2ms/step | 15.6 samples/s +Step 15150/44000 | Loss: 3.0675 | LR: 3.75e-04 | 1285.8ms/step | 15.6 samples/s +Step 15160/44000 | Loss: 3.0896 | LR: 3.75e-04 | 1285.4ms/step | 15.6 samples/s +Step 15170/44000 | Loss: 3.0618 | LR: 3.75e-04 | 1285.3ms/step | 15.6 samples/s +Step 15180/44000 | Loss: 3.0953 | LR: 3.75e-04 | 1284.4ms/step | 15.6 samples/s +Step 15190/44000 | Loss: 3.0590 | LR: 3.75e-04 | 1285.2ms/step | 15.6 samples/s +Step 15200/44000 | Loss: 3.0947 | LR: 3.74e-04 | 1287.0ms/step | 15.5 samples/s +Step 15210/44000 | Loss: 3.0605 | LR: 3.74e-04 | 1287.8ms/step | 15.5 samples/s +Step 15220/44000 | Loss: 3.0517 | LR: 3.74e-04 | 1286.4ms/step | 15.5 samples/s +Step 15230/44000 | Loss: 3.0373 | LR: 3.74e-04 | 1286.6ms/step | 15.5 samples/s +Step 15240/44000 | Loss: 3.0177 | LR: 3.74e-04 | 1286.4ms/step | 15.5 samples/s +Step 15250/44000 | Loss: 3.0745 | LR: 3.74e-04 | 1281.8ms/step | 15.6 samples/s +Validation at step 15250: Loss = 3.0767, Perplexity = 21.69 +Step 15260/44000 | Loss: 3.0492 | LR: 3.74e-04 | 3454.9ms/step | 5.8 samples/s +Step 15270/44000 | Loss: 3.0461 | LR: 3.73e-04 | 1290.3ms/step | 15.5 samples/s +Step 15280/44000 | Loss: 3.0500 | LR: 3.73e-04 | 1293.7ms/step | 15.5 samples/s +Step 15290/44000 | Loss: 3.0593 | LR: 3.73e-04 | 1295.1ms/step | 15.4 samples/s +Step 15300/44000 | Loss: 3.0645 | LR: 3.73e-04 | 1297.7ms/step | 15.4 samples/s +Step 15310/44000 | Loss: 3.0761 | LR: 3.73e-04 | 1299.2ms/step | 15.4 samples/s +Step 15320/44000 | Loss: 3.0667 | LR: 3.73e-04 | 1299.5ms/step | 15.4 samples/s +Step 15330/44000 | Loss: 3.0508 | LR: 3.72e-04 | 1291.9ms/step | 15.5 samples/s +Step 15340/44000 | Loss: 3.0557 | LR: 3.72e-04 | 1297.5ms/step | 15.4 samples/s +Step 15350/44000 | Loss: 3.0473 | LR: 3.72e-04 | 1290.4ms/step | 15.5 samples/s +Step 15360/44000 | Loss: 3.0524 | LR: 3.72e-04 | 1284.6ms/step | 15.6 samples/s +Step 15370/44000 | Loss: 3.0554 | LR: 3.72e-04 | 1284.0ms/step | 15.6 samples/s +Step 15380/44000 | Loss: 3.0587 | LR: 3.72e-04 | 1282.4ms/step | 15.6 samples/s +Step 15390/44000 | Loss: 3.0274 | LR: 3.71e-04 | 1283.6ms/step | 15.6 samples/s +Step 15400/44000 | Loss: 3.0366 | LR: 3.71e-04 | 1285.8ms/step | 15.6 samples/s +Step 15410/44000 | Loss: 3.0318 | LR: 3.71e-04 | 1284.2ms/step | 15.6 samples/s +Step 15420/44000 | Loss: 3.0596 | LR: 3.71e-04 | 1280.0ms/step | 15.6 samples/s +Step 15430/44000 | Loss: 3.0463 | LR: 3.71e-04 | 1280.8ms/step | 15.6 samples/s +Step 15440/44000 | Loss: 3.0848 | LR: 3.71e-04 | 1286.6ms/step | 15.5 samples/s +Step 15450/44000 | Loss: 3.0631 | LR: 3.71e-04 | 1284.8ms/step | 15.6 samples/s +Step 15460/44000 | Loss: 3.0606 | LR: 3.70e-04 | 1285.2ms/step | 15.6 samples/s +Step 15470/44000 | Loss: 3.0559 | LR: 3.70e-04 | 1284.1ms/step | 15.6 samples/s +Step 15480/44000 | Loss: 3.0538 | LR: 3.70e-04 | 1285.0ms/step | 15.6 samples/s +Step 15490/44000 | Loss: 3.0473 | LR: 3.70e-04 | 1286.4ms/step | 15.5 samples/s +Step 15500/44000 | Loss: 3.0887 | LR: 3.70e-04 | 1285.5ms/step | 15.6 samples/s +Validation at step 15500: Loss = 3.0731, Perplexity = 21.61 +Step 15510/44000 | Loss: 3.0568 | LR: 3.70e-04 | 3434.6ms/step | 5.8 samples/s +Step 15520/44000 | Loss: 3.0800 | LR: 3.69e-04 | 1284.3ms/step | 15.6 samples/s +Step 15530/44000 | Loss: 3.0442 | LR: 3.69e-04 | 1286.1ms/step | 15.6 samples/s +Step 15540/44000 | Loss: 3.0307 | LR: 3.69e-04 | 1280.8ms/step | 15.6 samples/s +Step 15550/44000 | Loss: 3.0350 | LR: 3.69e-04 | 1279.9ms/step | 15.6 samples/s +Step 15560/44000 | Loss: 3.0492 | LR: 3.69e-04 | 1283.2ms/step | 15.6 samples/s +Step 15570/44000 | Loss: 3.0587 | LR: 3.69e-04 | 1281.2ms/step | 15.6 samples/s +Step 15580/44000 | Loss: 3.0632 | LR: 3.68e-04 | 1283.2ms/step | 15.6 samples/s +Step 15590/44000 | Loss: 3.0720 | LR: 3.68e-04 | 1284.4ms/step | 15.6 samples/s +Step 15600/44000 | Loss: 3.0524 | LR: 3.68e-04 | 1285.7ms/step | 15.6 samples/s +Step 15610/44000 | Loss: 3.0666 | LR: 3.68e-04 | 1285.7ms/step | 15.6 samples/s +Step 15620/44000 | Loss: 3.0768 | LR: 3.68e-04 | 1284.7ms/step | 15.6 samples/s +Step 15630/44000 | Loss: 3.0382 | LR: 3.68e-04 | 1285.7ms/step | 15.6 samples/s +Step 15640/44000 | Loss: 3.0588 | LR: 3.67e-04 | 1285.4ms/step | 15.6 samples/s +Step 15650/44000 | Loss: 3.0592 | LR: 3.67e-04 | 1282.4ms/step | 15.6 samples/s +Step 15660/44000 | Loss: 3.0322 | LR: 3.67e-04 | 1283.5ms/step | 15.6 samples/s +Step 15670/44000 | Loss: 3.0450 | LR: 3.67e-04 | 1282.6ms/step | 15.6 samples/s +Step 15680/44000 | Loss: 3.1043 | LR: 3.67e-04 | 1284.2ms/step | 15.6 samples/s +Step 15690/44000 | Loss: 3.0765 | LR: 3.67e-04 | 1284.5ms/step | 15.6 samples/s +Step 15700/44000 | Loss: 3.0900 | LR: 3.67e-04 | 1285.5ms/step | 15.6 samples/s +Step 15710/44000 | Loss: 3.0993 | LR: 3.66e-04 | 1289.4ms/step | 15.5 samples/s +Step 15720/44000 | Loss: 3.0405 | LR: 3.66e-04 | 1288.0ms/step | 15.5 samples/s +Step 15730/44000 | Loss: 3.0514 | LR: 3.66e-04 | 1287.4ms/step | 15.5 samples/s +Step 15740/44000 | Loss: 3.0452 | LR: 3.66e-04 | 1288.2ms/step | 15.5 samples/s +Step 15750/44000 | Loss: 3.0419 | LR: 3.66e-04 | 1291.1ms/step | 15.5 samples/s +Validation at step 15750: Loss = 3.0697, Perplexity = 21.54 +Step 15760/44000 | Loss: 3.0442 | LR: 3.66e-04 | 3491.0ms/step | 5.7 samples/s +Step 15770/44000 | Loss: 3.0359 | LR: 3.65e-04 | 1288.5ms/step | 15.5 samples/s +Step 15780/44000 | Loss: 3.0506 | LR: 3.65e-04 | 1291.5ms/step | 15.5 samples/s +Step 15790/44000 | Loss: 3.0573 | LR: 3.65e-04 | 1292.0ms/step | 15.5 samples/s +Step 15800/44000 | Loss: 3.0506 | LR: 3.65e-04 | 1284.6ms/step | 15.6 samples/s +Step 15810/44000 | Loss: 3.0354 | LR: 3.65e-04 | 1285.0ms/step | 15.6 samples/s +Step 15820/44000 | Loss: 3.0391 | LR: 3.65e-04 | 1285.5ms/step | 15.6 samples/s +Step 15830/44000 | Loss: 3.0641 | LR: 3.64e-04 | 1285.7ms/step | 15.6 samples/s +Step 15840/44000 | Loss: 3.0520 | LR: 3.64e-04 | 1285.3ms/step | 15.6 samples/s +Step 15850/44000 | Loss: 3.0375 | LR: 3.64e-04 | 1286.6ms/step | 15.5 samples/s +Step 15860/44000 | Loss: 3.0424 | LR: 3.64e-04 | 1285.9ms/step | 15.6 samples/s +Step 15870/44000 | Loss: 3.0527 | LR: 3.64e-04 | 1283.9ms/step | 15.6 samples/s +Step 15880/44000 | Loss: 3.0579 | LR: 3.64e-04 | 1282.5ms/step | 15.6 samples/s +Step 15890/44000 | Loss: 3.0248 | LR: 3.63e-04 | 1285.4ms/step | 15.6 samples/s +Step 15900/44000 | Loss: 3.0678 | LR: 3.63e-04 | 1286.0ms/step | 15.6 samples/s +Step 15910/44000 | Loss: 3.0639 | LR: 3.63e-04 | 1291.0ms/step | 15.5 samples/s +Step 15920/44000 | Loss: 3.0618 | LR: 3.63e-04 | 1287.5ms/step | 15.5 samples/s +Step 15930/44000 | Loss: 3.0182 | LR: 3.63e-04 | 1289.8ms/step | 15.5 samples/s +Step 15940/44000 | Loss: 3.0410 | LR: 3.63e-04 | 1286.2ms/step | 15.6 samples/s +Step 15950/44000 | Loss: 3.0232 | LR: 3.62e-04 | 1283.9ms/step | 15.6 samples/s +Step 15960/44000 | Loss: 3.0314 | LR: 3.62e-04 | 1284.6ms/step | 15.6 samples/s +Step 15970/44000 | Loss: 3.0437 | LR: 3.62e-04 | 1291.4ms/step | 15.5 samples/s +Step 15980/44000 | Loss: 3.0532 | LR: 3.62e-04 | 1291.3ms/step | 15.5 samples/s +Step 15990/44000 | Loss: 3.0582 | LR: 3.62e-04 | 1286.0ms/step | 15.6 samples/s +Step 16000/44000 | Loss: 3.0620 | LR: 3.62e-04 | 1290.6ms/step | 15.5 samples/s +Validation at step 16000: Loss = 3.0662, Perplexity = 21.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-16000 +Step 16010/44000 | Loss: 3.0124 | LR: 3.62e-04 | 3811.5ms/step | 5.2 samples/s +Step 16020/44000 | Loss: 3.0321 | LR: 3.61e-04 | 1285.4ms/step | 15.6 samples/s +Step 16030/44000 | Loss: 3.0500 | LR: 3.61e-04 | 1282.4ms/step | 15.6 samples/s +Step 16040/44000 | Loss: 3.0356 | LR: 3.61e-04 | 1283.0ms/step | 15.6 samples/s +Step 16050/44000 | Loss: 3.0704 | LR: 3.61e-04 | 1287.3ms/step | 15.5 samples/s +Step 16060/44000 | Loss: 3.0395 | LR: 3.61e-04 | 1283.1ms/step | 15.6 samples/s +Step 16070/44000 | Loss: 3.0620 | LR: 3.61e-04 | 1284.2ms/step | 15.6 samples/s +Step 16080/44000 | Loss: 3.0507 | LR: 3.60e-04 | 1285.2ms/step | 15.6 samples/s +Step 16090/44000 | Loss: 3.0167 | LR: 3.60e-04 | 1287.6ms/step | 15.5 samples/s +Step 16100/44000 | Loss: 3.0382 | LR: 3.60e-04 | 1284.6ms/step | 15.6 samples/s +Step 16110/44000 | Loss: 3.0543 | LR: 3.60e-04 | 1284.4ms/step | 15.6 samples/s +Step 16120/44000 | Loss: 3.0534 | LR: 3.60e-04 | 1285.0ms/step | 15.6 samples/s +Step 16130/44000 | Loss: 3.0371 | LR: 3.60e-04 | 1284.6ms/step | 15.6 samples/s +Step 16140/44000 | Loss: 3.0457 | LR: 3.59e-04 | 1284.9ms/step | 15.6 samples/s +Step 16150/44000 | Loss: 3.0243 | LR: 3.59e-04 | 1286.6ms/step | 15.5 samples/s +Step 16160/44000 | Loss: 3.0408 | LR: 3.59e-04 | 1287.3ms/step | 15.5 samples/s +Step 16170/44000 | Loss: 3.0647 | LR: 3.59e-04 | 1284.5ms/step | 15.6 samples/s +Step 16180/44000 | Loss: 3.0686 | LR: 3.59e-04 | 1285.9ms/step | 15.6 samples/s +Step 16190/44000 | Loss: 3.0600 | LR: 3.59e-04 | 1286.4ms/step | 15.5 samples/s +Step 16200/44000 | Loss: 3.0303 | LR: 3.58e-04 | 1283.3ms/step | 15.6 samples/s +Step 16210/44000 | Loss: 3.0297 | LR: 3.58e-04 | 1282.6ms/step | 15.6 samples/s +Step 16220/44000 | Loss: 3.0760 | LR: 3.58e-04 | 1282.7ms/step | 15.6 samples/s +Step 16230/44000 | Loss: 3.0131 | LR: 3.58e-04 | 1282.6ms/step | 15.6 samples/s +Step 16240/44000 | Loss: 3.0276 | LR: 3.58e-04 | 1281.0ms/step | 15.6 samples/s +Step 16250/44000 | Loss: 3.0706 | LR: 3.58e-04 | 1285.1ms/step | 15.6 samples/s +Validation at step 16250: Loss = 3.0627, Perplexity = 21.38 +Step 16260/44000 | Loss: 3.0303 | LR: 3.57e-04 | 3427.9ms/step | 5.8 samples/s +Step 16270/44000 | Loss: 3.0872 | LR: 3.57e-04 | 1280.7ms/step | 15.6 samples/s +Step 16280/44000 | Loss: 3.0468 | LR: 3.57e-04 | 1281.6ms/step | 15.6 samples/s +Step 16290/44000 | Loss: 3.0264 | LR: 3.57e-04 | 1281.3ms/step | 15.6 samples/s +Step 16300/44000 | Loss: 3.0739 | LR: 3.57e-04 | 1282.8ms/step | 15.6 samples/s +Step 16310/44000 | Loss: 3.0590 | LR: 3.57e-04 | 1280.5ms/step | 15.6 samples/s +Step 16320/44000 | Loss: 3.0155 | LR: 3.56e-04 | 1283.5ms/step | 15.6 samples/s +Step 16330/44000 | Loss: 3.0324 | LR: 3.56e-04 | 1284.2ms/step | 15.6 samples/s +Step 16340/44000 | Loss: 3.0574 | LR: 3.56e-04 | 1284.1ms/step | 15.6 samples/s +Step 16350/44000 | Loss: 3.0559 | LR: 3.56e-04 | 1281.6ms/step | 15.6 samples/s +Step 16360/44000 | Loss: 3.0621 | LR: 3.56e-04 | 1284.1ms/step | 15.6 samples/s +Step 16370/44000 | Loss: 3.0332 | LR: 3.56e-04 | 1285.0ms/step | 15.6 samples/s +Step 16380/44000 | Loss: 3.0467 | LR: 3.55e-04 | 1282.3ms/step | 15.6 samples/s +Step 16390/44000 | Loss: 3.0818 | LR: 3.55e-04 | 1284.6ms/step | 15.6 samples/s +Step 16400/44000 | Loss: 3.0288 | LR: 3.55e-04 | 1287.1ms/step | 15.5 samples/s +Step 16410/44000 | Loss: 3.0402 | LR: 3.55e-04 | 1288.1ms/step | 15.5 samples/s +Step 16420/44000 | Loss: 3.0361 | LR: 3.55e-04 | 1288.6ms/step | 15.5 samples/s +Step 16430/44000 | Loss: 3.0558 | LR: 3.55e-04 | 1285.3ms/step | 15.6 samples/s +Step 16440/44000 | Loss: 3.0420 | LR: 3.54e-04 | 1285.2ms/step | 15.6 samples/s +Step 16450/44000 | Loss: 3.0457 | LR: 3.54e-04 | 1285.6ms/step | 15.6 samples/s +Step 16460/44000 | Loss: 3.0574 | LR: 3.54e-04 | 1284.4ms/step | 15.6 samples/s +Step 16470/44000 | Loss: 3.0222 | LR: 3.54e-04 | 1284.7ms/step | 15.6 samples/s +Step 16480/44000 | Loss: 3.0274 | LR: 3.54e-04 | 1280.8ms/step | 15.6 samples/s +Step 16490/44000 | Loss: 3.0245 | LR: 3.54e-04 | 1280.1ms/step | 15.6 samples/s +Step 16500/44000 | Loss: 3.0369 | LR: 3.53e-04 | 1281.3ms/step | 15.6 samples/s +Validation at step 16500: Loss = 3.0596, Perplexity = 21.32 +Step 16510/44000 | Loss: 3.0587 | LR: 3.53e-04 | 3432.8ms/step | 5.8 samples/s +Step 16520/44000 | Loss: 3.0352 | LR: 3.53e-04 | 1279.4ms/step | 15.6 samples/s +Step 16530/44000 | Loss: 3.0097 | LR: 3.53e-04 | 1280.6ms/step | 15.6 samples/s +Step 16540/44000 | Loss: 3.0463 | LR: 3.53e-04 | 1279.6ms/step | 15.6 samples/s +Step 16550/44000 | Loss: 3.0009 | LR: 3.53e-04 | 1278.8ms/step | 15.6 samples/s +Step 16560/44000 | Loss: 3.0270 | LR: 3.52e-04 | 1282.4ms/step | 15.6 samples/s +Step 16570/44000 | Loss: 3.0602 | LR: 3.52e-04 | 1282.4ms/step | 15.6 samples/s +Step 16580/44000 | Loss: 3.0265 | LR: 3.52e-04 | 1280.2ms/step | 15.6 samples/s +Step 16590/44000 | Loss: 3.0388 | LR: 3.52e-04 | 1280.8ms/step | 15.6 samples/s +Step 16600/44000 | Loss: 3.0228 | LR: 3.52e-04 | 1282.7ms/step | 15.6 samples/s +Step 16610/44000 | Loss: 3.0425 | LR: 3.52e-04 | 1289.1ms/step | 15.5 samples/s +Step 16620/44000 | Loss: 3.0454 | LR: 3.51e-04 | 1288.8ms/step | 15.5 samples/s +Step 16630/44000 | Loss: 3.0226 | LR: 3.51e-04 | 1289.8ms/step | 15.5 samples/s +Step 16640/44000 | Loss: 3.0564 | LR: 3.51e-04 | 1289.1ms/step | 15.5 samples/s +Step 16650/44000 | Loss: 3.0230 | LR: 3.51e-04 | 1286.5ms/step | 15.5 samples/s +Step 16660/44000 | Loss: 3.0227 | LR: 3.51e-04 | 1287.3ms/step | 15.5 samples/s +Step 16670/44000 | Loss: 3.0491 | LR: 3.51e-04 | 1289.2ms/step | 15.5 samples/s +Step 16680/44000 | Loss: 3.0036 | LR: 3.50e-04 | 1292.7ms/step | 15.5 samples/s +Step 16690/44000 | Loss: 3.0298 | LR: 3.50e-04 | 1288.2ms/step | 15.5 samples/s +Step 16700/44000 | Loss: 3.0641 | LR: 3.50e-04 | 1296.0ms/step | 15.4 samples/s +Step 16710/44000 | Loss: 3.0045 | LR: 3.50e-04 | 1298.9ms/step | 15.4 samples/s +Step 16720/44000 | Loss: 3.0490 | LR: 3.50e-04 | 1289.6ms/step | 15.5 samples/s +Step 16730/44000 | Loss: 3.0162 | LR: 3.50e-04 | 1285.8ms/step | 15.6 samples/s +Step 16740/44000 | Loss: 3.0196 | LR: 3.49e-04 | 1285.9ms/step | 15.6 samples/s +Step 16750/44000 | Loss: 3.0150 | LR: 3.49e-04 | 1286.0ms/step | 15.6 samples/s +Validation at step 16750: Loss = 3.0567, Perplexity = 21.26 +Step 16760/44000 | Loss: 3.0594 | LR: 3.49e-04 | 3430.8ms/step | 5.8 samples/s +Step 16770/44000 | Loss: 3.0625 | LR: 3.49e-04 | 1283.5ms/step | 15.6 samples/s +Step 16780/44000 | Loss: 3.0520 | LR: 3.49e-04 | 1285.7ms/step | 15.6 samples/s +Step 16790/44000 | Loss: 3.0692 | LR: 3.49e-04 | 1281.8ms/step | 15.6 samples/s +Step 16800/44000 | Loss: 3.0445 | LR: 3.48e-04 | 1279.6ms/step | 15.6 samples/s +Step 16810/44000 | Loss: 3.0208 | LR: 3.48e-04 | 1281.4ms/step | 15.6 samples/s +Step 16820/44000 | Loss: 3.0675 | LR: 3.48e-04 | 1282.6ms/step | 15.6 samples/s +Step 16830/44000 | Loss: 3.0628 | LR: 3.48e-04 | 1280.6ms/step | 15.6 samples/s +Step 16840/44000 | Loss: 3.0339 | LR: 3.48e-04 | 1279.5ms/step | 15.6 samples/s +Step 16850/44000 | Loss: 3.0215 | LR: 3.48e-04 | 1280.5ms/step | 15.6 samples/s +Step 16860/44000 | Loss: 3.0616 | LR: 3.47e-04 | 1282.1ms/step | 15.6 samples/s +Step 16870/44000 | Loss: 3.0347 | LR: 3.47e-04 | 1286.1ms/step | 15.6 samples/s +Step 16880/44000 | Loss: 3.0572 | LR: 3.47e-04 | 1285.8ms/step | 15.6 samples/s +Step 16890/44000 | Loss: 3.0239 | LR: 3.47e-04 | 1287.2ms/step | 15.5 samples/s +Step 16900/44000 | Loss: 3.0497 | LR: 3.47e-04 | 1286.9ms/step | 15.5 samples/s +Step 16910/44000 | Loss: 3.0365 | LR: 3.47e-04 | 1286.1ms/step | 15.6 samples/s +Step 16920/44000 | Loss: 3.0499 | LR: 3.46e-04 | 1286.6ms/step | 15.5 samples/s +Step 16930/44000 | Loss: 3.0707 | LR: 3.46e-04 | 1285.7ms/step | 15.6 samples/s +Step 16940/44000 | Loss: 3.0315 | LR: 3.46e-04 | 1285.4ms/step | 15.6 samples/s +Step 16950/44000 | Loss: 3.0215 | LR: 3.46e-04 | 1287.0ms/step | 15.5 samples/s +Step 16960/44000 | Loss: 3.0342 | LR: 3.46e-04 | 1288.0ms/step | 15.5 samples/s +Step 16970/44000 | Loss: 3.0612 | LR: 3.46e-04 | 1281.4ms/step | 15.6 samples/s +Step 16980/44000 | Loss: 3.0686 | LR: 3.45e-04 | 1281.2ms/step | 15.6 samples/s +Step 16990/44000 | Loss: 3.0262 | LR: 3.45e-04 | 1282.0ms/step | 15.6 samples/s +Step 17000/44000 | Loss: 3.0383 | LR: 3.45e-04 | 1279.9ms/step | 15.6 samples/s +Validation at step 17000: Loss = 3.0539, Perplexity = 21.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-17000 +Step 17010/44000 | Loss: 3.0536 | LR: 3.45e-04 | 3734.7ms/step | 5.4 samples/s +Step 17020/44000 | Loss: 3.0330 | LR: 3.45e-04 | 1285.7ms/step | 15.6 samples/s +Step 17030/44000 | Loss: 3.0396 | LR: 3.45e-04 | 1286.2ms/step | 15.5 samples/s +Step 17040/44000 | Loss: 3.0738 | LR: 3.44e-04 | 1282.2ms/step | 15.6 samples/s +Step 17050/44000 | Loss: 3.0665 | LR: 3.44e-04 | 1289.2ms/step | 15.5 samples/s +Step 17060/44000 | Loss: 3.0193 | LR: 3.44e-04 | 1286.8ms/step | 15.5 samples/s +Step 17070/44000 | Loss: 3.0040 | LR: 3.44e-04 | 1289.2ms/step | 15.5 samples/s +Step 17080/44000 | Loss: 3.0496 | LR: 3.44e-04 | 1296.7ms/step | 15.4 samples/s +Step 17090/44000 | Loss: 3.0335 | LR: 3.44e-04 | 1298.6ms/step | 15.4 samples/s +Step 17100/44000 | Loss: 3.0394 | LR: 3.43e-04 | 1290.4ms/step | 15.5 samples/s +Step 17110/44000 | Loss: 3.0318 | LR: 3.43e-04 | 1285.8ms/step | 15.6 samples/s +Step 17120/44000 | Loss: 3.0372 | LR: 3.43e-04 | 1282.5ms/step | 15.6 samples/s +Step 17130/44000 | Loss: 3.0251 | LR: 3.43e-04 | 1279.6ms/step | 15.6 samples/s +Step 17140/44000 | Loss: 3.0532 | LR: 3.43e-04 | 1282.8ms/step | 15.6 samples/s +Step 17150/44000 | Loss: 3.0553 | LR: 3.43e-04 | 1289.4ms/step | 15.5 samples/s +Step 17160/44000 | Loss: 3.0192 | LR: 3.42e-04 | 1283.2ms/step | 15.6 samples/s +Step 17170/44000 | Loss: 3.0321 | LR: 3.42e-04 | 1282.7ms/step | 15.6 samples/s +Step 17180/44000 | Loss: 3.0137 | LR: 3.42e-04 | 1279.7ms/step | 15.6 samples/s +Step 17190/44000 | Loss: 3.0567 | LR: 3.42e-04 | 1278.9ms/step | 15.6 samples/s +Step 17200/44000 | Loss: 3.0300 | LR: 3.42e-04 | 1279.4ms/step | 15.6 samples/s +Step 17210/44000 | Loss: 3.0496 | LR: 3.42e-04 | 1284.6ms/step | 15.6 samples/s +Step 17220/44000 | Loss: 3.0390 | LR: 3.41e-04 | 1284.2ms/step | 15.6 samples/s +Step 17230/44000 | Loss: 3.0322 | LR: 3.41e-04 | 1283.2ms/step | 15.6 samples/s +Step 17240/44000 | Loss: 3.0331 | LR: 3.41e-04 | 1283.2ms/step | 15.6 samples/s +Step 17250/44000 | Loss: 3.0446 | LR: 3.41e-04 | 1284.7ms/step | 15.6 samples/s +Validation at step 17250: Loss = 3.0509, Perplexity = 21.13 +Step 17260/44000 | Loss: 3.0369 | LR: 3.41e-04 | 3431.6ms/step | 5.8 samples/s +Step 17270/44000 | Loss: 3.0482 | LR: 3.41e-04 | 1284.3ms/step | 15.6 samples/s +Step 17280/44000 | Loss: 3.0362 | LR: 3.40e-04 | 1282.6ms/step | 15.6 samples/s +Step 17290/44000 | Loss: 3.0347 | LR: 3.40e-04 | 1285.9ms/step | 15.6 samples/s +Step 17300/44000 | Loss: 3.0215 | LR: 3.40e-04 | 1284.1ms/step | 15.6 samples/s +Step 17310/44000 | Loss: 3.0288 | LR: 3.40e-04 | 1284.1ms/step | 15.6 samples/s +Step 17320/44000 | Loss: 3.0343 | LR: 3.40e-04 | 1283.3ms/step | 15.6 samples/s +Step 17330/44000 | Loss: 3.0383 | LR: 3.40e-04 | 1283.9ms/step | 15.6 samples/s +Step 17340/44000 | Loss: 3.0362 | LR: 3.39e-04 | 1285.5ms/step | 15.6 samples/s +Step 17350/44000 | Loss: 3.0342 | LR: 3.39e-04 | 1284.2ms/step | 15.6 samples/s +Step 17360/44000 | Loss: 3.0300 | LR: 3.39e-04 | 1284.6ms/step | 15.6 samples/s +Step 17370/44000 | Loss: 3.0059 | LR: 3.39e-04 | 1285.3ms/step | 15.6 samples/s +Step 17380/44000 | Loss: 3.0545 | LR: 3.39e-04 | 1283.4ms/step | 15.6 samples/s +Step 17390/44000 | Loss: 3.0280 | LR: 3.39e-04 | 1280.9ms/step | 15.6 samples/s +Step 17400/44000 | Loss: 3.0575 | LR: 3.38e-04 | 1279.7ms/step | 15.6 samples/s +Step 17410/44000 | Loss: 3.0365 | LR: 3.38e-04 | 1278.7ms/step | 15.6 samples/s +Step 17420/44000 | Loss: 3.0114 | LR: 3.38e-04 | 1282.5ms/step | 15.6 samples/s +Step 17430/44000 | Loss: 3.0396 | LR: 3.38e-04 | 1281.9ms/step | 15.6 samples/s +Step 17440/44000 | Loss: 3.0339 | LR: 3.38e-04 | 1279.7ms/step | 15.6 samples/s +Step 17450/44000 | Loss: 3.0083 | LR: 3.38e-04 | 1283.7ms/step | 15.6 samples/s +Step 17460/44000 | Loss: 3.0264 | LR: 3.37e-04 | 1284.7ms/step | 15.6 samples/s +Step 17470/44000 | Loss: 3.0279 | LR: 3.37e-04 | 1283.8ms/step | 15.6 samples/s +Step 17480/44000 | Loss: 3.0517 | LR: 3.37e-04 | 1284.2ms/step | 15.6 samples/s +Step 17490/44000 | Loss: 3.0063 | LR: 3.37e-04 | 1285.8ms/step | 15.6 samples/s +Step 17500/44000 | Loss: 3.0574 | LR: 3.37e-04 | 1285.7ms/step | 15.6 samples/s +Validation at step 17500: Loss = 3.0470, Perplexity = 21.05 +Step 17510/44000 | Loss: 3.0325 | LR: 3.37e-04 | 3425.8ms/step | 5.8 samples/s +Step 17520/44000 | Loss: 2.9941 | LR: 3.36e-04 | 1283.4ms/step | 15.6 samples/s +Step 17530/44000 | Loss: 3.0313 | LR: 3.36e-04 | 1285.1ms/step | 15.6 samples/s +Step 17540/44000 | Loss: 3.0572 | LR: 3.36e-04 | 1281.7ms/step | 15.6 samples/s +Step 17550/44000 | Loss: 3.0196 | LR: 3.36e-04 | 1284.1ms/step | 15.6 samples/s +Step 17560/44000 | Loss: 3.0186 | LR: 3.36e-04 | 1283.9ms/step | 15.6 samples/s +Step 17570/44000 | Loss: 3.0554 | LR: 3.35e-04 | 1282.8ms/step | 15.6 samples/s +Step 17580/44000 | Loss: 3.0308 | LR: 3.35e-04 | 1281.9ms/step | 15.6 samples/s +Step 17590/44000 | Loss: 3.0340 | LR: 3.35e-04 | 1283.3ms/step | 15.6 samples/s +Step 17600/44000 | Loss: 3.0430 | LR: 3.35e-04 | 1282.8ms/step | 15.6 samples/s +Step 17610/44000 | Loss: 3.0396 | LR: 3.35e-04 | 1283.3ms/step | 15.6 samples/s +Step 17620/44000 | Loss: 3.0906 | LR: 3.35e-04 | 1282.4ms/step | 15.6 samples/s +Step 17630/44000 | Loss: 3.0588 | LR: 3.34e-04 | 1283.7ms/step | 15.6 samples/s +Step 17640/44000 | Loss: 3.0448 | LR: 3.34e-04 | 1285.1ms/step | 15.6 samples/s +Step 17650/44000 | Loss: 3.0134 | LR: 3.34e-04 | 1284.3ms/step | 15.6 samples/s +Step 17660/44000 | Loss: 3.0437 | LR: 3.34e-04 | 1284.3ms/step | 15.6 samples/s +Step 17670/44000 | Loss: 3.0554 | LR: 3.34e-04 | 1284.5ms/step | 15.6 samples/s +Step 17680/44000 | Loss: 3.0043 | LR: 3.34e-04 | 1285.0ms/step | 15.6 samples/s +Step 17690/44000 | Loss: 3.0255 | LR: 3.33e-04 | 1284.0ms/step | 15.6 samples/s +Step 17700/44000 | Loss: 3.0500 | LR: 3.33e-04 | 1284.7ms/step | 15.6 samples/s +Step 17710/44000 | Loss: 3.0223 | LR: 3.33e-04 | 1279.7ms/step | 15.6 samples/s +Step 17720/44000 | Loss: 3.0154 | LR: 3.33e-04 | 1274.2ms/step | 15.7 samples/s +Step 17730/44000 | Loss: 2.9979 | LR: 3.33e-04 | 1279.4ms/step | 15.6 samples/s +Step 17740/44000 | Loss: 3.0116 | LR: 3.33e-04 | 1281.1ms/step | 15.6 samples/s +Step 17750/44000 | Loss: 3.0464 | LR: 3.32e-04 | 1282.8ms/step | 15.6 samples/s +Validation at step 17750: Loss = 3.0450, Perplexity = 21.01 +Step 17760/44000 | Loss: 3.0257 | LR: 3.32e-04 | 3425.9ms/step | 5.8 samples/s +Step 17770/44000 | Loss: 3.0403 | LR: 3.32e-04 | 1280.4ms/step | 15.6 samples/s +Step 17780/44000 | Loss: 3.0456 | LR: 3.32e-04 | 1281.6ms/step | 15.6 samples/s +Step 17790/44000 | Loss: 3.0250 | LR: 3.32e-04 | 1280.0ms/step | 15.6 samples/s +Step 17800/44000 | Loss: 3.0286 | LR: 3.32e-04 | 1283.6ms/step | 15.6 samples/s +Step 17810/44000 | Loss: 3.0285 | LR: 3.31e-04 | 1286.3ms/step | 15.5 samples/s +Step 17820/44000 | Loss: 3.0594 | LR: 3.31e-04 | 1285.5ms/step | 15.6 samples/s +Step 17830/44000 | Loss: 3.0208 | LR: 3.31e-04 | 1283.7ms/step | 15.6 samples/s +Step 17840/44000 | Loss: 3.0366 | LR: 3.31e-04 | 1284.8ms/step | 15.6 samples/s +Step 17850/44000 | Loss: 3.0246 | LR: 3.31e-04 | 1284.8ms/step | 15.6 samples/s +Step 17860/44000 | Loss: 3.0099 | LR: 3.31e-04 | 1286.0ms/step | 15.6 samples/s +Step 17870/44000 | Loss: 3.0264 | LR: 3.30e-04 | 1280.8ms/step | 15.6 samples/s +Step 17880/44000 | Loss: 3.0160 | LR: 3.30e-04 | 1279.3ms/step | 15.6 samples/s +Step 17890/44000 | Loss: 2.9986 | LR: 3.30e-04 | 1279.0ms/step | 15.6 samples/s +Step 17900/44000 | Loss: 3.0390 | LR: 3.30e-04 | 1279.4ms/step | 15.6 samples/s +Step 17910/44000 | Loss: 3.0471 | LR: 3.30e-04 | 1278.7ms/step | 15.6 samples/s +Step 17920/44000 | Loss: 3.0211 | LR: 3.29e-04 | 1280.2ms/step | 15.6 samples/s +Step 17930/44000 | Loss: 3.0241 | LR: 3.29e-04 | 1281.8ms/step | 15.6 samples/s +Step 17940/44000 | Loss: 3.0236 | LR: 3.29e-04 | 1285.6ms/step | 15.6 samples/s +Step 17950/44000 | Loss: 3.0216 | LR: 3.29e-04 | 1285.1ms/step | 15.6 samples/s +Step 17960/44000 | Loss: 3.0383 | LR: 3.29e-04 | 1284.2ms/step | 15.6 samples/s +Step 17970/44000 | Loss: 3.0423 | LR: 3.29e-04 | 1283.4ms/step | 15.6 samples/s +Step 17980/44000 | Loss: 3.0002 | LR: 3.28e-04 | 1288.0ms/step | 15.5 samples/s +Step 17990/44000 | Loss: 3.0341 | LR: 3.28e-04 | 1290.2ms/step | 15.5 samples/s +Step 18000/44000 | Loss: 3.0417 | LR: 3.28e-04 | 1289.0ms/step | 15.5 samples/s +Validation at step 18000: Loss = 3.0422, Perplexity = 20.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-18000 +Step 18010/44000 | Loss: 3.0201 | LR: 3.28e-04 | 3789.2ms/step | 5.3 samples/s +Step 18020/44000 | Loss: 3.0181 | LR: 3.28e-04 | 1284.1ms/step | 15.6 samples/s +Step 18030/44000 | Loss: 2.9943 | LR: 3.28e-04 | 1285.7ms/step | 15.6 samples/s +Step 18040/44000 | Loss: 3.0486 | LR: 3.27e-04 | 1284.7ms/step | 15.6 samples/s +Step 18050/44000 | Loss: 3.0135 | LR: 3.27e-04 | 1292.5ms/step | 15.5 samples/s +Step 18060/44000 | Loss: 3.0187 | LR: 3.27e-04 | 1294.2ms/step | 15.5 samples/s +Step 18070/44000 | Loss: 3.0018 | LR: 3.27e-04 | 1285.8ms/step | 15.6 samples/s +Step 18080/44000 | Loss: 3.0140 | LR: 3.27e-04 | 1282.8ms/step | 15.6 samples/s +Step 18090/44000 | Loss: 3.0258 | LR: 3.27e-04 | 1285.5ms/step | 15.6 samples/s +Step 18100/44000 | Loss: 3.0060 | LR: 3.26e-04 | 1282.3ms/step | 15.6 samples/s +Step 18110/44000 | Loss: 3.0436 | LR: 3.26e-04 | 1287.2ms/step | 15.5 samples/s +Step 18120/44000 | Loss: 3.0402 | LR: 3.26e-04 | 1286.1ms/step | 15.6 samples/s +Step 18130/44000 | Loss: 3.0282 | LR: 3.26e-04 | 1284.9ms/step | 15.6 samples/s +Step 18140/44000 | Loss: 3.0137 | LR: 3.26e-04 | 1285.4ms/step | 15.6 samples/s +Step 18150/44000 | Loss: 3.0026 | LR: 3.26e-04 | 1286.1ms/step | 15.6 samples/s +Step 18160/44000 | Loss: 3.0279 | LR: 3.25e-04 | 1290.7ms/step | 15.5 samples/s +Step 18170/44000 | Loss: 3.0295 | LR: 3.25e-04 | 1290.0ms/step | 15.5 samples/s +Step 18180/44000 | Loss: 3.0169 | LR: 3.25e-04 | 1287.0ms/step | 15.5 samples/s +Step 18190/44000 | Loss: 3.0428 | LR: 3.25e-04 | 1282.7ms/step | 15.6 samples/s +Step 18200/44000 | Loss: 3.0308 | LR: 3.25e-04 | 1286.1ms/step | 15.6 samples/s +Step 18210/44000 | Loss: 3.0253 | LR: 3.24e-04 | 1289.7ms/step | 15.5 samples/s +Step 18220/44000 | Loss: 3.0148 | LR: 3.24e-04 | 1287.3ms/step | 15.5 samples/s +Step 18230/44000 | Loss: 3.0173 | LR: 3.24e-04 | 1289.4ms/step | 15.5 samples/s +Step 18240/44000 | Loss: 3.0172 | LR: 3.24e-04 | 1288.8ms/step | 15.5 samples/s +Step 18250/44000 | Loss: 3.0118 | LR: 3.24e-04 | 1280.1ms/step | 15.6 samples/s +Validation at step 18250: Loss = 3.0386, Perplexity = 20.88 +Step 18260/44000 | Loss: 3.0257 | LR: 3.24e-04 | 3428.8ms/step | 5.8 samples/s +Step 18270/44000 | Loss: 3.0300 | LR: 3.23e-04 | 1284.0ms/step | 15.6 samples/s +Step 18280/44000 | Loss: 3.0386 | LR: 3.23e-04 | 1280.0ms/step | 15.6 samples/s +Step 18290/44000 | Loss: 3.0057 | LR: 3.23e-04 | 1279.3ms/step | 15.6 samples/s +Step 18300/44000 | Loss: 3.0167 | LR: 3.23e-04 | 1282.7ms/step | 15.6 samples/s +Step 18310/44000 | Loss: 3.0528 | LR: 3.23e-04 | 1284.6ms/step | 15.6 samples/s +Step 18320/44000 | Loss: 3.0043 | LR: 3.23e-04 | 1285.9ms/step | 15.6 samples/s +Step 18330/44000 | Loss: 3.0571 | LR: 3.22e-04 | 1275.7ms/step | 15.7 samples/s +Step 18340/44000 | Loss: 3.0351 | LR: 3.22e-04 | 1274.1ms/step | 15.7 samples/s +Step 18350/44000 | Loss: 3.0359 | LR: 3.22e-04 | 1277.2ms/step | 15.7 samples/s +Step 18360/44000 | Loss: 3.0542 | LR: 3.22e-04 | 1276.9ms/step | 15.7 samples/s +Step 18370/44000 | Loss: 3.0174 | LR: 3.22e-04 | 1278.5ms/step | 15.6 samples/s +Step 18380/44000 | Loss: 3.0129 | LR: 3.22e-04 | 1280.7ms/step | 15.6 samples/s +Step 18390/44000 | Loss: 3.0458 | LR: 3.21e-04 | 1283.1ms/step | 15.6 samples/s +Step 18400/44000 | Loss: 3.0398 | LR: 3.21e-04 | 1283.9ms/step | 15.6 samples/s +Step 18410/44000 | Loss: 3.0260 | LR: 3.21e-04 | 1285.6ms/step | 15.6 samples/s +Step 18420/44000 | Loss: 2.9988 | LR: 3.21e-04 | 1290.1ms/step | 15.5 samples/s +Step 18430/44000 | Loss: 3.0371 | LR: 3.21e-04 | 1288.2ms/step | 15.5 samples/s +Step 18440/44000 | Loss: 3.0149 | LR: 3.20e-04 | 1290.7ms/step | 15.5 samples/s +Step 18450/44000 | Loss: 3.0377 | LR: 3.20e-04 | 1289.4ms/step | 15.5 samples/s +Step 18460/44000 | Loss: 3.0518 | LR: 3.20e-04 | 1290.7ms/step | 15.5 samples/s +Step 18470/44000 | Loss: 2.9960 | LR: 3.20e-04 | 1292.3ms/step | 15.5 samples/s +Step 18480/44000 | Loss: 3.0310 | LR: 3.20e-04 | 1291.5ms/step | 15.5 samples/s +Step 18490/44000 | Loss: 3.0320 | LR: 3.20e-04 | 1288.5ms/step | 15.5 samples/s +Step 18500/44000 | Loss: 3.0265 | LR: 3.19e-04 | 1282.0ms/step | 15.6 samples/s +Validation at step 18500: Loss = 3.0370, Perplexity = 20.84 +Step 18510/44000 | Loss: 3.0014 | LR: 3.19e-04 | 3456.9ms/step | 5.8 samples/s +Step 18520/44000 | Loss: 3.0038 | LR: 3.19e-04 | 1289.6ms/step | 15.5 samples/s +Step 18530/44000 | Loss: 2.9976 | LR: 3.19e-04 | 1288.0ms/step | 15.5 samples/s +Step 18540/44000 | Loss: 3.0493 | LR: 3.19e-04 | 1287.1ms/step | 15.5 samples/s +Step 18550/44000 | Loss: 3.0205 | LR: 3.19e-04 | 1287.3ms/step | 15.5 samples/s +Step 18560/44000 | Loss: 2.9893 | LR: 3.18e-04 | 1285.0ms/step | 15.6 samples/s +Step 18570/44000 | Loss: 3.0251 | LR: 3.18e-04 | 1284.2ms/step | 15.6 samples/s +Step 18580/44000 | Loss: 2.9934 | LR: 3.18e-04 | 1285.0ms/step | 15.6 samples/s +Step 18590/44000 | Loss: 3.0118 | LR: 3.18e-04 | 1282.2ms/step | 15.6 samples/s +Step 18600/44000 | Loss: 3.0366 | LR: 3.18e-04 | 1279.6ms/step | 15.6 samples/s +Step 18610/44000 | Loss: 3.0458 | LR: 3.18e-04 | 1281.3ms/step | 15.6 samples/s +Step 18620/44000 | Loss: 3.0070 | LR: 3.17e-04 | 1283.5ms/step | 15.6 samples/s +Step 18630/44000 | Loss: 3.0097 | LR: 3.17e-04 | 1283.3ms/step | 15.6 samples/s +Step 18640/44000 | Loss: 3.0135 | LR: 3.17e-04 | 1279.3ms/step | 15.6 samples/s +Step 18650/44000 | Loss: 2.9997 | LR: 3.17e-04 | 1280.5ms/step | 15.6 samples/s +Step 18660/44000 | Loss: 2.9760 | LR: 3.17e-04 | 1279.3ms/step | 15.6 samples/s +Step 18670/44000 | Loss: 2.9950 | LR: 3.16e-04 | 1280.9ms/step | 15.6 samples/s +Step 18680/44000 | Loss: 3.0115 | LR: 3.16e-04 | 1282.7ms/step | 15.6 samples/s +Step 18690/44000 | Loss: 3.0160 | LR: 3.16e-04 | 1282.5ms/step | 15.6 samples/s +Step 18700/44000 | Loss: 2.9900 | LR: 3.16e-04 | 1278.9ms/step | 15.6 samples/s +Step 18710/44000 | Loss: 3.0242 | LR: 3.16e-04 | 1286.6ms/step | 15.5 samples/s +Step 18720/44000 | Loss: 3.0313 | LR: 3.16e-04 | 1286.8ms/step | 15.5 samples/s +Step 18730/44000 | Loss: 3.0340 | LR: 3.15e-04 | 1283.0ms/step | 15.6 samples/s +Step 18740/44000 | Loss: 3.0342 | LR: 3.15e-04 | 1285.8ms/step | 15.6 samples/s +Step 18750/44000 | Loss: 3.0379 | LR: 3.15e-04 | 1283.6ms/step | 15.6 samples/s +Validation at step 18750: Loss = 3.0332, Perplexity = 20.76 +Step 18760/44000 | Loss: 3.0182 | LR: 3.15e-04 | 3424.6ms/step | 5.8 samples/s +Step 18770/44000 | Loss: 3.0186 | LR: 3.15e-04 | 1284.8ms/step | 15.6 samples/s +Step 18780/44000 | Loss: 3.0116 | LR: 3.15e-04 | 1285.6ms/step | 15.6 samples/s +Step 18790/44000 | Loss: 3.0095 | LR: 3.14e-04 | 1282.9ms/step | 15.6 samples/s +Step 18800/44000 | Loss: 3.0307 | LR: 3.14e-04 | 1278.5ms/step | 15.6 samples/s +Step 18810/44000 | Loss: 3.0227 | LR: 3.14e-04 | 1280.1ms/step | 15.6 samples/s +Step 18820/44000 | Loss: 3.0142 | LR: 3.14e-04 | 1279.7ms/step | 15.6 samples/s +Step 18830/44000 | Loss: 3.0035 | LR: 3.14e-04 | 1279.8ms/step | 15.6 samples/s +Step 18840/44000 | Loss: 3.0294 | LR: 3.13e-04 | 1280.7ms/step | 15.6 samples/s +Step 18850/44000 | Loss: 2.9878 | LR: 3.13e-04 | 1279.8ms/step | 15.6 samples/s +Step 18860/44000 | Loss: 3.0176 | LR: 3.13e-04 | 1281.2ms/step | 15.6 samples/s +Step 18870/44000 | Loss: 3.0322 | LR: 3.13e-04 | 1280.7ms/step | 15.6 samples/s +Step 18880/44000 | Loss: 3.0012 | LR: 3.13e-04 | 1284.7ms/step | 15.6 samples/s +Step 18890/44000 | Loss: 3.0192 | LR: 3.13e-04 | 1287.8ms/step | 15.5 samples/s +Step 18900/44000 | Loss: 2.9801 | LR: 3.12e-04 | 1286.2ms/step | 15.5 samples/s +Step 18910/44000 | Loss: 3.0202 | LR: 3.12e-04 | 1285.7ms/step | 15.6 samples/s +Step 18920/44000 | Loss: 3.0227 | LR: 3.12e-04 | 1286.2ms/step | 15.5 samples/s +Step 18930/44000 | Loss: 3.0204 | LR: 3.12e-04 | 1285.1ms/step | 15.6 samples/s +Step 18940/44000 | Loss: 3.0281 | LR: 3.12e-04 | 1286.5ms/step | 15.5 samples/s +Step 18950/44000 | Loss: 2.9886 | LR: 3.12e-04 | 1286.9ms/step | 15.5 samples/s +Step 18960/44000 | Loss: 3.0032 | LR: 3.11e-04 | 1288.7ms/step | 15.5 samples/s +Step 18970/44000 | Loss: 3.0258 | LR: 3.11e-04 | 1289.4ms/step | 15.5 samples/s +Step 18980/44000 | Loss: 2.9975 | LR: 3.11e-04 | 1287.8ms/step | 15.5 samples/s +Step 18990/44000 | Loss: 3.0163 | LR: 3.11e-04 | 1284.8ms/step | 15.6 samples/s +Step 19000/44000 | Loss: 3.0136 | LR: 3.11e-04 | 1284.4ms/step | 15.6 samples/s +Validation at step 19000: Loss = 3.0297, Perplexity = 20.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-19000 +Step 19010/44000 | Loss: 2.9856 | LR: 3.11e-04 | 3755.3ms/step | 5.3 samples/s +Step 19020/44000 | Loss: 3.0237 | LR: 3.10e-04 | 1287.0ms/step | 15.5 samples/s +Step 19030/44000 | Loss: 3.0011 | LR: 3.10e-04 | 1286.3ms/step | 15.5 samples/s +Step 19040/44000 | Loss: 3.0148 | LR: 3.10e-04 | 1285.7ms/step | 15.6 samples/s +Step 19050/44000 | Loss: 3.0309 | LR: 3.10e-04 | 1283.6ms/step | 15.6 samples/s +Step 19060/44000 | Loss: 3.0458 | LR: 3.10e-04 | 1285.9ms/step | 15.6 samples/s +Step 19070/44000 | Loss: 3.0190 | LR: 3.09e-04 | 1285.4ms/step | 15.6 samples/s +Step 19080/44000 | Loss: 2.9891 | LR: 3.09e-04 | 1283.0ms/step | 15.6 samples/s +Step 19090/44000 | Loss: 3.0086 | LR: 3.09e-04 | 1281.7ms/step | 15.6 samples/s +Step 19100/44000 | Loss: 2.9926 | LR: 3.09e-04 | 1284.4ms/step | 15.6 samples/s +Step 19110/44000 | Loss: 3.0220 | LR: 3.09e-04 | 1283.3ms/step | 15.6 samples/s +Step 19120/44000 | Loss: 3.0168 | LR: 3.09e-04 | 1284.0ms/step | 15.6 samples/s +Step 19130/44000 | Loss: 3.0300 | LR: 3.08e-04 | 1285.3ms/step | 15.6 samples/s +Step 19140/44000 | Loss: 2.9956 | LR: 3.08e-04 | 1287.0ms/step | 15.5 samples/s +Step 19150/44000 | Loss: 3.0076 | LR: 3.08e-04 | 1283.1ms/step | 15.6 samples/s +Step 19160/44000 | Loss: 3.0099 | LR: 3.08e-04 | 1282.3ms/step | 15.6 samples/s +Step 19170/44000 | Loss: 2.9722 | LR: 3.08e-04 | 1282.9ms/step | 15.6 samples/s +Step 19180/44000 | Loss: 3.0345 | LR: 3.08e-04 | 1286.2ms/step | 15.5 samples/s +Step 19190/44000 | Loss: 2.9859 | LR: 3.07e-04 | 1285.8ms/step | 15.6 samples/s +Step 19200/44000 | Loss: 2.9921 | LR: 3.07e-04 | 1284.5ms/step | 15.6 samples/s +Step 19210/44000 | Loss: 3.0178 | LR: 3.07e-04 | 1288.0ms/step | 15.5 samples/s +Step 19220/44000 | Loss: 3.0041 | LR: 3.07e-04 | 1284.0ms/step | 15.6 samples/s +Step 19230/44000 | Loss: 3.0685 | LR: 3.07e-04 | 1284.1ms/step | 15.6 samples/s +Step 19240/44000 | Loss: 3.0199 | LR: 3.06e-04 | 1285.3ms/step | 15.6 samples/s +Step 19250/44000 | Loss: 3.0035 | LR: 3.06e-04 | 1284.1ms/step | 15.6 samples/s +Validation at step 19250: Loss = 3.0272, Perplexity = 20.64 +Step 19260/44000 | Loss: 3.0148 | LR: 3.06e-04 | 3430.9ms/step | 5.8 samples/s +Step 19270/44000 | Loss: 3.0004 | LR: 3.06e-04 | 1285.2ms/step | 15.6 samples/s +Step 19280/44000 | Loss: 2.9888 | LR: 3.06e-04 | 1285.6ms/step | 15.6 samples/s +Step 19290/44000 | Loss: 3.0090 | LR: 3.06e-04 | 1284.5ms/step | 15.6 samples/s +Step 19300/44000 | Loss: 2.9990 | LR: 3.05e-04 | 1285.6ms/step | 15.6 samples/s +Step 19310/44000 | Loss: 3.0261 | LR: 3.05e-04 | 1284.7ms/step | 15.6 samples/s +Step 19320/44000 | Loss: 3.0245 | LR: 3.05e-04 | 1288.5ms/step | 15.5 samples/s +Step 19330/44000 | Loss: 3.0342 | LR: 3.05e-04 | 1286.0ms/step | 15.6 samples/s +Step 19340/44000 | Loss: 3.0287 | LR: 3.05e-04 | 1282.6ms/step | 15.6 samples/s +Step 19350/44000 | Loss: 3.0138 | LR: 3.04e-04 | 1289.3ms/step | 15.5 samples/s +Step 19360/44000 | Loss: 3.0266 | LR: 3.04e-04 | 1281.7ms/step | 15.6 samples/s +Step 19370/44000 | Loss: 3.0206 | LR: 3.04e-04 | 1280.1ms/step | 15.6 samples/s +Step 19380/44000 | Loss: 2.9848 | LR: 3.04e-04 | 1278.1ms/step | 15.6 samples/s +Step 19390/44000 | Loss: 2.9930 | LR: 3.04e-04 | 1278.9ms/step | 15.6 samples/s +Step 19400/44000 | Loss: 3.0142 | LR: 3.04e-04 | 1282.3ms/step | 15.6 samples/s +Step 19410/44000 | Loss: 3.0038 | LR: 3.03e-04 | 1284.2ms/step | 15.6 samples/s +Step 19420/44000 | Loss: 3.0205 | LR: 3.03e-04 | 1279.4ms/step | 15.6 samples/s +Step 19430/44000 | Loss: 2.9804 | LR: 3.03e-04 | 1275.6ms/step | 15.7 samples/s +Step 19440/44000 | Loss: 3.0327 | LR: 3.03e-04 | 1280.4ms/step | 15.6 samples/s +Step 19450/44000 | Loss: 3.0045 | LR: 3.03e-04 | 1278.7ms/step | 15.6 samples/s +Step 19460/44000 | Loss: 3.0194 | LR: 3.03e-04 | 1279.0ms/step | 15.6 samples/s +Step 19470/44000 | Loss: 3.0266 | LR: 3.02e-04 | 1281.6ms/step | 15.6 samples/s +Step 19480/44000 | Loss: 2.9833 | LR: 3.02e-04 | 1282.9ms/step | 15.6 samples/s +Step 19490/44000 | Loss: 3.0332 | LR: 3.02e-04 | 1283.5ms/step | 15.6 samples/s +Step 19500/44000 | Loss: 2.9956 | LR: 3.02e-04 | 1282.9ms/step | 15.6 samples/s +Validation at step 19500: Loss = 3.0237, Perplexity = 20.57 +Step 19510/44000 | Loss: 3.0189 | LR: 3.02e-04 | 3427.2ms/step | 5.8 samples/s +Step 19520/44000 | Loss: 2.9986 | LR: 3.01e-04 | 1279.6ms/step | 15.6 samples/s +Step 19530/44000 | Loss: 2.9789 | LR: 3.01e-04 | 1282.1ms/step | 15.6 samples/s +Step 19540/44000 | Loss: 3.0090 | LR: 3.01e-04 | 1282.3ms/step | 15.6 samples/s +Step 19550/44000 | Loss: 2.9972 | LR: 3.01e-04 | 1280.1ms/step | 15.6 samples/s +Step 19560/44000 | Loss: 3.0046 | LR: 3.01e-04 | 1280.1ms/step | 15.6 samples/s +Step 19570/44000 | Loss: 3.0183 | LR: 3.01e-04 | 1280.1ms/step | 15.6 samples/s +Step 19580/44000 | Loss: 3.0276 | LR: 3.00e-04 | 1279.7ms/step | 15.6 samples/s +Step 19590/44000 | Loss: 3.0119 | LR: 3.00e-04 | 1281.8ms/step | 15.6 samples/s +Step 19600/44000 | Loss: 2.9887 | LR: 3.00e-04 | 1283.7ms/step | 15.6 samples/s +Step 19610/44000 | Loss: 3.0080 | LR: 3.00e-04 | 1282.0ms/step | 15.6 samples/s +Step 19620/44000 | Loss: 3.0044 | LR: 3.00e-04 | 1281.5ms/step | 15.6 samples/s +Step 19630/44000 | Loss: 3.0300 | LR: 3.00e-04 | 1281.6ms/step | 15.6 samples/s +Step 19640/44000 | Loss: 3.0253 | LR: 2.99e-04 | 1278.2ms/step | 15.6 samples/s +Step 19650/44000 | Loss: 2.9944 | LR: 2.99e-04 | 1274.4ms/step | 15.7 samples/s +Step 19660/44000 | Loss: 3.0081 | LR: 2.99e-04 | 1274.9ms/step | 15.7 samples/s +Step 19670/44000 | Loss: 3.0191 | LR: 2.99e-04 | 1281.3ms/step | 15.6 samples/s +Step 19680/44000 | Loss: 3.0340 | LR: 2.99e-04 | 1284.2ms/step | 15.6 samples/s +Step 19690/44000 | Loss: 2.9923 | LR: 2.98e-04 | 1283.7ms/step | 15.6 samples/s +Step 19700/44000 | Loss: 3.0258 | LR: 2.98e-04 | 1282.8ms/step | 15.6 samples/s +Step 19710/44000 | Loss: 3.0201 | LR: 2.98e-04 | 1286.0ms/step | 15.6 samples/s +Step 19720/44000 | Loss: 3.0088 | LR: 2.98e-04 | 1283.4ms/step | 15.6 samples/s +Step 19730/44000 | Loss: 3.0336 | LR: 2.98e-04 | 1282.7ms/step | 15.6 samples/s +Step 19740/44000 | Loss: 3.0070 | LR: 2.98e-04 | 1283.7ms/step | 15.6 samples/s +Step 19750/44000 | Loss: 2.9886 | LR: 2.97e-04 | 1286.8ms/step | 15.5 samples/s +Validation at step 19750: Loss = 3.0214, Perplexity = 20.52 +Step 19760/44000 | Loss: 3.0224 | LR: 2.97e-04 | 3439.8ms/step | 5.8 samples/s +Step 19770/44000 | Loss: 3.0302 | LR: 2.97e-04 | 1289.3ms/step | 15.5 samples/s +Step 19780/44000 | Loss: 3.0246 | LR: 2.97e-04 | 1287.7ms/step | 15.5 samples/s +Step 19790/44000 | Loss: 3.0038 | LR: 2.97e-04 | 1294.3ms/step | 15.5 samples/s +Step 19800/44000 | Loss: 3.0311 | LR: 2.96e-04 | 1291.2ms/step | 15.5 samples/s +Step 19810/44000 | Loss: 2.9905 | LR: 2.96e-04 | 1295.2ms/step | 15.4 samples/s +Step 19820/44000 | Loss: 2.9819 | LR: 2.96e-04 | 1289.7ms/step | 15.5 samples/s +Step 19830/44000 | Loss: 3.0291 | LR: 2.96e-04 | 1287.1ms/step | 15.5 samples/s +Step 19840/44000 | Loss: 3.0043 | LR: 2.96e-04 | 1283.1ms/step | 15.6 samples/s +Step 19850/44000 | Loss: 3.0019 | LR: 2.96e-04 | 1296.4ms/step | 15.4 samples/s +Step 19860/44000 | Loss: 3.0321 | LR: 2.95e-04 | 1292.4ms/step | 15.5 samples/s +Step 19870/44000 | Loss: 3.0281 | LR: 2.95e-04 | 1287.2ms/step | 15.5 samples/s +Step 19880/44000 | Loss: 3.0138 | LR: 2.95e-04 | 1283.4ms/step | 15.6 samples/s +Step 19890/44000 | Loss: 3.0152 | LR: 2.95e-04 | 1284.0ms/step | 15.6 samples/s +Step 19900/44000 | Loss: 2.9991 | LR: 2.95e-04 | 1285.6ms/step | 15.6 samples/s +Step 19910/44000 | Loss: 3.0106 | LR: 2.95e-04 | 1285.4ms/step | 15.6 samples/s +Step 19920/44000 | Loss: 3.0052 | LR: 2.94e-04 | 1282.8ms/step | 15.6 samples/s +Step 19930/44000 | Loss: 3.0160 | LR: 2.94e-04 | 1282.9ms/step | 15.6 samples/s +Step 19940/44000 | Loss: 2.9801 | LR: 2.94e-04 | 1283.4ms/step | 15.6 samples/s +Step 19950/44000 | Loss: 3.0005 | LR: 2.94e-04 | 1284.1ms/step | 15.6 samples/s +Step 19960/44000 | Loss: 3.0386 | LR: 2.94e-04 | 1283.9ms/step | 15.6 samples/s +Step 19970/44000 | Loss: 3.0263 | LR: 2.93e-04 | 1282.0ms/step | 15.6 samples/s +Step 19980/44000 | Loss: 3.0226 | LR: 2.93e-04 | 1282.3ms/step | 15.6 samples/s +Step 19990/44000 | Loss: 3.0116 | LR: 2.93e-04 | 1283.5ms/step | 15.6 samples/s +Step 20000/44000 | Loss: 3.0177 | LR: 2.93e-04 | 1284.5ms/step | 15.6 samples/s +Validation at step 20000: Loss = 3.0181, Perplexity = 20.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-20000 +Step 20010/44000 | Loss: 2.9929 | LR: 2.93e-04 | 3749.8ms/step | 5.3 samples/s +Step 20020/44000 | Loss: 3.0172 | LR: 2.93e-04 | 1279.6ms/step | 15.6 samples/s +Step 20030/44000 | Loss: 2.9919 | LR: 2.92e-04 | 1276.7ms/step | 15.7 samples/s +Step 20040/44000 | Loss: 2.9946 | LR: 2.92e-04 | 1278.1ms/step | 15.6 samples/s +Step 20050/44000 | Loss: 3.0095 | LR: 2.92e-04 | 1279.3ms/step | 15.6 samples/s +Step 20060/44000 | Loss: 2.9785 | LR: 2.92e-04 | 1280.4ms/step | 15.6 samples/s +Step 20070/44000 | Loss: 3.0052 | LR: 2.92e-04 | 1280.7ms/step | 15.6 samples/s +Step 20080/44000 | Loss: 2.9874 | LR: 2.91e-04 | 1284.3ms/step | 15.6 samples/s +Step 20090/44000 | Loss: 2.9970 | LR: 2.91e-04 | 1284.8ms/step | 15.6 samples/s +Step 20100/44000 | Loss: 3.0134 | LR: 2.91e-04 | 1282.9ms/step | 15.6 samples/s +Step 20110/44000 | Loss: 3.0052 | LR: 2.91e-04 | 1283.6ms/step | 15.6 samples/s +Step 20120/44000 | Loss: 3.0156 | LR: 2.91e-04 | 1282.8ms/step | 15.6 samples/s +Step 20130/44000 | Loss: 3.0108 | LR: 2.91e-04 | 1283.9ms/step | 15.6 samples/s +Step 20140/44000 | Loss: 3.0126 | LR: 2.90e-04 | 1283.2ms/step | 15.6 samples/s +Step 20150/44000 | Loss: 3.0004 | LR: 2.90e-04 | 1284.4ms/step | 15.6 samples/s +Step 20160/44000 | Loss: 3.0157 | LR: 2.90e-04 | 1283.9ms/step | 15.6 samples/s +Step 20170/44000 | Loss: 2.9701 | LR: 2.90e-04 | 1283.9ms/step | 15.6 samples/s +Step 20180/44000 | Loss: 2.9909 | LR: 2.90e-04 | 1280.2ms/step | 15.6 samples/s +Step 20190/44000 | Loss: 3.0205 | LR: 2.90e-04 | 1280.4ms/step | 15.6 samples/s +Step 20200/44000 | Loss: 2.9804 | LR: 2.89e-04 | 1282.9ms/step | 15.6 samples/s +Step 20210/44000 | Loss: 2.9712 | LR: 2.89e-04 | 1284.6ms/step | 15.6 samples/s +Step 20220/44000 | Loss: 3.0221 | LR: 2.89e-04 | 1285.0ms/step | 15.6 samples/s +Step 20230/44000 | Loss: 3.0086 | LR: 2.89e-04 | 1285.8ms/step | 15.6 samples/s +Step 20240/44000 | Loss: 3.0114 | LR: 2.89e-04 | 1286.5ms/step | 15.5 samples/s +Step 20250/44000 | Loss: 2.9937 | LR: 2.88e-04 | 1285.9ms/step | 15.6 samples/s +Validation at step 20250: Loss = 3.0161, Perplexity = 20.41 +Step 20260/44000 | Loss: 2.9631 | LR: 2.88e-04 | 3429.0ms/step | 5.8 samples/s +Step 20270/44000 | Loss: 3.0146 | LR: 2.88e-04 | 1284.0ms/step | 15.6 samples/s +Step 20280/44000 | Loss: 3.0096 | LR: 2.88e-04 | 1281.8ms/step | 15.6 samples/s +Step 20290/44000 | Loss: 3.0023 | LR: 2.88e-04 | 1284.0ms/step | 15.6 samples/s +Step 20300/44000 | Loss: 3.0249 | LR: 2.88e-04 | 1284.4ms/step | 15.6 samples/s +Step 20310/44000 | Loss: 2.9922 | LR: 2.87e-04 | 1283.5ms/step | 15.6 samples/s +Step 20320/44000 | Loss: 3.0019 | LR: 2.87e-04 | 1282.6ms/step | 15.6 samples/s +Step 20330/44000 | Loss: 2.9905 | LR: 2.87e-04 | 1283.5ms/step | 15.6 samples/s +Step 20340/44000 | Loss: 2.9797 | LR: 2.87e-04 | 1281.9ms/step | 15.6 samples/s +Step 20350/44000 | Loss: 3.0064 | LR: 2.87e-04 | 1284.1ms/step | 15.6 samples/s +Step 20360/44000 | Loss: 3.0193 | LR: 2.86e-04 | 1284.5ms/step | 15.6 samples/s +Step 20370/44000 | Loss: 2.9755 | LR: 2.86e-04 | 1283.7ms/step | 15.6 samples/s +Step 20380/44000 | Loss: 2.9841 | LR: 2.86e-04 | 1284.7ms/step | 15.6 samples/s +Step 20390/44000 | Loss: 3.0006 | LR: 2.86e-04 | 1284.4ms/step | 15.6 samples/s +Step 20400/44000 | Loss: 3.0176 | LR: 2.86e-04 | 1285.4ms/step | 15.6 samples/s +Step 20410/44000 | Loss: 2.9917 | LR: 2.86e-04 | 1285.1ms/step | 15.6 samples/s +Step 20420/44000 | Loss: 3.0150 | LR: 2.85e-04 | 1284.1ms/step | 15.6 samples/s +Step 20430/44000 | Loss: 2.9801 | LR: 2.85e-04 | 1284.1ms/step | 15.6 samples/s +Step 20440/44000 | Loss: 3.0092 | LR: 2.85e-04 | 1284.2ms/step | 15.6 samples/s +Step 20450/44000 | Loss: 2.9999 | LR: 2.85e-04 | 1286.9ms/step | 15.5 samples/s +Step 20460/44000 | Loss: 3.0027 | LR: 2.85e-04 | 1285.6ms/step | 15.6 samples/s +Step 20470/44000 | Loss: 3.0217 | LR: 2.84e-04 | 1286.5ms/step | 15.5 samples/s +Step 20480/44000 | Loss: 2.9969 | LR: 2.84e-04 | 1285.9ms/step | 15.6 samples/s +Step 20490/44000 | Loss: 2.9981 | LR: 2.84e-04 | 1285.5ms/step | 15.6 samples/s +Step 20500/44000 | Loss: 2.9795 | LR: 2.84e-04 | 1286.0ms/step | 15.6 samples/s +Validation at step 20500: Loss = 3.0131, Perplexity = 20.35 +Step 20510/44000 | Loss: 3.0245 | LR: 2.84e-04 | 3434.3ms/step | 5.8 samples/s +Step 20520/44000 | Loss: 2.9809 | LR: 2.84e-04 | 1284.3ms/step | 15.6 samples/s +Step 20530/44000 | Loss: 2.9710 | LR: 2.83e-04 | 1279.8ms/step | 15.6 samples/s +Step 20540/44000 | Loss: 2.9999 | LR: 2.83e-04 | 1279.4ms/step | 15.6 samples/s +Step 20550/44000 | Loss: 3.0111 | LR: 2.83e-04 | 1280.7ms/step | 15.6 samples/s +Step 20560/44000 | Loss: 3.0153 | LR: 2.83e-04 | 1279.9ms/step | 15.6 samples/s +Step 20570/44000 | Loss: 3.0111 | LR: 2.83e-04 | 1282.7ms/step | 15.6 samples/s +Step 20580/44000 | Loss: 2.9812 | LR: 2.83e-04 | 1282.9ms/step | 15.6 samples/s +Step 20590/44000 | Loss: 3.0195 | LR: 2.82e-04 | 1283.9ms/step | 15.6 samples/s +Step 20600/44000 | Loss: 2.9984 | LR: 2.82e-04 | 1283.9ms/step | 15.6 samples/s +Step 20610/44000 | Loss: 3.0054 | LR: 2.82e-04 | 1284.4ms/step | 15.6 samples/s +Step 20620/44000 | Loss: 3.0004 | LR: 2.82e-04 | 1281.6ms/step | 15.6 samples/s +Step 20630/44000 | Loss: 3.0069 | LR: 2.82e-04 | 1281.6ms/step | 15.6 samples/s +Step 20640/44000 | Loss: 2.9676 | LR: 2.81e-04 | 1283.9ms/step | 15.6 samples/s +Step 20650/44000 | Loss: 3.0147 | LR: 2.81e-04 | 1283.1ms/step | 15.6 samples/s +Step 20660/44000 | Loss: 2.9665 | LR: 2.81e-04 | 1284.2ms/step | 15.6 samples/s +Step 20670/44000 | Loss: 3.0132 | LR: 2.81e-04 | 1284.7ms/step | 15.6 samples/s +Step 20680/44000 | Loss: 3.0050 | LR: 2.81e-04 | 1285.4ms/step | 15.6 samples/s +Step 20690/44000 | Loss: 2.9922 | LR: 2.81e-04 | 1285.6ms/step | 15.6 samples/s +Step 20700/44000 | Loss: 3.0035 | LR: 2.80e-04 | 1286.0ms/step | 15.6 samples/s +Step 20710/44000 | Loss: 3.0095 | LR: 2.80e-04 | 1289.4ms/step | 15.5 samples/s +Step 20720/44000 | Loss: 3.0107 | LR: 2.80e-04 | 1284.6ms/step | 15.6 samples/s +Step 20730/44000 | Loss: 3.0238 | LR: 2.80e-04 | 1285.9ms/step | 15.6 samples/s +Step 20740/44000 | Loss: 3.0079 | LR: 2.80e-04 | 1289.4ms/step | 15.5 samples/s +Step 20750/44000 | Loss: 3.0030 | LR: 2.79e-04 | 1289.6ms/step | 15.5 samples/s +Validation at step 20750: Loss = 3.0103, Perplexity = 20.29 +Step 20760/44000 | Loss: 3.0086 | LR: 2.79e-04 | 3445.6ms/step | 5.8 samples/s +Step 20770/44000 | Loss: 2.9920 | LR: 2.79e-04 | 1286.1ms/step | 15.6 samples/s +Step 20780/44000 | Loss: 2.9695 | LR: 2.79e-04 | 1285.3ms/step | 15.6 samples/s +Step 20790/44000 | Loss: 2.9871 | LR: 2.79e-04 | 1284.2ms/step | 15.6 samples/s +Step 20800/44000 | Loss: 2.9890 | LR: 2.79e-04 | 1284.4ms/step | 15.6 samples/s +Step 20810/44000 | Loss: 3.0068 | LR: 2.78e-04 | 1283.7ms/step | 15.6 samples/s +Step 20820/44000 | Loss: 3.0037 | LR: 2.78e-04 | 1285.7ms/step | 15.6 samples/s +Step 20830/44000 | Loss: 3.0067 | LR: 2.78e-04 | 1286.0ms/step | 15.6 samples/s +Step 20840/44000 | Loss: 2.9940 | LR: 2.78e-04 | 1286.0ms/step | 15.6 samples/s +Step 20850/44000 | Loss: 2.9946 | LR: 2.78e-04 | 1286.3ms/step | 15.5 samples/s +Step 20860/44000 | Loss: 3.0144 | LR: 2.77e-04 | 1285.4ms/step | 15.6 samples/s +Step 20870/44000 | Loss: 3.0187 | LR: 2.77e-04 | 1283.7ms/step | 15.6 samples/s +Step 20880/44000 | Loss: 2.9900 | LR: 2.77e-04 | 1284.1ms/step | 15.6 samples/s +Step 20890/44000 | Loss: 2.9752 | LR: 2.77e-04 | 1282.8ms/step | 15.6 samples/s +Step 20900/44000 | Loss: 2.9776 | LR: 2.77e-04 | 1282.5ms/step | 15.6 samples/s +Step 20910/44000 | Loss: 3.0097 | LR: 2.77e-04 | 1282.9ms/step | 15.6 samples/s +Step 20920/44000 | Loss: 2.9725 | LR: 2.76e-04 | 1281.1ms/step | 15.6 samples/s +Step 20930/44000 | Loss: 2.9806 | LR: 2.76e-04 | 1280.6ms/step | 15.6 samples/s +Step 20940/44000 | Loss: 3.0048 | LR: 2.76e-04 | 1285.4ms/step | 15.6 samples/s +Step 20950/44000 | Loss: 2.9694 | LR: 2.76e-04 | 1285.3ms/step | 15.6 samples/s +Step 20960/44000 | Loss: 2.9854 | LR: 2.76e-04 | 1283.1ms/step | 15.6 samples/s +Step 20970/44000 | Loss: 2.9682 | LR: 2.75e-04 | 1280.3ms/step | 15.6 samples/s +Step 20980/44000 | Loss: 2.9736 | LR: 2.75e-04 | 1279.9ms/step | 15.6 samples/s +Step 20990/44000 | Loss: 2.9807 | LR: 2.75e-04 | 1279.7ms/step | 15.6 samples/s +Step 21000/44000 | Loss: 2.9814 | LR: 2.75e-04 | 1281.9ms/step | 15.6 samples/s +Validation at step 21000: Loss = 3.0090, Perplexity = 20.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-21000 +Step 21010/44000 | Loss: 2.9882 | LR: 2.75e-04 | 3731.9ms/step | 5.4 samples/s +Step 21020/44000 | Loss: 2.9880 | LR: 2.75e-04 | 1282.4ms/step | 15.6 samples/s +Step 21030/44000 | Loss: 3.0264 | LR: 2.74e-04 | 1279.9ms/step | 15.6 samples/s +Step 21040/44000 | Loss: 2.9714 | LR: 2.74e-04 | 1281.8ms/step | 15.6 samples/s +Step 21050/44000 | Loss: 2.9884 | LR: 2.74e-04 | 1285.9ms/step | 15.6 samples/s +Step 21060/44000 | Loss: 2.9728 | LR: 2.74e-04 | 1283.4ms/step | 15.6 samples/s +Step 21070/44000 | Loss: 3.0152 | LR: 2.74e-04 | 1284.4ms/step | 15.6 samples/s +Step 21080/44000 | Loss: 2.9600 | LR: 2.73e-04 | 1285.1ms/step | 15.6 samples/s +Step 21090/44000 | Loss: 3.0210 | LR: 2.73e-04 | 1280.3ms/step | 15.6 samples/s +Step 21100/44000 | Loss: 3.0027 | LR: 2.73e-04 | 1283.5ms/step | 15.6 samples/s +Step 21110/44000 | Loss: 2.9933 | LR: 2.73e-04 | 1282.6ms/step | 15.6 samples/s +Step 21120/44000 | Loss: 3.0043 | LR: 2.73e-04 | 1285.5ms/step | 15.6 samples/s +Step 21130/44000 | Loss: 3.0261 | LR: 2.73e-04 | 1284.7ms/step | 15.6 samples/s +Step 21140/44000 | Loss: 2.9880 | LR: 2.72e-04 | 1285.8ms/step | 15.6 samples/s +Step 21150/44000 | Loss: 2.9977 | LR: 2.72e-04 | 1285.8ms/step | 15.6 samples/s +Step 21160/44000 | Loss: 2.9782 | LR: 2.72e-04 | 1289.2ms/step | 15.5 samples/s +Step 21170/44000 | Loss: 2.9933 | LR: 2.72e-04 | 1285.6ms/step | 15.6 samples/s +Step 21180/44000 | Loss: 3.0018 | LR: 2.72e-04 | 1282.8ms/step | 15.6 samples/s +Step 21190/44000 | Loss: 2.9420 | LR: 2.71e-04 | 1283.0ms/step | 15.6 samples/s +Step 21200/44000 | Loss: 3.0118 | LR: 2.71e-04 | 1284.1ms/step | 15.6 samples/s +Step 21210/44000 | Loss: 2.9946 | LR: 2.71e-04 | 1284.9ms/step | 15.6 samples/s +Step 21220/44000 | Loss: 2.9834 | LR: 2.71e-04 | 1286.6ms/step | 15.5 samples/s +Step 21230/44000 | Loss: 2.9972 | LR: 2.71e-04 | 1284.6ms/step | 15.6 samples/s +Step 21240/44000 | Loss: 2.9864 | LR: 2.71e-04 | 1280.4ms/step | 15.6 samples/s +Step 21250/44000 | Loss: 2.9794 | LR: 2.70e-04 | 1281.3ms/step | 15.6 samples/s +Validation at step 21250: Loss = 3.0053, Perplexity = 20.19 +Step 21260/44000 | Loss: 3.0218 | LR: 2.70e-04 | 3431.7ms/step | 5.8 samples/s +Step 21270/44000 | Loss: 3.0002 | LR: 2.70e-04 | 1284.3ms/step | 15.6 samples/s +Step 21280/44000 | Loss: 2.9889 | LR: 2.70e-04 | 1283.6ms/step | 15.6 samples/s +Step 21290/44000 | Loss: 2.9852 | LR: 2.70e-04 | 1283.2ms/step | 15.6 samples/s +Step 21300/44000 | Loss: 2.9940 | LR: 2.70e-04 | 1281.4ms/step | 15.6 samples/s +Step 21310/44000 | Loss: 2.9960 | LR: 2.69e-04 | 1281.9ms/step | 15.6 samples/s +Step 21320/44000 | Loss: 3.0075 | LR: 2.69e-04 | 1281.3ms/step | 15.6 samples/s +Step 21330/44000 | Loss: 2.9919 | LR: 2.69e-04 | 1283.3ms/step | 15.6 samples/s +Step 21340/44000 | Loss: 2.9855 | LR: 2.69e-04 | 1285.5ms/step | 15.6 samples/s +Step 21350/44000 | Loss: 3.0197 | LR: 2.69e-04 | 1283.2ms/step | 15.6 samples/s +Step 21360/44000 | Loss: 2.9889 | LR: 2.68e-04 | 1283.0ms/step | 15.6 samples/s +Step 21370/44000 | Loss: 2.9922 | LR: 2.68e-04 | 1280.8ms/step | 15.6 samples/s +Step 21380/44000 | Loss: 2.9806 | LR: 2.68e-04 | 1281.8ms/step | 15.6 samples/s +Step 21390/44000 | Loss: 3.0110 | LR: 2.68e-04 | 1283.7ms/step | 15.6 samples/s +Step 21400/44000 | Loss: 2.9724 | LR: 2.68e-04 | 1283.2ms/step | 15.6 samples/s +Step 21410/44000 | Loss: 3.0140 | LR: 2.68e-04 | 1284.2ms/step | 15.6 samples/s +Step 21420/44000 | Loss: 2.9624 | LR: 2.67e-04 | 1281.3ms/step | 15.6 samples/s +Step 21430/44000 | Loss: 2.9778 | LR: 2.67e-04 | 1284.0ms/step | 15.6 samples/s +Step 21440/44000 | Loss: 2.9697 | LR: 2.67e-04 | 1284.9ms/step | 15.6 samples/s +Step 21450/44000 | Loss: 2.9863 | LR: 2.67e-04 | 1284.8ms/step | 15.6 samples/s +Step 21460/44000 | Loss: 2.9948 | LR: 2.67e-04 | 1286.1ms/step | 15.6 samples/s +Step 21470/44000 | Loss: 2.9995 | LR: 2.66e-04 | 1284.1ms/step | 15.6 samples/s +Step 21480/44000 | Loss: 2.9793 | LR: 2.66e-04 | 1284.3ms/step | 15.6 samples/s +Step 21490/44000 | Loss: 2.9951 | LR: 2.66e-04 | 1285.6ms/step | 15.6 samples/s +Step 21500/44000 | Loss: 2.9941 | LR: 2.66e-04 | 1283.8ms/step | 15.6 samples/s +Validation at step 21500: Loss = 3.0030, Perplexity = 20.15 +Step 21510/44000 | Loss: 2.9968 | LR: 2.66e-04 | 3429.2ms/step | 5.8 samples/s +Step 21520/44000 | Loss: 2.9848 | LR: 2.66e-04 | 1283.2ms/step | 15.6 samples/s +Step 21530/44000 | Loss: 2.9807 | LR: 2.65e-04 | 1283.5ms/step | 15.6 samples/s +Step 21540/44000 | Loss: 2.9920 | LR: 2.65e-04 | 1283.0ms/step | 15.6 samples/s +Step 21550/44000 | Loss: 2.9916 | LR: 2.65e-04 | 1283.6ms/step | 15.6 samples/s +Step 21560/44000 | Loss: 2.9950 | LR: 2.65e-04 | 1284.4ms/step | 15.6 samples/s +Step 21570/44000 | Loss: 2.9738 | LR: 2.65e-04 | 1283.3ms/step | 15.6 samples/s +Step 21580/44000 | Loss: 2.9674 | LR: 2.64e-04 | 1284.1ms/step | 15.6 samples/s +Step 21590/44000 | Loss: 2.9846 | LR: 2.64e-04 | 1283.3ms/step | 15.6 samples/s +Step 21600/44000 | Loss: 2.9847 | LR: 2.64e-04 | 1284.3ms/step | 15.6 samples/s +Step 21610/44000 | Loss: 2.9796 | LR: 2.64e-04 | 1285.2ms/step | 15.6 samples/s +Step 21620/44000 | Loss: 3.0022 | LR: 2.64e-04 | 1284.7ms/step | 15.6 samples/s +Step 21630/44000 | Loss: 2.9730 | LR: 2.64e-04 | 1283.2ms/step | 15.6 samples/s +Step 21640/44000 | Loss: 2.9655 | LR: 2.63e-04 | 1284.2ms/step | 15.6 samples/s +Step 21650/44000 | Loss: 2.9913 | LR: 2.63e-04 | 1282.2ms/step | 15.6 samples/s +Step 21660/44000 | Loss: 3.0270 | LR: 2.63e-04 | 1283.7ms/step | 15.6 samples/s +Step 21670/44000 | Loss: 2.9715 | LR: 2.63e-04 | 1283.6ms/step | 15.6 samples/s +Step 21680/44000 | Loss: 2.9931 | LR: 2.63e-04 | 1283.3ms/step | 15.6 samples/s +Step 21690/44000 | Loss: 3.0234 | LR: 2.62e-04 | 1282.4ms/step | 15.6 samples/s +Step 21700/44000 | Loss: 3.0039 | LR: 2.62e-04 | 1281.8ms/step | 15.6 samples/s +Step 21710/44000 | Loss: 2.9870 | LR: 2.62e-04 | 1282.8ms/step | 15.6 samples/s +Step 21720/44000 | Loss: 2.9606 | LR: 2.62e-04 | 1280.2ms/step | 15.6 samples/s +Step 21730/44000 | Loss: 2.9856 | LR: 2.62e-04 | 1280.6ms/step | 15.6 samples/s +Step 21740/44000 | Loss: 2.9753 | LR: 2.62e-04 | 1281.4ms/step | 15.6 samples/s +Step 21750/44000 | Loss: 2.9769 | LR: 2.61e-04 | 1281.0ms/step | 15.6 samples/s +Validation at step 21750: Loss = 3.0004, Perplexity = 20.09 +Step 21760/44000 | Loss: 3.0007 | LR: 2.61e-04 | 3425.0ms/step | 5.8 samples/s +Step 21770/44000 | Loss: 2.9844 | LR: 2.61e-04 | 1279.8ms/step | 15.6 samples/s +Step 21780/44000 | Loss: 2.9844 | LR: 2.61e-04 | 1278.7ms/step | 15.6 samples/s +Step 21790/44000 | Loss: 2.9510 | LR: 2.61e-04 | 1279.2ms/step | 15.6 samples/s +Step 21800/44000 | Loss: 2.9774 | LR: 2.60e-04 | 1280.4ms/step | 15.6 samples/s +Step 21810/44000 | Loss: 2.9737 | LR: 2.60e-04 | 1280.3ms/step | 15.6 samples/s +Step 21820/44000 | Loss: 2.9937 | LR: 2.60e-04 | 1280.2ms/step | 15.6 samples/s +Step 21830/44000 | Loss: 2.9648 | LR: 2.60e-04 | 1280.1ms/step | 15.6 samples/s +Step 21840/44000 | Loss: 2.9895 | LR: 2.60e-04 | 1279.9ms/step | 15.6 samples/s +Step 21850/44000 | Loss: 2.9847 | LR: 2.60e-04 | 1281.2ms/step | 15.6 samples/s +Step 21860/44000 | Loss: 2.9883 | LR: 2.59e-04 | 1279.4ms/step | 15.6 samples/s +Step 21870/44000 | Loss: 3.0005 | LR: 2.59e-04 | 1283.5ms/step | 15.6 samples/s +Step 21880/44000 | Loss: 2.9700 | LR: 2.59e-04 | 1284.8ms/step | 15.6 samples/s +Step 21890/44000 | Loss: 2.9729 | LR: 2.59e-04 | 1284.9ms/step | 15.6 samples/s +Step 21900/44000 | Loss: 2.9796 | LR: 2.59e-04 | 1284.7ms/step | 15.6 samples/s +Step 21910/44000 | Loss: 2.9797 | LR: 2.58e-04 | 1285.4ms/step | 15.6 samples/s +Step 21920/44000 | Loss: 2.9890 | LR: 2.58e-04 | 1284.5ms/step | 15.6 samples/s +Step 21930/44000 | Loss: 3.0130 | LR: 2.58e-04 | 1284.2ms/step | 15.6 samples/s +Step 21940/44000 | Loss: 2.9566 | LR: 2.58e-04 | 1283.6ms/step | 15.6 samples/s +Step 21950/44000 | Loss: 2.9868 | LR: 2.58e-04 | 1286.8ms/step | 15.5 samples/s +Step 21960/44000 | Loss: 2.9747 | LR: 2.58e-04 | 1283.5ms/step | 15.6 samples/s +Step 21970/44000 | Loss: 2.9702 | LR: 2.57e-04 | 1284.8ms/step | 15.6 samples/s +Step 21980/44000 | Loss: 2.9910 | LR: 2.57e-04 | 1285.3ms/step | 15.6 samples/s +Step 21990/44000 | Loss: 2.9943 | LR: 2.57e-04 | 1283.0ms/step | 15.6 samples/s +Step 22000/44000 | Loss: 2.9841 | LR: 2.57e-04 | 1282.0ms/step | 15.6 samples/s +Validation at step 22000: Loss = 2.9980, Perplexity = 20.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-22000 +Step 22010/44000 | Loss: 2.9852 | LR: 2.57e-04 | 3721.9ms/step | 5.4 samples/s +Step 22020/44000 | Loss: 2.9969 | LR: 2.56e-04 | 1278.7ms/step | 15.6 samples/s +Step 22030/44000 | Loss: 2.9926 | LR: 2.56e-04 | 1277.2ms/step | 15.7 samples/s +Step 22040/44000 | Loss: 3.0075 | LR: 2.56e-04 | 1274.8ms/step | 15.7 samples/s +Step 22050/44000 | Loss: 2.9786 | LR: 2.56e-04 | 1279.8ms/step | 15.6 samples/s +Step 22060/44000 | Loss: 2.9775 | LR: 2.56e-04 | 1277.1ms/step | 15.7 samples/s +Step 22070/44000 | Loss: 2.9821 | LR: 2.56e-04 | 1282.1ms/step | 15.6 samples/s +Step 22080/44000 | Loss: 2.9744 | LR: 2.55e-04 | 1283.7ms/step | 15.6 samples/s +Step 22090/44000 | Loss: 3.0005 | LR: 2.55e-04 | 1290.2ms/step | 15.5 samples/s +Step 22100/44000 | Loss: 2.9948 | LR: 2.55e-04 | 1288.6ms/step | 15.5 samples/s +Step 22110/44000 | Loss: 2.9796 | LR: 2.55e-04 | 1285.1ms/step | 15.6 samples/s +Step 22120/44000 | Loss: 2.9612 | LR: 2.55e-04 | 1286.3ms/step | 15.5 samples/s +Step 22130/44000 | Loss: 2.9707 | LR: 2.54e-04 | 1296.7ms/step | 15.4 samples/s +Step 22140/44000 | Loss: 2.9824 | LR: 2.54e-04 | 1279.8ms/step | 15.6 samples/s +Step 22150/44000 | Loss: 2.9611 | LR: 2.54e-04 | 1276.8ms/step | 15.7 samples/s +Step 22160/44000 | Loss: 2.9817 | LR: 2.54e-04 | 1274.2ms/step | 15.7 samples/s +Step 22170/44000 | Loss: 2.9997 | LR: 2.54e-04 | 1274.9ms/step | 15.7 samples/s +Step 22180/44000 | Loss: 2.9842 | LR: 2.54e-04 | 1275.0ms/step | 15.7 samples/s +Step 22190/44000 | Loss: 2.9827 | LR: 2.53e-04 | 1275.7ms/step | 15.7 samples/s +Step 22200/44000 | Loss: 2.9796 | LR: 2.53e-04 | 1275.9ms/step | 15.7 samples/s +Step 22210/44000 | Loss: 2.9834 | LR: 2.53e-04 | 1276.0ms/step | 15.7 samples/s +Step 22220/44000 | Loss: 2.9811 | LR: 2.53e-04 | 1278.0ms/step | 15.6 samples/s +Step 22230/44000 | Loss: 2.9973 | LR: 2.53e-04 | 1281.9ms/step | 15.6 samples/s +Step 22240/44000 | Loss: 3.0108 | LR: 2.52e-04 | 1280.5ms/step | 15.6 samples/s +Step 22250/44000 | Loss: 2.9963 | LR: 2.52e-04 | 1282.0ms/step | 15.6 samples/s +Validation at step 22250: Loss = 2.9958, Perplexity = 20.00 +Step 22260/44000 | Loss: 2.9634 | LR: 2.52e-04 | 3426.6ms/step | 5.8 samples/s +Step 22270/44000 | Loss: 2.9997 | LR: 2.52e-04 | 1277.8ms/step | 15.7 samples/s +Step 22280/44000 | Loss: 2.9783 | LR: 2.52e-04 | 1277.1ms/step | 15.7 samples/s +Step 22290/44000 | Loss: 2.9770 | LR: 2.52e-04 | 1275.6ms/step | 15.7 samples/s +Step 22300/44000 | Loss: 3.0130 | LR: 2.51e-04 | 1277.8ms/step | 15.7 samples/s +Step 22310/44000 | Loss: 2.9736 | LR: 2.51e-04 | 1284.8ms/step | 15.6 samples/s +Step 22320/44000 | Loss: 2.9613 | LR: 2.51e-04 | 1288.3ms/step | 15.5 samples/s +Step 22330/44000 | Loss: 2.9613 | LR: 2.51e-04 | 1287.5ms/step | 15.5 samples/s +Step 22340/44000 | Loss: 3.0103 | LR: 2.51e-04 | 1278.8ms/step | 15.6 samples/s +Step 22350/44000 | Loss: 2.9736 | LR: 2.50e-04 | 1281.4ms/step | 15.6 samples/s +Step 22360/44000 | Loss: 2.9743 | LR: 2.50e-04 | 1283.1ms/step | 15.6 samples/s +Step 22370/44000 | Loss: 2.9769 | LR: 2.50e-04 | 1282.5ms/step | 15.6 samples/s +Step 22380/44000 | Loss: 2.9812 | LR: 2.50e-04 | 1279.9ms/step | 15.6 samples/s +Step 22390/44000 | Loss: 2.9692 | LR: 2.50e-04 | 1279.7ms/step | 15.6 samples/s +Step 22400/44000 | Loss: 2.9537 | LR: 2.50e-04 | 1279.9ms/step | 15.6 samples/s +Step 22410/44000 | Loss: 2.9712 | LR: 2.49e-04 | 1279.5ms/step | 15.6 samples/s +Step 22420/44000 | Loss: 2.9722 | LR: 2.49e-04 | 1275.6ms/step | 15.7 samples/s +Step 22430/44000 | Loss: 2.9887 | LR: 2.49e-04 | 1274.2ms/step | 15.7 samples/s +Step 22440/44000 | Loss: 2.9777 | LR: 2.49e-04 | 1274.5ms/step | 15.7 samples/s +Step 22450/44000 | Loss: 2.9931 | LR: 2.49e-04 | 1273.5ms/step | 15.7 samples/s +Step 22460/44000 | Loss: 2.9721 | LR: 2.48e-04 | 1273.8ms/step | 15.7 samples/s +Step 22470/44000 | Loss: 2.9475 | LR: 2.48e-04 | 1274.8ms/step | 15.7 samples/s +Step 22480/44000 | Loss: 2.9615 | LR: 2.48e-04 | 1276.1ms/step | 15.7 samples/s +Step 22490/44000 | Loss: 2.9678 | LR: 2.48e-04 | 1278.6ms/step | 15.6 samples/s +Step 22500/44000 | Loss: 2.9534 | LR: 2.48e-04 | 1282.4ms/step | 15.6 samples/s +Validation at step 22500: Loss = 2.9928, Perplexity = 19.94 +Step 22510/44000 | Loss: 2.9618 | LR: 2.48e-04 | 3436.7ms/step | 5.8 samples/s +Step 22520/44000 | Loss: 3.0076 | LR: 2.47e-04 | 1290.1ms/step | 15.5 samples/s +Step 22530/44000 | Loss: 2.9466 | LR: 2.47e-04 | 1290.9ms/step | 15.5 samples/s +Step 22540/44000 | Loss: 2.9803 | LR: 2.47e-04 | 1290.4ms/step | 15.5 samples/s +Step 22550/44000 | Loss: 2.9486 | LR: 2.47e-04 | 1291.8ms/step | 15.5 samples/s +Step 22560/44000 | Loss: 2.9750 | LR: 2.47e-04 | 1292.7ms/step | 15.5 samples/s +Step 22570/44000 | Loss: 2.9501 | LR: 2.46e-04 | 1295.2ms/step | 15.4 samples/s +Step 22580/44000 | Loss: 2.9900 | LR: 2.46e-04 | 1288.1ms/step | 15.5 samples/s +Step 22590/44000 | Loss: 2.9848 | LR: 2.46e-04 | 1284.5ms/step | 15.6 samples/s +Step 22600/44000 | Loss: 2.9585 | LR: 2.46e-04 | 1282.8ms/step | 15.6 samples/s +Step 22610/44000 | Loss: 2.9733 | LR: 2.46e-04 | 1280.1ms/step | 15.6 samples/s +Step 22620/44000 | Loss: 2.9916 | LR: 2.46e-04 | 1279.2ms/step | 15.6 samples/s +Step 22630/44000 | Loss: 2.9701 | LR: 2.45e-04 | 1278.9ms/step | 15.6 samples/s +Step 22640/44000 | Loss: 2.9834 | LR: 2.45e-04 | 1279.7ms/step | 15.6 samples/s +Step 22650/44000 | Loss: 2.9731 | LR: 2.45e-04 | 1281.8ms/step | 15.6 samples/s +Step 22660/44000 | Loss: 3.0062 | LR: 2.45e-04 | 1283.5ms/step | 15.6 samples/s +Step 22670/44000 | Loss: 2.9804 | LR: 2.45e-04 | 1283.0ms/step | 15.6 samples/s +Step 22680/44000 | Loss: 2.9590 | LR: 2.44e-04 | 1284.2ms/step | 15.6 samples/s +Step 22690/44000 | Loss: 2.9899 | LR: 2.44e-04 | 1285.0ms/step | 15.6 samples/s +Step 22700/44000 | Loss: 2.9930 | LR: 2.44e-04 | 1278.4ms/step | 15.6 samples/s +Step 22710/44000 | Loss: 2.9473 | LR: 2.44e-04 | 1277.8ms/step | 15.7 samples/s +Step 22720/44000 | Loss: 2.9773 | LR: 2.44e-04 | 1277.0ms/step | 15.7 samples/s +Step 22730/44000 | Loss: 2.9928 | LR: 2.44e-04 | 1277.6ms/step | 15.7 samples/s +Step 22740/44000 | Loss: 2.9787 | LR: 2.43e-04 | 1277.9ms/step | 15.7 samples/s +Step 22750/44000 | Loss: 2.9553 | LR: 2.43e-04 | 1279.4ms/step | 15.6 samples/s +Validation at step 22750: Loss = 2.9905, Perplexity = 19.90 +Step 22760/44000 | Loss: 2.9905 | LR: 2.43e-04 | 3418.6ms/step | 5.9 samples/s +Step 22770/44000 | Loss: 2.9722 | LR: 2.43e-04 | 1279.3ms/step | 15.6 samples/s +Step 22780/44000 | Loss: 2.9810 | LR: 2.43e-04 | 1278.9ms/step | 15.6 samples/s +Step 22790/44000 | Loss: 2.9957 | LR: 2.42e-04 | 1278.2ms/step | 15.6 samples/s +Step 22800/44000 | Loss: 2.9976 | LR: 2.42e-04 | 1279.1ms/step | 15.6 samples/s +Step 22810/44000 | Loss: 2.9631 | LR: 2.42e-04 | 1277.7ms/step | 15.7 samples/s +Step 22820/44000 | Loss: 2.9767 | LR: 2.42e-04 | 1278.1ms/step | 15.6 samples/s +Step 22830/44000 | Loss: 3.0040 | LR: 2.42e-04 | 1279.4ms/step | 15.6 samples/s +Step 22840/44000 | Loss: 2.9864 | LR: 2.42e-04 | 1277.5ms/step | 15.7 samples/s +Step 22850/44000 | Loss: 2.9588 | LR: 2.41e-04 | 1280.6ms/step | 15.6 samples/s +Step 22860/44000 | Loss: 2.9984 | LR: 2.41e-04 | 1285.5ms/step | 15.6 samples/s +Step 22870/44000 | Loss: 2.9697 | LR: 2.41e-04 | 1283.0ms/step | 15.6 samples/s +Step 22880/44000 | Loss: 2.9707 | LR: 2.41e-04 | 1282.1ms/step | 15.6 samples/s +Step 22890/44000 | Loss: 2.9591 | LR: 2.41e-04 | 1284.7ms/step | 15.6 samples/s +Step 22900/44000 | Loss: 3.0043 | LR: 2.40e-04 | 1287.1ms/step | 15.5 samples/s +Step 22910/44000 | Loss: 2.9909 | LR: 2.40e-04 | 1284.9ms/step | 15.6 samples/s +Step 22920/44000 | Loss: 2.9781 | LR: 2.40e-04 | 1284.8ms/step | 15.6 samples/s +Step 22930/44000 | Loss: 2.9600 | LR: 2.40e-04 | 1284.5ms/step | 15.6 samples/s +Step 22940/44000 | Loss: 2.9636 | LR: 2.40e-04 | 1280.7ms/step | 15.6 samples/s +Step 22950/44000 | Loss: 2.9692 | LR: 2.40e-04 | 1283.8ms/step | 15.6 samples/s +Step 22960/44000 | Loss: 3.0100 | LR: 2.39e-04 | 1283.9ms/step | 15.6 samples/s +Step 22970/44000 | Loss: 2.9592 | LR: 2.39e-04 | 1285.9ms/step | 15.6 samples/s +Step 22980/44000 | Loss: 2.9734 | LR: 2.39e-04 | 1285.5ms/step | 15.6 samples/s +Step 22990/44000 | Loss: 2.9628 | LR: 2.39e-04 | 1281.4ms/step | 15.6 samples/s +Step 23000/44000 | Loss: 2.9590 | LR: 2.39e-04 | 1284.1ms/step | 15.6 samples/s +Validation at step 23000: Loss = 2.9884, Perplexity = 19.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-23000 +Step 23010/44000 | Loss: 2.9625 | LR: 2.38e-04 | 3732.7ms/step | 5.4 samples/s +Step 23020/44000 | Loss: 2.9733 | LR: 2.38e-04 | 1277.5ms/step | 15.7 samples/s +Step 23030/44000 | Loss: 2.9797 | LR: 2.38e-04 | 1280.9ms/step | 15.6 samples/s +Step 23040/44000 | Loss: 2.9931 | LR: 2.38e-04 | 1279.8ms/step | 15.6 samples/s +Step 23050/44000 | Loss: 3.0126 | LR: 2.38e-04 | 1280.3ms/step | 15.6 samples/s +Step 23060/44000 | Loss: 2.9751 | LR: 2.38e-04 | 1282.0ms/step | 15.6 samples/s +Step 23070/44000 | Loss: 2.9724 | LR: 2.37e-04 | 1280.8ms/step | 15.6 samples/s +Step 23080/44000 | Loss: 2.9756 | LR: 2.37e-04 | 1277.6ms/step | 15.7 samples/s +Step 23090/44000 | Loss: 2.9865 | LR: 2.37e-04 | 1279.4ms/step | 15.6 samples/s +Step 23100/44000 | Loss: 2.9744 | LR: 2.37e-04 | 1281.4ms/step | 15.6 samples/s +Step 23110/44000 | Loss: 2.9597 | LR: 2.37e-04 | 1281.0ms/step | 15.6 samples/s +Step 23120/44000 | Loss: 2.9669 | LR: 2.36e-04 | 1280.5ms/step | 15.6 samples/s +Step 23130/44000 | Loss: 2.9683 | LR: 2.36e-04 | 1280.7ms/step | 15.6 samples/s +Step 23140/44000 | Loss: 2.9468 | LR: 2.36e-04 | 1281.3ms/step | 15.6 samples/s +Step 23150/44000 | Loss: 2.9577 | LR: 2.36e-04 | 1281.4ms/step | 15.6 samples/s +Step 23160/44000 | Loss: 2.9786 | LR: 2.36e-04 | 1280.4ms/step | 15.6 samples/s +Step 23170/44000 | Loss: 2.9654 | LR: 2.36e-04 | 1291.9ms/step | 15.5 samples/s +Step 23180/44000 | Loss: 2.9810 | LR: 2.35e-04 | 1293.1ms/step | 15.5 samples/s +Step 23190/44000 | Loss: 2.9737 | LR: 2.35e-04 | 1293.7ms/step | 15.5 samples/s +Step 23200/44000 | Loss: 2.9743 | LR: 2.35e-04 | 1289.6ms/step | 15.5 samples/s +Step 23210/44000 | Loss: 2.9592 | LR: 2.35e-04 | 1292.3ms/step | 15.5 samples/s +Step 23220/44000 | Loss: 2.9779 | LR: 2.35e-04 | 1284.4ms/step | 15.6 samples/s +Step 23230/44000 | Loss: 2.9564 | LR: 2.34e-04 | 1285.7ms/step | 15.6 samples/s +Step 23240/44000 | Loss: 2.9773 | LR: 2.34e-04 | 1284.8ms/step | 15.6 samples/s +Step 23250/44000 | Loss: 2.9791 | LR: 2.34e-04 | 1286.4ms/step | 15.5 samples/s +Validation at step 23250: Loss = 2.9864, Perplexity = 19.81 +Step 23260/44000 | Loss: 2.9516 | LR: 2.34e-04 | 3428.1ms/step | 5.8 samples/s +Step 23270/44000 | Loss: 2.9731 | LR: 2.34e-04 | 1282.6ms/step | 15.6 samples/s +Step 23280/44000 | Loss: 2.9866 | LR: 2.34e-04 | 1283.4ms/step | 15.6 samples/s +Step 23290/44000 | Loss: 2.9661 | LR: 2.33e-04 | 1284.7ms/step | 15.6 samples/s +Step 23300/44000 | Loss: 2.9584 | LR: 2.33e-04 | 1287.9ms/step | 15.5 samples/s +Step 23310/44000 | Loss: 2.9381 | LR: 2.33e-04 | 1287.0ms/step | 15.5 samples/s +Step 23320/44000 | Loss: 2.9641 | LR: 2.33e-04 | 1285.0ms/step | 15.6 samples/s +Step 23330/44000 | Loss: 2.9547 | LR: 2.33e-04 | 1286.0ms/step | 15.6 samples/s +Step 23340/44000 | Loss: 2.9818 | LR: 2.32e-04 | 1284.5ms/step | 15.6 samples/s +Step 23350/44000 | Loss: 2.9663 | LR: 2.32e-04 | 1284.6ms/step | 15.6 samples/s +Step 23360/44000 | Loss: 3.0091 | LR: 2.32e-04 | 1283.9ms/step | 15.6 samples/s +Step 23370/44000 | Loss: 2.9603 | LR: 2.32e-04 | 1282.0ms/step | 15.6 samples/s +Step 23380/44000 | Loss: 2.9535 | LR: 2.32e-04 | 1283.0ms/step | 15.6 samples/s +Step 23390/44000 | Loss: 2.9391 | LR: 2.32e-04 | 1284.3ms/step | 15.6 samples/s +Step 23400/44000 | Loss: 2.9783 | LR: 2.31e-04 | 1286.8ms/step | 15.5 samples/s +Step 23410/44000 | Loss: 2.9666 | LR: 2.31e-04 | 1291.0ms/step | 15.5 samples/s +Step 23420/44000 | Loss: 2.9882 | LR: 2.31e-04 | 1289.8ms/step | 15.5 samples/s +Step 23430/44000 | Loss: 2.9577 | LR: 2.31e-04 | 1293.8ms/step | 15.5 samples/s +Step 23440/44000 | Loss: 2.9544 | LR: 2.31e-04 | 1296.8ms/step | 15.4 samples/s +Step 23450/44000 | Loss: 2.9758 | LR: 2.30e-04 | 1291.5ms/step | 15.5 samples/s +Step 23460/44000 | Loss: 2.9591 | LR: 2.30e-04 | 1293.9ms/step | 15.5 samples/s +Step 23470/44000 | Loss: 2.9805 | LR: 2.30e-04 | 1287.4ms/step | 15.5 samples/s +Step 23480/44000 | Loss: 2.9551 | LR: 2.30e-04 | 1288.8ms/step | 15.5 samples/s +Step 23490/44000 | Loss: 2.9722 | LR: 2.30e-04 | 1289.6ms/step | 15.5 samples/s +Step 23500/44000 | Loss: 2.9568 | LR: 2.30e-04 | 1289.7ms/step | 15.5 samples/s +Validation at step 23500: Loss = 2.9841, Perplexity = 19.77 +Step 23510/44000 | Loss: 2.9747 | LR: 2.29e-04 | 3459.0ms/step | 5.8 samples/s +Step 23520/44000 | Loss: 2.9644 | LR: 2.29e-04 | 1280.7ms/step | 15.6 samples/s +Step 23530/44000 | Loss: 2.9622 | LR: 2.29e-04 | 1281.2ms/step | 15.6 samples/s +Step 23540/44000 | Loss: 2.9474 | LR: 2.29e-04 | 1280.0ms/step | 15.6 samples/s +Step 23550/44000 | Loss: 2.9465 | LR: 2.29e-04 | 1279.2ms/step | 15.6 samples/s +Step 23560/44000 | Loss: 2.9648 | LR: 2.29e-04 | 1278.8ms/step | 15.6 samples/s +Step 23570/44000 | Loss: 2.9893 | LR: 2.28e-04 | 1280.0ms/step | 15.6 samples/s +Step 23580/44000 | Loss: 2.9420 | LR: 2.28e-04 | 1280.0ms/step | 15.6 samples/s +Step 23590/44000 | Loss: 2.9822 | LR: 2.28e-04 | 1280.4ms/step | 15.6 samples/s +Step 23600/44000 | Loss: 2.9968 | LR: 2.28e-04 | 1279.9ms/step | 15.6 samples/s +Step 23610/44000 | Loss: 2.9578 | LR: 2.28e-04 | 1280.6ms/step | 15.6 samples/s +Step 23620/44000 | Loss: 2.9623 | LR: 2.27e-04 | 1282.9ms/step | 15.6 samples/s +Step 23630/44000 | Loss: 2.9666 | LR: 2.27e-04 | 1283.2ms/step | 15.6 samples/s +Step 23640/44000 | Loss: 2.9828 | LR: 2.27e-04 | 1281.1ms/step | 15.6 samples/s +Step 23650/44000 | Loss: 2.9906 | LR: 2.27e-04 | 1279.7ms/step | 15.6 samples/s +Step 23660/44000 | Loss: 2.9698 | LR: 2.27e-04 | 1280.3ms/step | 15.6 samples/s +Step 23670/44000 | Loss: 2.9784 | LR: 2.27e-04 | 1280.4ms/step | 15.6 samples/s +Step 23680/44000 | Loss: 2.9767 | LR: 2.26e-04 | 1279.6ms/step | 15.6 samples/s +Step 23690/44000 | Loss: 2.9789 | LR: 2.26e-04 | 1279.2ms/step | 15.6 samples/s +Step 23700/44000 | Loss: 2.9624 | LR: 2.26e-04 | 1279.3ms/step | 15.6 samples/s +Step 23710/44000 | Loss: 2.9469 | LR: 2.26e-04 | 1279.1ms/step | 15.6 samples/s +Step 23720/44000 | Loss: 2.9532 | LR: 2.26e-04 | 1278.8ms/step | 15.6 samples/s +Step 23730/44000 | Loss: 2.9728 | LR: 2.25e-04 | 1279.6ms/step | 15.6 samples/s +Step 23740/44000 | Loss: 2.9700 | LR: 2.25e-04 | 1280.3ms/step | 15.6 samples/s +Step 23750/44000 | Loss: 2.9549 | LR: 2.25e-04 | 1285.1ms/step | 15.6 samples/s +Validation at step 23750: Loss = 2.9816, Perplexity = 19.72 +Step 23760/44000 | Loss: 2.9808 | LR: 2.25e-04 | 3431.6ms/step | 5.8 samples/s +Step 23770/44000 | Loss: 2.9556 | LR: 2.25e-04 | 1284.7ms/step | 15.6 samples/s +Step 23780/44000 | Loss: 2.9537 | LR: 2.25e-04 | 1283.2ms/step | 15.6 samples/s +Step 23790/44000 | Loss: 2.9470 | LR: 2.24e-04 | 1285.7ms/step | 15.6 samples/s +Step 23800/44000 | Loss: 2.9727 | LR: 2.24e-04 | 1285.6ms/step | 15.6 samples/s +Step 23810/44000 | Loss: 2.9470 | LR: 2.24e-04 | 1284.5ms/step | 15.6 samples/s +Step 23820/44000 | Loss: 2.9915 | LR: 2.24e-04 | 1285.0ms/step | 15.6 samples/s +Step 23830/44000 | Loss: 2.9595 | LR: 2.24e-04 | 1286.8ms/step | 15.5 samples/s +Step 23840/44000 | Loss: 2.9728 | LR: 2.23e-04 | 1289.8ms/step | 15.5 samples/s +Step 23850/44000 | Loss: 2.9755 | LR: 2.23e-04 | 1290.7ms/step | 15.5 samples/s +Step 23860/44000 | Loss: 2.9651 | LR: 2.23e-04 | 1295.3ms/step | 15.4 samples/s +Step 23870/44000 | Loss: 2.9775 | LR: 2.23e-04 | 1294.4ms/step | 15.5 samples/s +Step 23880/44000 | Loss: 2.9596 | LR: 2.23e-04 | 1289.9ms/step | 15.5 samples/s +Step 23890/44000 | Loss: 2.9616 | LR: 2.23e-04 | 1290.1ms/step | 15.5 samples/s +Step 23900/44000 | Loss: 2.9583 | LR: 2.22e-04 | 1289.8ms/step | 15.5 samples/s +Step 23910/44000 | Loss: 2.9549 | LR: 2.22e-04 | 1287.2ms/step | 15.5 samples/s +Step 23920/44000 | Loss: 2.9982 | LR: 2.22e-04 | 1288.4ms/step | 15.5 samples/s +Step 23930/44000 | Loss: 2.9721 | LR: 2.22e-04 | 1290.0ms/step | 15.5 samples/s +Step 23940/44000 | Loss: 2.9831 | LR: 2.22e-04 | 1287.5ms/step | 15.5 samples/s +Step 23950/44000 | Loss: 2.9767 | LR: 2.21e-04 | 1280.9ms/step | 15.6 samples/s +Step 23960/44000 | Loss: 3.0002 | LR: 2.21e-04 | 1281.9ms/step | 15.6 samples/s +Step 23970/44000 | Loss: 2.9792 | LR: 2.21e-04 | 1282.3ms/step | 15.6 samples/s +Step 23980/44000 | Loss: 2.9591 | LR: 2.21e-04 | 1283.2ms/step | 15.6 samples/s +Step 23990/44000 | Loss: 2.9381 | LR: 2.21e-04 | 1280.3ms/step | 15.6 samples/s +Step 24000/44000 | Loss: 2.9588 | LR: 2.21e-04 | 1281.3ms/step | 15.6 samples/s +Validation at step 24000: Loss = 2.9798, Perplexity = 19.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-24000 +Step 24010/44000 | Loss: 2.9701 | LR: 2.20e-04 | 3736.4ms/step | 5.4 samples/s +Step 24020/44000 | Loss: 2.9654 | LR: 2.20e-04 | 1284.3ms/step | 15.6 samples/s +Step 24030/44000 | Loss: 2.9874 | LR: 2.20e-04 | 1283.3ms/step | 15.6 samples/s +Step 24040/44000 | Loss: 3.0001 | LR: 2.20e-04 | 1282.3ms/step | 15.6 samples/s +Step 24050/44000 | Loss: 2.9902 | LR: 2.20e-04 | 1282.0ms/step | 15.6 samples/s +Step 24060/44000 | Loss: 2.9754 | LR: 2.19e-04 | 1281.9ms/step | 15.6 samples/s +Step 24070/44000 | Loss: 2.9918 | LR: 2.19e-04 | 1280.4ms/step | 15.6 samples/s +Step 24080/44000 | Loss: 2.9503 | LR: 2.19e-04 | 1282.0ms/step | 15.6 samples/s +Step 24090/44000 | Loss: 2.9498 | LR: 2.19e-04 | 1283.0ms/step | 15.6 samples/s +Step 24100/44000 | Loss: 2.9679 | LR: 2.19e-04 | 1280.4ms/step | 15.6 samples/s +Step 24110/44000 | Loss: 2.9777 | LR: 2.19e-04 | 1279.8ms/step | 15.6 samples/s +Step 24120/44000 | Loss: 2.9323 | LR: 2.18e-04 | 1282.1ms/step | 15.6 samples/s +Step 24130/44000 | Loss: 2.9804 | LR: 2.18e-04 | 1284.8ms/step | 15.6 samples/s +Step 24140/44000 | Loss: 2.9575 | LR: 2.18e-04 | 1286.0ms/step | 15.6 samples/s +Step 24150/44000 | Loss: 2.9835 | LR: 2.18e-04 | 1285.5ms/step | 15.6 samples/s +Step 24160/44000 | Loss: 2.9949 | LR: 2.18e-04 | 1282.5ms/step | 15.6 samples/s +Step 24170/44000 | Loss: 2.9511 | LR: 2.17e-04 | 1283.1ms/step | 15.6 samples/s +Step 24180/44000 | Loss: 2.9598 | LR: 2.17e-04 | 1284.4ms/step | 15.6 samples/s +Step 24190/44000 | Loss: 2.9787 | LR: 2.17e-04 | 1286.8ms/step | 15.5 samples/s +Step 24200/44000 | Loss: 2.9527 | LR: 2.17e-04 | 1287.8ms/step | 15.5 samples/s +Step 24210/44000 | Loss: 2.9450 | LR: 2.17e-04 | 1288.0ms/step | 15.5 samples/s +Step 24220/44000 | Loss: 2.9629 | LR: 2.17e-04 | 1285.9ms/step | 15.6 samples/s +Step 24230/44000 | Loss: 2.9851 | LR: 2.16e-04 | 1285.6ms/step | 15.6 samples/s +Step 24240/44000 | Loss: 2.9480 | LR: 2.16e-04 | 1283.6ms/step | 15.6 samples/s +Step 24250/44000 | Loss: 2.9714 | LR: 2.16e-04 | 1283.3ms/step | 15.6 samples/s +Validation at step 24250: Loss = 2.9780, Perplexity = 19.65 +Step 24260/44000 | Loss: 2.9432 | LR: 2.16e-04 | 3430.8ms/step | 5.8 samples/s +Step 24270/44000 | Loss: 2.9617 | LR: 2.16e-04 | 1283.3ms/step | 15.6 samples/s +Step 24280/44000 | Loss: 2.9861 | LR: 2.16e-04 | 1283.2ms/step | 15.6 samples/s +Step 24290/44000 | Loss: 2.9682 | LR: 2.15e-04 | 1282.2ms/step | 15.6 samples/s +Step 24300/44000 | Loss: 2.9540 | LR: 2.15e-04 | 1284.6ms/step | 15.6 samples/s +Step 24310/44000 | Loss: 2.9669 | LR: 2.15e-04 | 1285.6ms/step | 15.6 samples/s +Step 24320/44000 | Loss: 2.9668 | LR: 2.15e-04 | 1284.7ms/step | 15.6 samples/s +Step 24330/44000 | Loss: 2.9844 | LR: 2.15e-04 | 1280.7ms/step | 15.6 samples/s +Step 24340/44000 | Loss: 2.9485 | LR: 2.14e-04 | 1281.5ms/step | 15.6 samples/s +Step 24350/44000 | Loss: 2.9394 | LR: 2.14e-04 | 1283.0ms/step | 15.6 samples/s +Step 24360/44000 | Loss: 2.9641 | LR: 2.14e-04 | 1284.7ms/step | 15.6 samples/s +Step 24370/44000 | Loss: 2.9340 | LR: 2.14e-04 | 1284.2ms/step | 15.6 samples/s +Step 24380/44000 | Loss: 2.9280 | LR: 2.14e-04 | 1289.5ms/step | 15.5 samples/s +Step 24390/44000 | Loss: 2.9357 | LR: 2.14e-04 | 1291.2ms/step | 15.5 samples/s +Step 24400/44000 | Loss: 2.9661 | LR: 2.13e-04 | 1291.3ms/step | 15.5 samples/s +Step 24410/44000 | Loss: 2.9649 | LR: 2.13e-04 | 1285.2ms/step | 15.6 samples/s +Step 24420/44000 | Loss: 2.9780 | LR: 2.13e-04 | 1286.3ms/step | 15.5 samples/s +Step 24430/44000 | Loss: 2.9647 | LR: 2.13e-04 | 1290.1ms/step | 15.5 samples/s +Step 24440/44000 | Loss: 2.9696 | LR: 2.13e-04 | 1288.1ms/step | 15.5 samples/s +Step 24450/44000 | Loss: 2.9368 | LR: 2.12e-04 | 1288.7ms/step | 15.5 samples/s +Step 24460/44000 | Loss: 2.9425 | LR: 2.12e-04 | 1285.9ms/step | 15.6 samples/s +Step 24470/44000 | Loss: 2.9689 | LR: 2.12e-04 | 1288.0ms/step | 15.5 samples/s +Step 24480/44000 | Loss: 2.9630 | LR: 2.12e-04 | 1292.1ms/step | 15.5 samples/s +Step 24490/44000 | Loss: 2.9590 | LR: 2.12e-04 | 1290.4ms/step | 15.5 samples/s +Step 24500/44000 | Loss: 2.9741 | LR: 2.12e-04 | 1287.4ms/step | 15.5 samples/s +Validation at step 24500: Loss = 2.9759, Perplexity = 19.61 +Step 24510/44000 | Loss: 2.9650 | LR: 2.11e-04 | 3430.0ms/step | 5.8 samples/s +Step 24520/44000 | Loss: 2.9341 | LR: 2.11e-04 | 1280.1ms/step | 15.6 samples/s +Step 24530/44000 | Loss: 2.9578 | LR: 2.11e-04 | 1280.1ms/step | 15.6 samples/s +Step 24540/44000 | Loss: 2.9789 | LR: 2.11e-04 | 1279.4ms/step | 15.6 samples/s +Step 24550/44000 | Loss: 2.9919 | LR: 2.11e-04 | 1279.7ms/step | 15.6 samples/s +Step 24560/44000 | Loss: 2.9577 | LR: 2.10e-04 | 1282.5ms/step | 15.6 samples/s +Step 24570/44000 | Loss: 3.0265 | LR: 2.10e-04 | 1280.4ms/step | 15.6 samples/s +Step 24580/44000 | Loss: 2.9409 | LR: 2.10e-04 | 1279.5ms/step | 15.6 samples/s +Step 24590/44000 | Loss: 2.9887 | LR: 2.10e-04 | 1279.5ms/step | 15.6 samples/s +Step 24600/44000 | Loss: 2.9598 | LR: 2.10e-04 | 1278.2ms/step | 15.6 samples/s +Step 24610/44000 | Loss: 2.9764 | LR: 2.10e-04 | 1280.7ms/step | 15.6 samples/s +Step 24620/44000 | Loss: 2.9702 | LR: 2.09e-04 | 1282.6ms/step | 15.6 samples/s +Step 24630/44000 | Loss: 2.9937 | LR: 2.09e-04 | 1282.4ms/step | 15.6 samples/s +Step 24640/44000 | Loss: 2.9618 | LR: 2.09e-04 | 1283.6ms/step | 15.6 samples/s +Step 24650/44000 | Loss: 2.9629 | LR: 2.09e-04 | 1282.5ms/step | 15.6 samples/s +Step 24660/44000 | Loss: 2.9557 | LR: 2.09e-04 | 1279.4ms/step | 15.6 samples/s +Step 24670/44000 | Loss: 2.9568 | LR: 2.09e-04 | 1279.5ms/step | 15.6 samples/s +Step 24680/44000 | Loss: 2.9526 | LR: 2.08e-04 | 1280.2ms/step | 15.6 samples/s +Step 24690/44000 | Loss: 2.9579 | LR: 2.08e-04 | 1282.1ms/step | 15.6 samples/s +Step 24700/44000 | Loss: 2.9537 | LR: 2.08e-04 | 1281.4ms/step | 15.6 samples/s +Step 24710/44000 | Loss: 2.9562 | LR: 2.08e-04 | 1282.7ms/step | 15.6 samples/s +Step 24720/44000 | Loss: 2.9434 | LR: 2.08e-04 | 1285.4ms/step | 15.6 samples/s +Step 24730/44000 | Loss: 2.9548 | LR: 2.07e-04 | 1285.8ms/step | 15.6 samples/s +Step 24740/44000 | Loss: 3.0013 | LR: 2.07e-04 | 1287.5ms/step | 15.5 samples/s +Step 24750/44000 | Loss: 2.9392 | LR: 2.07e-04 | 1286.3ms/step | 15.5 samples/s +Validation at step 24750: Loss = 2.9741, Perplexity = 19.57 +Step 24760/44000 | Loss: 2.9616 | LR: 2.07e-04 | 3436.9ms/step | 5.8 samples/s +Step 24770/44000 | Loss: 2.9605 | LR: 2.07e-04 | 1287.4ms/step | 15.5 samples/s +Step 24780/44000 | Loss: 2.9832 | LR: 2.07e-04 | 1287.0ms/step | 15.5 samples/s +Step 24790/44000 | Loss: 2.9406 | LR: 2.06e-04 | 1283.6ms/step | 15.6 samples/s +Step 24800/44000 | Loss: 2.9441 | LR: 2.06e-04 | 1286.8ms/step | 15.5 samples/s +Step 24810/44000 | Loss: 2.9758 | LR: 2.06e-04 | 1290.8ms/step | 15.5 samples/s +Step 24820/44000 | Loss: 2.9597 | LR: 2.06e-04 | 1286.2ms/step | 15.5 samples/s +Step 24830/44000 | Loss: 2.9337 | LR: 2.06e-04 | 1283.4ms/step | 15.6 samples/s +Step 24840/44000 | Loss: 2.9511 | LR: 2.05e-04 | 1287.0ms/step | 15.5 samples/s +Step 24850/44000 | Loss: 2.9557 | LR: 2.05e-04 | 1282.6ms/step | 15.6 samples/s +Step 24860/44000 | Loss: 2.9550 | LR: 2.05e-04 | 1280.3ms/step | 15.6 samples/s +Step 24870/44000 | Loss: 2.9466 | LR: 2.05e-04 | 1281.6ms/step | 15.6 samples/s +Step 24880/44000 | Loss: 2.9722 | LR: 2.05e-04 | 1282.3ms/step | 15.6 samples/s +Step 24890/44000 | Loss: 2.9957 | LR: 2.05e-04 | 1282.6ms/step | 15.6 samples/s +Step 24900/44000 | Loss: 2.9514 | LR: 2.04e-04 | 1286.2ms/step | 15.5 samples/s +Step 24910/44000 | Loss: 2.9612 | LR: 2.04e-04 | 1286.2ms/step | 15.5 samples/s +Step 24920/44000 | Loss: 2.9681 | LR: 2.04e-04 | 1284.5ms/step | 15.6 samples/s +Step 24930/44000 | Loss: 2.9617 | LR: 2.04e-04 | 1285.4ms/step | 15.6 samples/s +Step 24940/44000 | Loss: 2.9707 | LR: 2.04e-04 | 1283.1ms/step | 15.6 samples/s +Step 24950/44000 | Loss: 2.9493 | LR: 2.04e-04 | 1282.8ms/step | 15.6 samples/s +Step 24960/44000 | Loss: 2.9960 | LR: 2.03e-04 | 1283.5ms/step | 15.6 samples/s +Step 24970/44000 | Loss: 2.9468 | LR: 2.03e-04 | 1282.6ms/step | 15.6 samples/s +Step 24980/44000 | Loss: 2.9817 | LR: 2.03e-04 | 1281.9ms/step | 15.6 samples/s +Step 24990/44000 | Loss: 2.9330 | LR: 2.03e-04 | 1283.1ms/step | 15.6 samples/s +Step 25000/44000 | Loss: 2.9757 | LR: 2.03e-04 | 1282.2ms/step | 15.6 samples/s +Validation at step 25000: Loss = 2.9719, Perplexity = 19.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-25000 +Step 25010/44000 | Loss: 2.9383 | LR: 2.02e-04 | 3749.0ms/step | 5.3 samples/s +Step 25020/44000 | Loss: 2.9694 | LR: 2.02e-04 | 1278.4ms/step | 15.6 samples/s +Step 25030/44000 | Loss: 2.9503 | LR: 2.02e-04 | 1281.2ms/step | 15.6 samples/s +Step 25040/44000 | Loss: 2.9638 | LR: 2.02e-04 | 1284.2ms/step | 15.6 samples/s +Step 25050/44000 | Loss: 2.9559 | LR: 2.02e-04 | 1283.3ms/step | 15.6 samples/s +Step 25060/44000 | Loss: 2.9523 | LR: 2.02e-04 | 1283.7ms/step | 15.6 samples/s +Step 25070/44000 | Loss: 2.9752 | LR: 2.01e-04 | 1284.1ms/step | 15.6 samples/s +Step 25080/44000 | Loss: 2.9383 | LR: 2.01e-04 | 1283.1ms/step | 15.6 samples/s +Step 25090/44000 | Loss: 2.9771 | LR: 2.01e-04 | 1283.2ms/step | 15.6 samples/s +Step 25100/44000 | Loss: 2.9383 | LR: 2.01e-04 | 1284.0ms/step | 15.6 samples/s +Step 25110/44000 | Loss: 2.9625 | LR: 2.01e-04 | 1282.6ms/step | 15.6 samples/s +Step 25120/44000 | Loss: 2.9359 | LR: 2.00e-04 | 1282.5ms/step | 15.6 samples/s +Step 25130/44000 | Loss: 2.9903 | LR: 2.00e-04 | 1282.5ms/step | 15.6 samples/s +Step 25140/44000 | Loss: 2.9993 | LR: 2.00e-04 | 1283.3ms/step | 15.6 samples/s +Step 25150/44000 | Loss: 2.9223 | LR: 2.00e-04 | 1283.4ms/step | 15.6 samples/s +Step 25160/44000 | Loss: 2.9371 | LR: 2.00e-04 | 1280.0ms/step | 15.6 samples/s +Step 25170/44000 | Loss: 2.9711 | LR: 2.00e-04 | 1280.4ms/step | 15.6 samples/s +Step 25180/44000 | Loss: 2.9098 | LR: 1.99e-04 | 1279.2ms/step | 15.6 samples/s +Step 25190/44000 | Loss: 2.9528 | LR: 1.99e-04 | 1281.6ms/step | 15.6 samples/s +Step 25200/44000 | Loss: 2.9756 | LR: 1.99e-04 | 1287.0ms/step | 15.5 samples/s +Step 25210/44000 | Loss: 2.9703 | LR: 1.99e-04 | 1288.3ms/step | 15.5 samples/s +Step 25220/44000 | Loss: 2.9388 | LR: 1.99e-04 | 1290.5ms/step | 15.5 samples/s +Step 25230/44000 | Loss: 2.9600 | LR: 1.99e-04 | 1289.8ms/step | 15.5 samples/s +Step 25240/44000 | Loss: 2.9415 | LR: 1.98e-04 | 1288.3ms/step | 15.5 samples/s +Step 25250/44000 | Loss: 2.9737 | LR: 1.98e-04 | 1286.4ms/step | 15.5 samples/s +Validation at step 25250: Loss = 2.9704, Perplexity = 19.50 +Step 25260/44000 | Loss: 2.9341 | LR: 1.98e-04 | 3455.8ms/step | 5.8 samples/s +Step 25270/44000 | Loss: 2.9646 | LR: 1.98e-04 | 1286.6ms/step | 15.5 samples/s +Step 25280/44000 | Loss: 2.9314 | LR: 1.98e-04 | 1286.3ms/step | 15.5 samples/s +Step 25290/44000 | Loss: 2.9522 | LR: 1.97e-04 | 1286.0ms/step | 15.6 samples/s +Step 25300/44000 | Loss: 2.9585 | LR: 1.97e-04 | 1284.4ms/step | 15.6 samples/s +Step 25310/44000 | Loss: 2.9088 | LR: 1.97e-04 | 1283.5ms/step | 15.6 samples/s +Step 25320/44000 | Loss: 2.9480 | LR: 1.97e-04 | 1284.3ms/step | 15.6 samples/s +Step 25330/44000 | Loss: 2.9594 | LR: 1.97e-04 | 1281.6ms/step | 15.6 samples/s +Step 25340/44000 | Loss: 2.9518 | LR: 1.97e-04 | 1280.5ms/step | 15.6 samples/s +Step 25350/44000 | Loss: 2.9443 | LR: 1.96e-04 | 1281.9ms/step | 15.6 samples/s +Step 25360/44000 | Loss: 2.9912 | LR: 1.96e-04 | 1281.6ms/step | 15.6 samples/s +Step 25370/44000 | Loss: 2.9376 | LR: 1.96e-04 | 1282.0ms/step | 15.6 samples/s +Step 25380/44000 | Loss: 2.9802 | LR: 1.96e-04 | 1281.9ms/step | 15.6 samples/s +Step 25390/44000 | Loss: 2.9517 | LR: 1.96e-04 | 1282.7ms/step | 15.6 samples/s +Step 25400/44000 | Loss: 2.9645 | LR: 1.96e-04 | 1282.4ms/step | 15.6 samples/s +Step 25410/44000 | Loss: 2.9938 | LR: 1.95e-04 | 1280.1ms/step | 15.6 samples/s +Step 25420/44000 | Loss: 2.9777 | LR: 1.95e-04 | 1278.9ms/step | 15.6 samples/s +Step 25430/44000 | Loss: 2.9571 | LR: 1.95e-04 | 1279.3ms/step | 15.6 samples/s +Step 25440/44000 | Loss: 2.9566 | LR: 1.95e-04 | 1280.0ms/step | 15.6 samples/s +Step 25450/44000 | Loss: 2.9928 | LR: 1.95e-04 | 1280.4ms/step | 15.6 samples/s +Step 25460/44000 | Loss: 2.9649 | LR: 1.94e-04 | 1279.5ms/step | 15.6 samples/s +Step 25470/44000 | Loss: 2.9440 | LR: 1.94e-04 | 1280.0ms/step | 15.6 samples/s +Step 25480/44000 | Loss: 2.9595 | LR: 1.94e-04 | 1280.0ms/step | 15.6 samples/s +Step 25490/44000 | Loss: 2.9591 | LR: 1.94e-04 | 1280.2ms/step | 15.6 samples/s +Step 25500/44000 | Loss: 2.9493 | LR: 1.94e-04 | 1280.2ms/step | 15.6 samples/s +Validation at step 25500: Loss = 2.9683, Perplexity = 19.46 +Step 25510/44000 | Loss: 2.9390 | LR: 1.94e-04 | 3426.7ms/step | 5.8 samples/s +Step 25520/44000 | Loss: 2.9371 | LR: 1.93e-04 | 1281.6ms/step | 15.6 samples/s +Step 25530/44000 | Loss: 2.9402 | LR: 1.93e-04 | 1278.9ms/step | 15.6 samples/s +Step 25540/44000 | Loss: 2.9471 | LR: 1.93e-04 | 1279.3ms/step | 15.6 samples/s +Step 25550/44000 | Loss: 2.9580 | LR: 1.93e-04 | 1279.6ms/step | 15.6 samples/s +Step 25560/44000 | Loss: 2.9660 | LR: 1.93e-04 | 1280.9ms/step | 15.6 samples/s +Step 25570/44000 | Loss: 2.9482 | LR: 1.92e-04 | 1280.3ms/step | 15.6 samples/s +Step 25580/44000 | Loss: 2.9821 | LR: 1.92e-04 | 1280.7ms/step | 15.6 samples/s +Step 25590/44000 | Loss: 2.9598 | LR: 1.92e-04 | 1279.7ms/step | 15.6 samples/s +Step 25600/44000 | Loss: 2.9281 | LR: 1.92e-04 | 1279.9ms/step | 15.6 samples/s +Step 25610/44000 | Loss: 2.9418 | LR: 1.92e-04 | 1281.1ms/step | 15.6 samples/s +Step 25620/44000 | Loss: 2.9305 | LR: 1.92e-04 | 1282.2ms/step | 15.6 samples/s +Step 25630/44000 | Loss: 2.9624 | LR: 1.91e-04 | 1281.0ms/step | 15.6 samples/s +Step 25640/44000 | Loss: 2.9459 | LR: 1.91e-04 | 1280.6ms/step | 15.6 samples/s +Step 25650/44000 | Loss: 2.9499 | LR: 1.91e-04 | 1280.9ms/step | 15.6 samples/s +Step 25660/44000 | Loss: 2.9718 | LR: 1.91e-04 | 1281.8ms/step | 15.6 samples/s +Step 25670/44000 | Loss: 2.9614 | LR: 1.91e-04 | 1285.9ms/step | 15.6 samples/s +Step 25680/44000 | Loss: 2.9282 | LR: 1.91e-04 | 1287.4ms/step | 15.5 samples/s +Step 25690/44000 | Loss: 2.9483 | LR: 1.90e-04 | 1288.4ms/step | 15.5 samples/s +Step 25700/44000 | Loss: 2.9315 | LR: 1.90e-04 | 1284.3ms/step | 15.6 samples/s +Step 25710/44000 | Loss: 2.9467 | LR: 1.90e-04 | 1283.8ms/step | 15.6 samples/s +Step 25720/44000 | Loss: 2.9415 | LR: 1.90e-04 | 1283.2ms/step | 15.6 samples/s +Step 25730/44000 | Loss: 2.9337 | LR: 1.90e-04 | 1285.8ms/step | 15.6 samples/s +Step 25740/44000 | Loss: 2.9477 | LR: 1.89e-04 | 1286.2ms/step | 15.5 samples/s +Step 25750/44000 | Loss: 2.9482 | LR: 1.89e-04 | 1286.1ms/step | 15.6 samples/s +Validation at step 25750: Loss = 2.9670, Perplexity = 19.43 +Step 25760/44000 | Loss: 2.9480 | LR: 1.89e-04 | 3426.8ms/step | 5.8 samples/s +Step 25770/44000 | Loss: 2.9723 | LR: 1.89e-04 | 1282.1ms/step | 15.6 samples/s +Step 25780/44000 | Loss: 2.9564 | LR: 1.89e-04 | 1281.2ms/step | 15.6 samples/s +Step 25790/44000 | Loss: 2.9910 | LR: 1.89e-04 | 1279.8ms/step | 15.6 samples/s +Step 25800/44000 | Loss: 2.9135 | LR: 1.88e-04 | 1279.0ms/step | 15.6 samples/s +Step 25810/44000 | Loss: 2.9492 | LR: 1.88e-04 | 1280.5ms/step | 15.6 samples/s +Step 25820/44000 | Loss: 2.9502 | LR: 1.88e-04 | 1280.0ms/step | 15.6 samples/s +Step 25830/44000 | Loss: 2.9522 | LR: 1.88e-04 | 1279.9ms/step | 15.6 samples/s +Step 25840/44000 | Loss: 2.9630 | LR: 1.88e-04 | 1278.9ms/step | 15.6 samples/s +Step 25850/44000 | Loss: 2.9268 | LR: 1.88e-04 | 1281.1ms/step | 15.6 samples/s +Step 25860/44000 | Loss: 2.9237 | LR: 1.87e-04 | 1282.6ms/step | 15.6 samples/s +Step 25870/44000 | Loss: 2.9270 | LR: 1.87e-04 | 1283.3ms/step | 15.6 samples/s +Step 25880/44000 | Loss: 2.9713 | LR: 1.87e-04 | 1285.3ms/step | 15.6 samples/s +Step 25890/44000 | Loss: 2.9515 | LR: 1.87e-04 | 1288.1ms/step | 15.5 samples/s +Step 25900/44000 | Loss: 2.9544 | LR: 1.87e-04 | 1285.1ms/step | 15.6 samples/s +Step 25910/44000 | Loss: 2.9398 | LR: 1.87e-04 | 1282.4ms/step | 15.6 samples/s +Step 25920/44000 | Loss: 2.9368 | LR: 1.86e-04 | 1282.3ms/step | 15.6 samples/s +Step 25930/44000 | Loss: 2.9686 | LR: 1.86e-04 | 1283.7ms/step | 15.6 samples/s +Step 25940/44000 | Loss: 2.9276 | LR: 1.86e-04 | 1285.7ms/step | 15.6 samples/s +Step 25950/44000 | Loss: 2.9290 | LR: 1.86e-04 | 1284.7ms/step | 15.6 samples/s +Step 25960/44000 | Loss: 2.9463 | LR: 1.86e-04 | 1283.8ms/step | 15.6 samples/s +Step 25970/44000 | Loss: 2.9227 | LR: 1.85e-04 | 1283.4ms/step | 15.6 samples/s +Step 25980/44000 | Loss: 2.9669 | LR: 1.85e-04 | 1279.4ms/step | 15.6 samples/s +Step 25990/44000 | Loss: 2.9604 | LR: 1.85e-04 | 1279.3ms/step | 15.6 samples/s +Step 26000/44000 | Loss: 2.9393 | LR: 1.85e-04 | 1281.0ms/step | 15.6 samples/s +Validation at step 26000: Loss = 2.9657, Perplexity = 19.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-26000 +Step 26010/44000 | Loss: 2.9547 | LR: 1.85e-04 | 3732.9ms/step | 5.4 samples/s +Step 26020/44000 | Loss: 2.9532 | LR: 1.85e-04 | 1282.7ms/step | 15.6 samples/s +Step 26030/44000 | Loss: 2.9394 | LR: 1.84e-04 | 1285.2ms/step | 15.6 samples/s +Step 26040/44000 | Loss: 2.9259 | LR: 1.84e-04 | 1284.8ms/step | 15.6 samples/s +Step 26050/44000 | Loss: 2.9626 | LR: 1.84e-04 | 1284.5ms/step | 15.6 samples/s +Step 26060/44000 | Loss: 2.9137 | LR: 1.84e-04 | 1283.5ms/step | 15.6 samples/s +Step 26070/44000 | Loss: 2.9426 | LR: 1.84e-04 | 1285.0ms/step | 15.6 samples/s +Step 26080/44000 | Loss: 2.9282 | LR: 1.84e-04 | 1286.0ms/step | 15.6 samples/s +Step 26090/44000 | Loss: 2.9427 | LR: 1.83e-04 | 1286.6ms/step | 15.5 samples/s +Step 26100/44000 | Loss: 2.9438 | LR: 1.83e-04 | 1286.0ms/step | 15.6 samples/s +Step 26110/44000 | Loss: 2.9618 | LR: 1.83e-04 | 1281.9ms/step | 15.6 samples/s +Step 26120/44000 | Loss: 2.9420 | LR: 1.83e-04 | 1283.8ms/step | 15.6 samples/s +Step 26130/44000 | Loss: 2.9629 | LR: 1.83e-04 | 1287.8ms/step | 15.5 samples/s +Step 26140/44000 | Loss: 2.9674 | LR: 1.82e-04 | 1289.0ms/step | 15.5 samples/s +Step 26150/44000 | Loss: 2.9579 | LR: 1.82e-04 | 1288.7ms/step | 15.5 samples/s +Step 26160/44000 | Loss: 2.9420 | LR: 1.82e-04 | 1288.3ms/step | 15.5 samples/s +Step 26170/44000 | Loss: 2.9447 | LR: 1.82e-04 | 1286.7ms/step | 15.5 samples/s +Step 26180/44000 | Loss: 2.9595 | LR: 1.82e-04 | 1289.6ms/step | 15.5 samples/s +Step 26190/44000 | Loss: 2.9311 | LR: 1.82e-04 | 1295.8ms/step | 15.4 samples/s +Step 26200/44000 | Loss: 2.9132 | LR: 1.81e-04 | 1293.5ms/step | 15.5 samples/s +Step 26210/44000 | Loss: 2.9766 | LR: 1.81e-04 | 1295.5ms/step | 15.4 samples/s +Step 26220/44000 | Loss: 2.9501 | LR: 1.81e-04 | 1300.0ms/step | 15.4 samples/s +Step 26230/44000 | Loss: 2.9599 | LR: 1.81e-04 | 1290.1ms/step | 15.5 samples/s +Step 26240/44000 | Loss: 2.9517 | LR: 1.81e-04 | 1286.1ms/step | 15.6 samples/s +Step 26250/44000 | Loss: 2.9594 | LR: 1.81e-04 | 1287.1ms/step | 15.5 samples/s +Validation at step 26250: Loss = 2.9640, Perplexity = 19.38 +Step 26260/44000 | Loss: 2.9432 | LR: 1.80e-04 | 3433.5ms/step | 5.8 samples/s +Step 26270/44000 | Loss: 2.9398 | LR: 1.80e-04 | 1283.8ms/step | 15.6 samples/s +Step 26280/44000 | Loss: 2.9596 | LR: 1.80e-04 | 1284.1ms/step | 15.6 samples/s +Step 26290/44000 | Loss: 2.9740 | LR: 1.80e-04 | 1285.1ms/step | 15.6 samples/s +Step 26300/44000 | Loss: 2.9553 | LR: 1.80e-04 | 1282.6ms/step | 15.6 samples/s +Step 26310/44000 | Loss: 2.9405 | LR: 1.80e-04 | 1283.2ms/step | 15.6 samples/s +Step 26320/44000 | Loss: 2.9583 | LR: 1.79e-04 | 1282.2ms/step | 15.6 samples/s +Step 26330/44000 | Loss: 2.9893 | LR: 1.79e-04 | 1283.1ms/step | 15.6 samples/s +Step 26340/44000 | Loss: 2.9450 | LR: 1.79e-04 | 1283.5ms/step | 15.6 samples/s +Step 26350/44000 | Loss: 2.9608 | LR: 1.79e-04 | 1279.6ms/step | 15.6 samples/s +Step 26360/44000 | Loss: 2.9364 | LR: 1.79e-04 | 1280.8ms/step | 15.6 samples/s +Step 26370/44000 | Loss: 2.9628 | LR: 1.78e-04 | 1281.2ms/step | 15.6 samples/s +Step 26380/44000 | Loss: 2.9657 | LR: 1.78e-04 | 1281.9ms/step | 15.6 samples/s +Step 26390/44000 | Loss: 2.9661 | LR: 1.78e-04 | 1287.1ms/step | 15.5 samples/s +Step 26400/44000 | Loss: 2.9658 | LR: 1.78e-04 | 1284.7ms/step | 15.6 samples/s +Step 26410/44000 | Loss: 2.9287 | LR: 1.78e-04 | 1283.0ms/step | 15.6 samples/s +Step 26420/44000 | Loss: 2.9255 | LR: 1.78e-04 | 1284.5ms/step | 15.6 samples/s +Step 26430/44000 | Loss: 2.9672 | LR: 1.77e-04 | 1284.6ms/step | 15.6 samples/s +Step 26440/44000 | Loss: 2.9236 | LR: 1.77e-04 | 1280.6ms/step | 15.6 samples/s +Step 26450/44000 | Loss: 2.9488 | LR: 1.77e-04 | 1280.1ms/step | 15.6 samples/s +Step 26460/44000 | Loss: 2.9524 | LR: 1.77e-04 | 1281.1ms/step | 15.6 samples/s +Step 26470/44000 | Loss: 2.9428 | LR: 1.77e-04 | 1281.2ms/step | 15.6 samples/s +Step 26480/44000 | Loss: 2.9425 | LR: 1.77e-04 | 1280.2ms/step | 15.6 samples/s +Step 26490/44000 | Loss: 2.9791 | LR: 1.76e-04 | 1279.0ms/step | 15.6 samples/s +Step 26500/44000 | Loss: 2.9619 | LR: 1.76e-04 | 1280.6ms/step | 15.6 samples/s +Validation at step 26500: Loss = 2.9625, Perplexity = 19.35 +Step 26510/44000 | Loss: 2.9725 | LR: 1.76e-04 | 3430.1ms/step | 5.8 samples/s +Step 26520/44000 | Loss: 2.9474 | LR: 1.76e-04 | 1280.4ms/step | 15.6 samples/s +Step 26530/44000 | Loss: 2.9524 | LR: 1.76e-04 | 1283.6ms/step | 15.6 samples/s +Step 26540/44000 | Loss: 2.9581 | LR: 1.76e-04 | 1283.8ms/step | 15.6 samples/s +Step 26550/44000 | Loss: 2.9645 | LR: 1.75e-04 | 1288.2ms/step | 15.5 samples/s +Step 26560/44000 | Loss: 2.9955 | LR: 1.75e-04 | 1287.8ms/step | 15.5 samples/s +Step 26570/44000 | Loss: 2.9224 | LR: 1.75e-04 | 1284.3ms/step | 15.6 samples/s +Step 26580/44000 | Loss: 2.9737 | LR: 1.75e-04 | 1285.5ms/step | 15.6 samples/s +Step 26590/44000 | Loss: 2.9774 | LR: 1.75e-04 | 1287.9ms/step | 15.5 samples/s +Step 26600/44000 | Loss: 2.9492 | LR: 1.74e-04 | 1287.8ms/step | 15.5 samples/s +Step 26610/44000 | Loss: 2.9563 | LR: 1.74e-04 | 1293.5ms/step | 15.5 samples/s +Step 26620/44000 | Loss: 2.9356 | LR: 1.74e-04 | 1285.6ms/step | 15.6 samples/s +Step 26630/44000 | Loss: 2.9422 | LR: 1.74e-04 | 1284.9ms/step | 15.6 samples/s +Step 26640/44000 | Loss: 2.9369 | LR: 1.74e-04 | 1286.7ms/step | 15.5 samples/s +Step 26650/44000 | Loss: 2.9694 | LR: 1.74e-04 | 1285.8ms/step | 15.6 samples/s +Step 26660/44000 | Loss: 2.9580 | LR: 1.73e-04 | 1284.2ms/step | 15.6 samples/s +Step 26670/44000 | Loss: 2.9455 | LR: 1.73e-04 | 1285.0ms/step | 15.6 samples/s +Step 26680/44000 | Loss: 2.9590 | LR: 1.73e-04 | 1287.7ms/step | 15.5 samples/s +Step 26690/44000 | Loss: 2.9683 | LR: 1.73e-04 | 1287.3ms/step | 15.5 samples/s +Step 26700/44000 | Loss: 2.9624 | LR: 1.73e-04 | 1284.8ms/step | 15.6 samples/s +Step 26710/44000 | Loss: 2.9326 | LR: 1.73e-04 | 1286.2ms/step | 15.5 samples/s +Step 26720/44000 | Loss: 2.9412 | LR: 1.72e-04 | 1286.5ms/step | 15.5 samples/s +Step 26730/44000 | Loss: 2.9285 | LR: 1.72e-04 | 1287.8ms/step | 15.5 samples/s +Step 26740/44000 | Loss: 2.9514 | LR: 1.72e-04 | 1286.8ms/step | 15.5 samples/s +Step 26750/44000 | Loss: 2.9894 | LR: 1.72e-04 | 1285.9ms/step | 15.6 samples/s +Validation at step 26750: Loss = 2.9614, Perplexity = 19.32 +Step 26760/44000 | Loss: 2.9423 | LR: 1.72e-04 | 3433.7ms/step | 5.8 samples/s +Step 26770/44000 | Loss: 2.9191 | LR: 1.72e-04 | 1284.3ms/step | 15.6 samples/s +Step 26780/44000 | Loss: 2.9305 | LR: 1.71e-04 | 1283.1ms/step | 15.6 samples/s +Step 26790/44000 | Loss: 2.9436 | LR: 1.71e-04 | 1283.3ms/step | 15.6 samples/s +Step 26800/44000 | Loss: 2.9543 | LR: 1.71e-04 | 1283.9ms/step | 15.6 samples/s +Step 26810/44000 | Loss: 2.9548 | LR: 1.71e-04 | 1284.1ms/step | 15.6 samples/s +Step 26820/44000 | Loss: 2.9718 | LR: 1.71e-04 | 1284.3ms/step | 15.6 samples/s +Step 26830/44000 | Loss: 2.9485 | LR: 1.71e-04 | 1281.2ms/step | 15.6 samples/s +Step 26840/44000 | Loss: 2.9699 | LR: 1.70e-04 | 1284.8ms/step | 15.6 samples/s +Step 26850/44000 | Loss: 2.9473 | LR: 1.70e-04 | 1284.3ms/step | 15.6 samples/s +Step 26860/44000 | Loss: 2.9322 | LR: 1.70e-04 | 1285.0ms/step | 15.6 samples/s +Step 26870/44000 | Loss: 2.9592 | LR: 1.70e-04 | 1283.3ms/step | 15.6 samples/s +Step 26880/44000 | Loss: 2.9239 | LR: 1.70e-04 | 1283.7ms/step | 15.6 samples/s +Step 26890/44000 | Loss: 2.9699 | LR: 1.69e-04 | 1284.7ms/step | 15.6 samples/s +Step 26900/44000 | Loss: 2.9187 | LR: 1.69e-04 | 1284.0ms/step | 15.6 samples/s +Step 26910/44000 | Loss: 2.9631 | LR: 1.69e-04 | 1282.3ms/step | 15.6 samples/s +Step 26920/44000 | Loss: 2.9363 | LR: 1.69e-04 | 1281.2ms/step | 15.6 samples/s +Step 26930/44000 | Loss: 2.9390 | LR: 1.69e-04 | 1282.3ms/step | 15.6 samples/s +Step 26940/44000 | Loss: 2.9513 | LR: 1.69e-04 | 1283.3ms/step | 15.6 samples/s +Step 26950/44000 | Loss: 2.9539 | LR: 1.68e-04 | 1286.3ms/step | 15.5 samples/s +Step 26960/44000 | Loss: 2.9380 | LR: 1.68e-04 | 1284.6ms/step | 15.6 samples/s +Step 26970/44000 | Loss: 2.9413 | LR: 1.68e-04 | 1288.3ms/step | 15.5 samples/s +Step 26980/44000 | Loss: 2.9722 | LR: 1.68e-04 | 1285.9ms/step | 15.6 samples/s +Step 26990/44000 | Loss: 2.9346 | LR: 1.68e-04 | 1283.4ms/step | 15.6 samples/s +Step 27000/44000 | Loss: 2.9483 | LR: 1.68e-04 | 1281.9ms/step | 15.6 samples/s +Validation at step 27000: Loss = 2.9599, Perplexity = 19.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-27000 +Step 27010/44000 | Loss: 2.9639 | LR: 1.67e-04 | 3724.7ms/step | 5.4 samples/s +Step 27020/44000 | Loss: 2.9448 | LR: 1.67e-04 | 1281.9ms/step | 15.6 samples/s +Step 27030/44000 | Loss: 2.9631 | LR: 1.67e-04 | 1282.8ms/step | 15.6 samples/s +Step 27040/44000 | Loss: 2.9510 | LR: 1.67e-04 | 1284.4ms/step | 15.6 samples/s +Step 27050/44000 | Loss: 2.9602 | LR: 1.67e-04 | 1284.7ms/step | 15.6 samples/s +Step 27060/44000 | Loss: 2.9198 | LR: 1.67e-04 | 1282.9ms/step | 15.6 samples/s +Step 27070/44000 | Loss: 2.9552 | LR: 1.66e-04 | 1282.6ms/step | 15.6 samples/s +Step 27080/44000 | Loss: 2.9560 | LR: 1.66e-04 | 1283.2ms/step | 15.6 samples/s +Step 27090/44000 | Loss: 2.9426 | LR: 1.66e-04 | 1282.3ms/step | 15.6 samples/s +Step 27100/44000 | Loss: 2.9307 | LR: 1.66e-04 | 1283.3ms/step | 15.6 samples/s +Step 27110/44000 | Loss: 2.9246 | LR: 1.66e-04 | 1283.6ms/step | 15.6 samples/s +Step 27120/44000 | Loss: 2.9317 | LR: 1.66e-04 | 1284.0ms/step | 15.6 samples/s +Step 27130/44000 | Loss: 2.9257 | LR: 1.65e-04 | 1279.5ms/step | 15.6 samples/s +Step 27140/44000 | Loss: 2.9390 | LR: 1.65e-04 | 1280.5ms/step | 15.6 samples/s +Step 27150/44000 | Loss: 2.9565 | LR: 1.65e-04 | 1281.3ms/step | 15.6 samples/s +Step 27160/44000 | Loss: 2.9235 | LR: 1.65e-04 | 1280.7ms/step | 15.6 samples/s +Step 27170/44000 | Loss: 2.9513 | LR: 1.65e-04 | 1279.5ms/step | 15.6 samples/s +Step 27180/44000 | Loss: 2.9579 | LR: 1.65e-04 | 1279.0ms/step | 15.6 samples/s +Step 27190/44000 | Loss: 2.9555 | LR: 1.64e-04 | 1278.4ms/step | 15.6 samples/s +Step 27200/44000 | Loss: 2.9608 | LR: 1.64e-04 | 1278.1ms/step | 15.6 samples/s +Step 27210/44000 | Loss: 2.9456 | LR: 1.64e-04 | 1278.5ms/step | 15.6 samples/s +Step 27220/44000 | Loss: 2.9280 | LR: 1.64e-04 | 1279.0ms/step | 15.6 samples/s +Step 27230/44000 | Loss: 2.9219 | LR: 1.64e-04 | 1279.0ms/step | 15.6 samples/s +Step 27240/44000 | Loss: 2.9537 | LR: 1.63e-04 | 1280.0ms/step | 15.6 samples/s +Step 27250/44000 | Loss: 2.9239 | LR: 1.63e-04 | 1282.7ms/step | 15.6 samples/s +Validation at step 27250: Loss = 2.9586, Perplexity = 19.27 +Step 27260/44000 | Loss: 2.9630 | LR: 1.63e-04 | 3426.0ms/step | 5.8 samples/s +Step 27270/44000 | Loss: 2.9750 | LR: 1.63e-04 | 1282.3ms/step | 15.6 samples/s +Step 27280/44000 | Loss: 2.9651 | LR: 1.63e-04 | 1282.4ms/step | 15.6 samples/s +Step 27290/44000 | Loss: 2.9348 | LR: 1.63e-04 | 1281.2ms/step | 15.6 samples/s +Step 27300/44000 | Loss: 2.9229 | LR: 1.62e-04 | 1280.9ms/step | 15.6 samples/s +Step 27310/44000 | Loss: 2.9547 | LR: 1.62e-04 | 1283.6ms/step | 15.6 samples/s +Step 27320/44000 | Loss: 2.9607 | LR: 1.62e-04 | 1284.2ms/step | 15.6 samples/s +Step 27330/44000 | Loss: 2.9627 | LR: 1.62e-04 | 1281.7ms/step | 15.6 samples/s +Step 27340/44000 | Loss: 2.9204 | LR: 1.62e-04 | 1283.6ms/step | 15.6 samples/s +Step 27350/44000 | Loss: 2.9174 | LR: 1.62e-04 | 1285.9ms/step | 15.6 samples/s +Step 27360/44000 | Loss: 2.9242 | LR: 1.61e-04 | 1286.4ms/step | 15.5 samples/s +Step 27370/44000 | Loss: 2.9326 | LR: 1.61e-04 | 1286.3ms/step | 15.5 samples/s +Step 27380/44000 | Loss: 2.9767 | LR: 1.61e-04 | 1285.2ms/step | 15.6 samples/s +Step 27390/44000 | Loss: 2.9343 | LR: 1.61e-04 | 1285.9ms/step | 15.6 samples/s +Step 27400/44000 | Loss: 2.9618 | LR: 1.61e-04 | 1284.8ms/step | 15.6 samples/s +Step 27410/44000 | Loss: 2.9439 | LR: 1.61e-04 | 1283.3ms/step | 15.6 samples/s +Step 27420/44000 | Loss: 2.9371 | LR: 1.60e-04 | 1282.2ms/step | 15.6 samples/s +Step 27430/44000 | Loss: 2.9602 | LR: 1.60e-04 | 1282.7ms/step | 15.6 samples/s +Step 27440/44000 | Loss: 2.9504 | LR: 1.60e-04 | 1282.7ms/step | 15.6 samples/s +Step 27450/44000 | Loss: 2.9182 | LR: 1.60e-04 | 1282.9ms/step | 15.6 samples/s +Step 27460/44000 | Loss: 2.9380 | LR: 1.60e-04 | 1285.3ms/step | 15.6 samples/s +Step 27470/44000 | Loss: 2.9683 | LR: 1.60e-04 | 1282.3ms/step | 15.6 samples/s +Step 27480/44000 | Loss: 2.9404 | LR: 1.59e-04 | 1283.3ms/step | 15.6 samples/s +Step 27490/44000 | Loss: 2.9599 | LR: 1.59e-04 | 1285.0ms/step | 15.6 samples/s +Step 27500/44000 | Loss: 2.9482 | LR: 1.59e-04 | 1285.8ms/step | 15.6 samples/s +Validation at step 27500: Loss = 2.9573, Perplexity = 19.25 +Step 27510/44000 | Loss: 2.9814 | LR: 1.59e-04 | 3512.3ms/step | 5.7 samples/s +Step 27520/44000 | Loss: 2.9404 | LR: 1.59e-04 | 1288.9ms/step | 15.5 samples/s +Step 27530/44000 | Loss: 2.9544 | LR: 1.59e-04 | 1282.1ms/step | 15.6 samples/s +Step 27540/44000 | Loss: 2.9129 | LR: 1.58e-04 | 1285.6ms/step | 15.6 samples/s +Step 27550/44000 | Loss: 2.9120 | LR: 1.58e-04 | 1288.3ms/step | 15.5 samples/s +Step 27560/44000 | Loss: 2.9275 | LR: 1.58e-04 | 1289.8ms/step | 15.5 samples/s +Step 27570/44000 | Loss: 2.9188 | LR: 1.58e-04 | 1283.3ms/step | 15.6 samples/s +Step 27580/44000 | Loss: 2.9700 | LR: 1.58e-04 | 1279.8ms/step | 15.6 samples/s +Step 27590/44000 | Loss: 2.9357 | LR: 1.58e-04 | 1280.0ms/step | 15.6 samples/s +Step 27600/44000 | Loss: 2.9480 | LR: 1.57e-04 | 1285.6ms/step | 15.6 samples/s +Step 27610/44000 | Loss: 2.9181 | LR: 1.57e-04 | 1286.0ms/step | 15.6 samples/s +Step 27620/44000 | Loss: 2.9502 | LR: 1.57e-04 | 1285.5ms/step | 15.6 samples/s +Step 27630/44000 | Loss: 2.9180 | LR: 1.57e-04 | 1284.5ms/step | 15.6 samples/s +Step 27640/44000 | Loss: 2.9778 | LR: 1.57e-04 | 1279.4ms/step | 15.6 samples/s +Step 27650/44000 | Loss: 2.9978 | LR: 1.57e-04 | 1284.4ms/step | 15.6 samples/s +Step 27660/44000 | Loss: 2.9479 | LR: 1.56e-04 | 1278.4ms/step | 15.6 samples/s +Step 27670/44000 | Loss: 2.9370 | LR: 1.56e-04 | 1278.7ms/step | 15.6 samples/s +Step 27680/44000 | Loss: 2.9530 | LR: 1.56e-04 | 1278.5ms/step | 15.6 samples/s +Step 27690/44000 | Loss: 2.9409 | LR: 1.56e-04 | 1277.8ms/step | 15.7 samples/s +Step 27700/44000 | Loss: 2.9342 | LR: 1.56e-04 | 1280.3ms/step | 15.6 samples/s +Step 27710/44000 | Loss: 2.9529 | LR: 1.56e-04 | 1280.3ms/step | 15.6 samples/s +Step 27720/44000 | Loss: 2.9363 | LR: 1.55e-04 | 1279.4ms/step | 15.6 samples/s +Step 27730/44000 | Loss: 2.9389 | LR: 1.55e-04 | 1282.0ms/step | 15.6 samples/s +Step 27740/44000 | Loss: 2.9369 | LR: 1.55e-04 | 1283.0ms/step | 15.6 samples/s +Step 27750/44000 | Loss: 2.9372 | LR: 1.55e-04 | 1282.8ms/step | 15.6 samples/s +Validation at step 27750: Loss = 2.9560, Perplexity = 19.22 +Step 27760/44000 | Loss: 2.9413 | LR: 1.55e-04 | 3428.8ms/step | 5.8 samples/s +Step 27770/44000 | Loss: 2.9377 | LR: 1.55e-04 | 1279.2ms/step | 15.6 samples/s +Step 27780/44000 | Loss: 2.9367 | LR: 1.54e-04 | 1281.2ms/step | 15.6 samples/s +Step 27790/44000 | Loss: 2.9002 | LR: 1.54e-04 | 1284.0ms/step | 15.6 samples/s +Step 27800/44000 | Loss: 2.9323 | LR: 1.54e-04 | 1283.6ms/step | 15.6 samples/s +Step 27810/44000 | Loss: 2.9045 | LR: 1.54e-04 | 1281.3ms/step | 15.6 samples/s +Step 27820/44000 | Loss: 2.9278 | LR: 1.54e-04 | 1277.9ms/step | 15.7 samples/s +Step 27830/44000 | Loss: 2.9450 | LR: 1.54e-04 | 1276.7ms/step | 15.7 samples/s +Step 27840/44000 | Loss: 2.9587 | LR: 1.53e-04 | 1278.1ms/step | 15.6 samples/s +Step 27850/44000 | Loss: 2.9270 | LR: 1.53e-04 | 1279.0ms/step | 15.6 samples/s +Step 27860/44000 | Loss: 2.9339 | LR: 1.53e-04 | 1284.6ms/step | 15.6 samples/s +Step 27870/44000 | Loss: 2.9651 | LR: 1.53e-04 | 1285.2ms/step | 15.6 samples/s +Step 27880/44000 | Loss: 2.9380 | LR: 1.53e-04 | 1281.2ms/step | 15.6 samples/s +Step 27890/44000 | Loss: 2.9164 | LR: 1.53e-04 | 1279.5ms/step | 15.6 samples/s +Step 27900/44000 | Loss: 2.9223 | LR: 1.52e-04 | 1279.8ms/step | 15.6 samples/s +Step 27910/44000 | Loss: 2.9355 | LR: 1.52e-04 | 1280.1ms/step | 15.6 samples/s +Step 27920/44000 | Loss: 2.9727 | LR: 1.52e-04 | 1280.5ms/step | 15.6 samples/s +Step 27930/44000 | Loss: 2.9497 | LR: 1.52e-04 | 1285.8ms/step | 15.6 samples/s +Step 27940/44000 | Loss: 2.9421 | LR: 1.52e-04 | 1285.4ms/step | 15.6 samples/s +Step 27950/44000 | Loss: 2.9621 | LR: 1.52e-04 | 1284.2ms/step | 15.6 samples/s +Step 27960/44000 | Loss: 2.9434 | LR: 1.51e-04 | 1288.9ms/step | 15.5 samples/s +Step 27970/44000 | Loss: 2.9365 | LR: 1.51e-04 | 1281.9ms/step | 15.6 samples/s +Step 27980/44000 | Loss: 2.9450 | LR: 1.51e-04 | 1275.2ms/step | 15.7 samples/s +Step 27990/44000 | Loss: 2.9350 | LR: 1.51e-04 | 1278.6ms/step | 15.6 samples/s +Step 28000/44000 | Loss: 2.9392 | LR: 1.51e-04 | 1283.1ms/step | 15.6 samples/s +Validation at step 28000: Loss = 2.9548, Perplexity = 19.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-28000 +Step 28010/44000 | Loss: 2.9386 | LR: 1.51e-04 | 3744.5ms/step | 5.3 samples/s +Step 28020/44000 | Loss: 2.9760 | LR: 1.50e-04 | 1283.9ms/step | 15.6 samples/s +Step 28030/44000 | Loss: 2.9329 | LR: 1.50e-04 | 1284.3ms/step | 15.6 samples/s +Step 28040/44000 | Loss: 2.9530 | LR: 1.50e-04 | 1283.9ms/step | 15.6 samples/s +Step 28050/44000 | Loss: 2.9575 | LR: 1.50e-04 | 1284.5ms/step | 15.6 samples/s +Step 28060/44000 | Loss: 2.9643 | LR: 1.50e-04 | 1284.6ms/step | 15.6 samples/s +Step 28070/44000 | Loss: 2.9358 | LR: 1.50e-04 | 1282.8ms/step | 15.6 samples/s +Step 28080/44000 | Loss: 2.9194 | LR: 1.49e-04 | 1282.7ms/step | 15.6 samples/s +Step 28090/44000 | Loss: 2.9450 | LR: 1.49e-04 | 1283.0ms/step | 15.6 samples/s +Step 28100/44000 | Loss: 2.9394 | LR: 1.49e-04 | 1284.2ms/step | 15.6 samples/s +Step 28110/44000 | Loss: 2.9175 | LR: 1.49e-04 | 1284.2ms/step | 15.6 samples/s +Step 28120/44000 | Loss: 2.9483 | LR: 1.49e-04 | 1283.9ms/step | 15.6 samples/s +Step 28130/44000 | Loss: 2.9473 | LR: 1.49e-04 | 1284.2ms/step | 15.6 samples/s +Step 28140/44000 | Loss: 2.9415 | LR: 1.48e-04 | 1285.2ms/step | 15.6 samples/s +Step 28150/44000 | Loss: 2.9349 | LR: 1.48e-04 | 1286.0ms/step | 15.6 samples/s +Step 28160/44000 | Loss: 2.9473 | LR: 1.48e-04 | 1284.3ms/step | 15.6 samples/s +Step 28170/44000 | Loss: 2.9231 | LR: 1.48e-04 | 1284.9ms/step | 15.6 samples/s +Step 28180/44000 | Loss: 2.9612 | LR: 1.48e-04 | 1284.2ms/step | 15.6 samples/s +Step 28190/44000 | Loss: 2.9525 | LR: 1.48e-04 | 1283.3ms/step | 15.6 samples/s +Step 28200/44000 | Loss: 2.9621 | LR: 1.47e-04 | 1285.9ms/step | 15.6 samples/s +Step 28210/44000 | Loss: 2.9088 | LR: 1.47e-04 | 1284.0ms/step | 15.6 samples/s +Step 28220/44000 | Loss: 2.9266 | LR: 1.47e-04 | 1285.8ms/step | 15.6 samples/s +Step 28230/44000 | Loss: 2.9472 | LR: 1.47e-04 | 1285.6ms/step | 15.6 samples/s +Step 28240/44000 | Loss: 2.9088 | LR: 1.47e-04 | 1285.5ms/step | 15.6 samples/s +Step 28250/44000 | Loss: 2.9218 | LR: 1.47e-04 | 1284.6ms/step | 15.6 samples/s +Validation at step 28250: Loss = 2.9540, Perplexity = 19.18 +Step 28260/44000 | Loss: 2.9422 | LR: 1.46e-04 | 3430.7ms/step | 5.8 samples/s +Step 28270/44000 | Loss: 2.9273 | LR: 1.46e-04 | 1282.7ms/step | 15.6 samples/s +Step 28280/44000 | Loss: 2.9685 | LR: 1.46e-04 | 1284.6ms/step | 15.6 samples/s +Step 28290/44000 | Loss: 2.8968 | LR: 1.46e-04 | 1282.9ms/step | 15.6 samples/s +Step 28300/44000 | Loss: 2.9378 | LR: 1.46e-04 | 1283.4ms/step | 15.6 samples/s +Step 28310/44000 | Loss: 2.9144 | LR: 1.46e-04 | 1284.4ms/step | 15.6 samples/s +Step 28320/44000 | Loss: 2.9765 | LR: 1.45e-04 | 1283.8ms/step | 15.6 samples/s +Step 28330/44000 | Loss: 2.9381 | LR: 1.45e-04 | 1284.3ms/step | 15.6 samples/s +Step 28340/44000 | Loss: 2.9505 | LR: 1.45e-04 | 1283.7ms/step | 15.6 samples/s +Step 28350/44000 | Loss: 2.9275 | LR: 1.45e-04 | 1286.0ms/step | 15.6 samples/s +Step 28360/44000 | Loss: 2.9372 | LR: 1.45e-04 | 1286.0ms/step | 15.6 samples/s +Step 28370/44000 | Loss: 2.9136 | LR: 1.45e-04 | 1285.6ms/step | 15.6 samples/s +Step 28380/44000 | Loss: 2.9221 | LR: 1.44e-04 | 1281.4ms/step | 15.6 samples/s +Step 28390/44000 | Loss: 2.9371 | LR: 1.44e-04 | 1280.7ms/step | 15.6 samples/s +Step 28400/44000 | Loss: 2.9362 | LR: 1.44e-04 | 1281.3ms/step | 15.6 samples/s +Step 28410/44000 | Loss: 2.9615 | LR: 1.44e-04 | 1279.7ms/step | 15.6 samples/s +Step 28420/44000 | Loss: 2.9831 | LR: 1.44e-04 | 1285.3ms/step | 15.6 samples/s +Step 28430/44000 | Loss: 2.9081 | LR: 1.44e-04 | 1284.5ms/step | 15.6 samples/s +Step 28440/44000 | Loss: 2.9479 | LR: 1.43e-04 | 1283.2ms/step | 15.6 samples/s +Step 28450/44000 | Loss: 2.9438 | LR: 1.43e-04 | 1279.6ms/step | 15.6 samples/s +Step 28460/44000 | Loss: 2.9508 | LR: 1.43e-04 | 1280.1ms/step | 15.6 samples/s +Step 28470/44000 | Loss: 2.9500 | LR: 1.43e-04 | 1274.3ms/step | 15.7 samples/s +Step 28480/44000 | Loss: 2.9301 | LR: 1.43e-04 | 1275.4ms/step | 15.7 samples/s +Step 28490/44000 | Loss: 2.9381 | LR: 1.43e-04 | 1282.2ms/step | 15.6 samples/s +Step 28500/44000 | Loss: 2.9388 | LR: 1.42e-04 | 1284.5ms/step | 15.6 samples/s +Validation at step 28500: Loss = 2.9526, Perplexity = 19.16 +Step 28510/44000 | Loss: 2.9616 | LR: 1.42e-04 | 3433.1ms/step | 5.8 samples/s +Step 28520/44000 | Loss: 2.9425 | LR: 1.42e-04 | 1284.4ms/step | 15.6 samples/s +Step 28530/44000 | Loss: 2.9309 | LR: 1.42e-04 | 1283.6ms/step | 15.6 samples/s +Step 28540/44000 | Loss: 2.9579 | LR: 1.42e-04 | 1284.2ms/step | 15.6 samples/s +Step 28550/44000 | Loss: 2.9751 | LR: 1.42e-04 | 1282.8ms/step | 15.6 samples/s +Step 28560/44000 | Loss: 2.9250 | LR: 1.41e-04 | 1284.5ms/step | 15.6 samples/s +Step 28570/44000 | Loss: 2.9175 | LR: 1.41e-04 | 1284.4ms/step | 15.6 samples/s +Step 28580/44000 | Loss: 2.9489 | LR: 1.41e-04 | 1282.6ms/step | 15.6 samples/s +Step 28590/44000 | Loss: 2.9657 | LR: 1.41e-04 | 1284.0ms/step | 15.6 samples/s +Step 28600/44000 | Loss: 2.9659 | LR: 1.41e-04 | 1284.2ms/step | 15.6 samples/s +Step 28610/44000 | Loss: 2.9307 | LR: 1.41e-04 | 1282.3ms/step | 15.6 samples/s +Step 28620/44000 | Loss: 2.9149 | LR: 1.40e-04 | 1281.9ms/step | 15.6 samples/s +Step 28630/44000 | Loss: 2.9255 | LR: 1.40e-04 | 1278.3ms/step | 15.6 samples/s +Step 28640/44000 | Loss: 2.9071 | LR: 1.40e-04 | 1277.5ms/step | 15.7 samples/s +Step 28650/44000 | Loss: 2.9310 | LR: 1.40e-04 | 1278.8ms/step | 15.6 samples/s +Step 28660/44000 | Loss: 2.9284 | LR: 1.40e-04 | 1279.2ms/step | 15.6 samples/s +Step 28670/44000 | Loss: 2.8973 | LR: 1.40e-04 | 1280.5ms/step | 15.6 samples/s +Step 28680/44000 | Loss: 2.9469 | LR: 1.39e-04 | 1282.8ms/step | 15.6 samples/s +Step 28690/44000 | Loss: 2.9429 | LR: 1.39e-04 | 1283.2ms/step | 15.6 samples/s +Step 28700/44000 | Loss: 2.9453 | LR: 1.39e-04 | 1285.7ms/step | 15.6 samples/s +Step 28710/44000 | Loss: 2.9510 | LR: 1.39e-04 | 1285.7ms/step | 15.6 samples/s +Step 28720/44000 | Loss: 2.9265 | LR: 1.39e-04 | 1287.4ms/step | 15.5 samples/s +Step 28730/44000 | Loss: 2.9290 | LR: 1.39e-04 | 1288.1ms/step | 15.5 samples/s +Step 28740/44000 | Loss: 2.9523 | LR: 1.38e-04 | 1287.7ms/step | 15.5 samples/s +Step 28750/44000 | Loss: 2.9149 | LR: 1.38e-04 | 1282.2ms/step | 15.6 samples/s +Validation at step 28750: Loss = 2.9515, Perplexity = 19.13 +Step 28760/44000 | Loss: 2.9424 | LR: 1.38e-04 | 3422.5ms/step | 5.8 samples/s +Step 28770/44000 | Loss: 2.9095 | LR: 1.38e-04 | 1280.1ms/step | 15.6 samples/s +Step 28780/44000 | Loss: 2.9446 | LR: 1.38e-04 | 1283.6ms/step | 15.6 samples/s +Step 28790/44000 | Loss: 2.9533 | LR: 1.38e-04 | 1282.1ms/step | 15.6 samples/s +Step 28800/44000 | Loss: 2.9567 | LR: 1.38e-04 | 1282.4ms/step | 15.6 samples/s +Step 28810/44000 | Loss: 2.9421 | LR: 1.37e-04 | 1283.3ms/step | 15.6 samples/s +Step 28820/44000 | Loss: 2.9164 | LR: 1.37e-04 | 1283.0ms/step | 15.6 samples/s +Step 28830/44000 | Loss: 2.9470 | LR: 1.37e-04 | 1287.7ms/step | 15.5 samples/s +Step 28840/44000 | Loss: 2.9421 | LR: 1.37e-04 | 1289.4ms/step | 15.5 samples/s +Step 28850/44000 | Loss: 2.9202 | LR: 1.37e-04 | 1284.2ms/step | 15.6 samples/s +Step 28860/44000 | Loss: 2.9516 | LR: 1.37e-04 | 1283.9ms/step | 15.6 samples/s +Step 28870/44000 | Loss: 2.9564 | LR: 1.36e-04 | 1284.4ms/step | 15.6 samples/s +Step 28880/44000 | Loss: 2.9297 | LR: 1.36e-04 | 1291.4ms/step | 15.5 samples/s +Step 28890/44000 | Loss: 2.9424 | LR: 1.36e-04 | 1292.0ms/step | 15.5 samples/s +Step 28900/44000 | Loss: 2.9357 | LR: 1.36e-04 | 1286.2ms/step | 15.5 samples/s +Step 28910/44000 | Loss: 2.9338 | LR: 1.36e-04 | 1289.7ms/step | 15.5 samples/s +Step 28920/44000 | Loss: 2.9467 | LR: 1.36e-04 | 1294.2ms/step | 15.5 samples/s +Step 28930/44000 | Loss: 2.9375 | LR: 1.35e-04 | 1287.6ms/step | 15.5 samples/s +Step 28940/44000 | Loss: 2.9322 | LR: 1.35e-04 | 1280.4ms/step | 15.6 samples/s +Step 28950/44000 | Loss: 2.9769 | LR: 1.35e-04 | 1279.4ms/step | 15.6 samples/s +Step 28960/44000 | Loss: 2.9361 | LR: 1.35e-04 | 1277.1ms/step | 15.7 samples/s +Step 28970/44000 | Loss: 2.9589 | LR: 1.35e-04 | 1276.1ms/step | 15.7 samples/s +Step 28980/44000 | Loss: 2.9432 | LR: 1.35e-04 | 1279.4ms/step | 15.6 samples/s +Step 28990/44000 | Loss: 2.9321 | LR: 1.34e-04 | 1281.9ms/step | 15.6 samples/s +Step 29000/44000 | Loss: 2.9316 | LR: 1.34e-04 | 1281.3ms/step | 15.6 samples/s +Validation at step 29000: Loss = 2.9506, Perplexity = 19.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-29000 +Step 29010/44000 | Loss: 2.9257 | LR: 1.34e-04 | 3718.0ms/step | 5.4 samples/s +Step 29020/44000 | Loss: 2.9379 | LR: 1.34e-04 | 1279.1ms/step | 15.6 samples/s +Step 29030/44000 | Loss: 2.9250 | LR: 1.34e-04 | 1280.1ms/step | 15.6 samples/s +Step 29040/44000 | Loss: 2.9115 | LR: 1.34e-04 | 1281.2ms/step | 15.6 samples/s +Step 29050/44000 | Loss: 2.9176 | LR: 1.33e-04 | 1285.4ms/step | 15.6 samples/s +Step 29060/44000 | Loss: 2.9470 | LR: 1.33e-04 | 1287.1ms/step | 15.5 samples/s +Step 29070/44000 | Loss: 2.9413 | LR: 1.33e-04 | 1285.0ms/step | 15.6 samples/s +Step 29080/44000 | Loss: 2.9116 | LR: 1.33e-04 | 1283.5ms/step | 15.6 samples/s +Step 29090/44000 | Loss: 2.9454 | LR: 1.33e-04 | 1285.2ms/step | 15.6 samples/s +Step 29100/44000 | Loss: 2.9551 | LR: 1.33e-04 | 1285.4ms/step | 15.6 samples/s +Step 29110/44000 | Loss: 2.9518 | LR: 1.33e-04 | 1287.6ms/step | 15.5 samples/s +Step 29120/44000 | Loss: 2.9191 | LR: 1.32e-04 | 1285.5ms/step | 15.6 samples/s +Step 29130/44000 | Loss: 2.9468 | LR: 1.32e-04 | 1286.6ms/step | 15.5 samples/s +Step 29140/44000 | Loss: 2.9397 | LR: 1.32e-04 | 1284.4ms/step | 15.6 samples/s +Step 29150/44000 | Loss: 2.9579 | LR: 1.32e-04 | 1285.4ms/step | 15.6 samples/s +Step 29160/44000 | Loss: 2.9334 | LR: 1.32e-04 | 1286.0ms/step | 15.6 samples/s +Step 29170/44000 | Loss: 2.9108 | LR: 1.32e-04 | 1284.0ms/step | 15.6 samples/s +Step 29180/44000 | Loss: 2.9122 | LR: 1.31e-04 | 1285.2ms/step | 15.6 samples/s +Step 29190/44000 | Loss: 2.9282 | LR: 1.31e-04 | 1284.6ms/step | 15.6 samples/s +Step 29200/44000 | Loss: 2.9339 | LR: 1.31e-04 | 1284.1ms/step | 15.6 samples/s +Step 29210/44000 | Loss: 2.8898 | LR: 1.31e-04 | 1285.4ms/step | 15.6 samples/s +Step 29220/44000 | Loss: 2.9309 | LR: 1.31e-04 | 1284.4ms/step | 15.6 samples/s +Step 29230/44000 | Loss: 2.9209 | LR: 1.31e-04 | 1280.6ms/step | 15.6 samples/s +Step 29240/44000 | Loss: 2.9479 | LR: 1.30e-04 | 1280.8ms/step | 15.6 samples/s +Step 29250/44000 | Loss: 2.9290 | LR: 1.30e-04 | 1285.1ms/step | 15.6 samples/s +Validation at step 29250: Loss = 2.9495, Perplexity = 19.10 +Step 29260/44000 | Loss: 2.9171 | LR: 1.30e-04 | 3434.6ms/step | 5.8 samples/s +Step 29270/44000 | Loss: 2.9345 | LR: 1.30e-04 | 1283.7ms/step | 15.6 samples/s +Step 29280/44000 | Loss: 2.9249 | LR: 1.30e-04 | 1284.2ms/step | 15.6 samples/s +Step 29290/44000 | Loss: 2.9403 | LR: 1.30e-04 | 1290.9ms/step | 15.5 samples/s +Step 29300/44000 | Loss: 2.9462 | LR: 1.29e-04 | 1288.9ms/step | 15.5 samples/s +Step 29310/44000 | Loss: 2.9506 | LR: 1.29e-04 | 1290.7ms/step | 15.5 samples/s +Step 29320/44000 | Loss: 2.9453 | LR: 1.29e-04 | 1292.7ms/step | 15.5 samples/s +Step 29330/44000 | Loss: 2.9507 | LR: 1.29e-04 | 1288.8ms/step | 15.5 samples/s +Step 29340/44000 | Loss: 2.9071 | LR: 1.29e-04 | 1285.7ms/step | 15.6 samples/s +Step 29350/44000 | Loss: 2.9509 | LR: 1.29e-04 | 1289.6ms/step | 15.5 samples/s +Step 29360/44000 | Loss: 2.9549 | LR: 1.29e-04 | 1288.1ms/step | 15.5 samples/s +Step 29370/44000 | Loss: 2.9257 | LR: 1.28e-04 | 1285.4ms/step | 15.6 samples/s +Step 29380/44000 | Loss: 2.9340 | LR: 1.28e-04 | 1285.0ms/step | 15.6 samples/s +Step 29390/44000 | Loss: 2.9528 | LR: 1.28e-04 | 1285.6ms/step | 15.6 samples/s +Step 29400/44000 | Loss: 2.9361 | LR: 1.28e-04 | 1283.6ms/step | 15.6 samples/s +Step 29410/44000 | Loss: 2.9258 | LR: 1.28e-04 | 1283.6ms/step | 15.6 samples/s +Step 29420/44000 | Loss: 2.9665 | LR: 1.28e-04 | 1283.2ms/step | 15.6 samples/s +Step 29430/44000 | Loss: 2.9398 | LR: 1.27e-04 | 1283.9ms/step | 15.6 samples/s +Step 29440/44000 | Loss: 2.9403 | LR: 1.27e-04 | 1283.9ms/step | 15.6 samples/s +Step 29450/44000 | Loss: 2.9594 | LR: 1.27e-04 | 1283.7ms/step | 15.6 samples/s +Step 29460/44000 | Loss: 2.9453 | LR: 1.27e-04 | 1284.6ms/step | 15.6 samples/s +Step 29470/44000 | Loss: 2.9410 | LR: 1.27e-04 | 1284.4ms/step | 15.6 samples/s +Step 29480/44000 | Loss: 2.9390 | LR: 1.27e-04 | 1284.7ms/step | 15.6 samples/s +Step 29490/44000 | Loss: 2.9280 | LR: 1.26e-04 | 1283.5ms/step | 15.6 samples/s +Step 29500/44000 | Loss: 2.9237 | LR: 1.26e-04 | 1283.1ms/step | 15.6 samples/s +Validation at step 29500: Loss = 2.9487, Perplexity = 19.08 +Step 29510/44000 | Loss: 2.9283 | LR: 1.26e-04 | 3429.4ms/step | 5.8 samples/s +Step 29520/44000 | Loss: 2.9423 | LR: 1.26e-04 | 1285.7ms/step | 15.6 samples/s +Step 29530/44000 | Loss: 2.9214 | LR: 1.26e-04 | 1284.7ms/step | 15.6 samples/s +Step 29540/44000 | Loss: 2.9180 | LR: 1.26e-04 | 1282.3ms/step | 15.6 samples/s +Step 29550/44000 | Loss: 2.9243 | LR: 1.26e-04 | 1284.0ms/step | 15.6 samples/s +Step 29560/44000 | Loss: 2.9239 | LR: 1.25e-04 | 1284.4ms/step | 15.6 samples/s +Step 29570/44000 | Loss: 2.9331 | LR: 1.25e-04 | 1282.5ms/step | 15.6 samples/s +Step 29580/44000 | Loss: 2.9224 | LR: 1.25e-04 | 1280.3ms/step | 15.6 samples/s +Step 29590/44000 | Loss: 2.9533 | LR: 1.25e-04 | 1282.8ms/step | 15.6 samples/s +Step 29600/44000 | Loss: 2.9430 | LR: 1.25e-04 | 1287.2ms/step | 15.5 samples/s +Step 29610/44000 | Loss: 2.9337 | LR: 1.25e-04 | 1283.4ms/step | 15.6 samples/s +Step 29620/44000 | Loss: 2.9170 | LR: 1.24e-04 | 1285.5ms/step | 15.6 samples/s +Step 29630/44000 | Loss: 2.9587 | LR: 1.24e-04 | 1284.3ms/step | 15.6 samples/s +Step 29640/44000 | Loss: 2.9288 | LR: 1.24e-04 | 1284.0ms/step | 15.6 samples/s +Step 29650/44000 | Loss: 2.9203 | LR: 1.24e-04 | 1285.0ms/step | 15.6 samples/s +Step 29660/44000 | Loss: 2.9770 | LR: 1.24e-04 | 1282.0ms/step | 15.6 samples/s +Step 29670/44000 | Loss: 2.9207 | LR: 1.24e-04 | 1284.1ms/step | 15.6 samples/s +Step 29680/44000 | Loss: 2.9315 | LR: 1.23e-04 | 1284.5ms/step | 15.6 samples/s +Step 29690/44000 | Loss: 2.9398 | LR: 1.23e-04 | 1284.0ms/step | 15.6 samples/s +Step 29700/44000 | Loss: 2.9427 | LR: 1.23e-04 | 1285.2ms/step | 15.6 samples/s +Step 29710/44000 | Loss: 2.9534 | LR: 1.23e-04 | 1281.5ms/step | 15.6 samples/s +Step 29720/44000 | Loss: 2.8938 | LR: 1.23e-04 | 1281.5ms/step | 15.6 samples/s +Step 29730/44000 | Loss: 2.9427 | LR: 1.23e-04 | 1282.7ms/step | 15.6 samples/s +Step 29740/44000 | Loss: 2.9095 | LR: 1.23e-04 | 1283.7ms/step | 15.6 samples/s +Step 29750/44000 | Loss: 2.9242 | LR: 1.22e-04 | 1284.3ms/step | 15.6 samples/s +Validation at step 29750: Loss = 2.9476, Perplexity = 19.06 +Step 29760/44000 | Loss: 2.9547 | LR: 1.22e-04 | 3427.3ms/step | 5.8 samples/s +Step 29770/44000 | Loss: 2.9256 | LR: 1.22e-04 | 1286.8ms/step | 15.5 samples/s +Step 29780/44000 | Loss: 2.9345 | LR: 1.22e-04 | 1287.2ms/step | 15.5 samples/s +Step 29790/44000 | Loss: 2.9040 | LR: 1.22e-04 | 1284.8ms/step | 15.6 samples/s +Step 29800/44000 | Loss: 2.9112 | LR: 1.22e-04 | 1286.9ms/step | 15.5 samples/s +Step 29810/44000 | Loss: 2.9184 | LR: 1.21e-04 | 1286.3ms/step | 15.5 samples/s +Step 29820/44000 | Loss: 2.9643 | LR: 1.21e-04 | 1286.3ms/step | 15.5 samples/s +Step 29830/44000 | Loss: 2.9506 | LR: 1.21e-04 | 1288.2ms/step | 15.5 samples/s +Step 29840/44000 | Loss: 2.8990 | LR: 1.21e-04 | 1280.8ms/step | 15.6 samples/s +Step 29850/44000 | Loss: 2.9234 | LR: 1.21e-04 | 1281.0ms/step | 15.6 samples/s +Step 29860/44000 | Loss: 2.9391 | LR: 1.21e-04 | 1281.9ms/step | 15.6 samples/s +Step 29870/44000 | Loss: 2.9084 | LR: 1.21e-04 | 1280.2ms/step | 15.6 samples/s +Step 29880/44000 | Loss: 2.9254 | LR: 1.20e-04 | 1279.6ms/step | 15.6 samples/s +Step 29890/44000 | Loss: 2.9319 | LR: 1.20e-04 | 1280.5ms/step | 15.6 samples/s +Step 29900/44000 | Loss: 2.9720 | LR: 1.20e-04 | 1284.4ms/step | 15.6 samples/s +Step 29910/44000 | Loss: 2.9280 | LR: 1.20e-04 | 1286.1ms/step | 15.6 samples/s +Step 29920/44000 | Loss: 2.9179 | LR: 1.20e-04 | 1287.2ms/step | 15.5 samples/s +Step 29930/44000 | Loss: 2.9235 | LR: 1.20e-04 | 1285.7ms/step | 15.6 samples/s +Step 29940/44000 | Loss: 2.9212 | LR: 1.19e-04 | 1284.0ms/step | 15.6 samples/s +Step 29950/44000 | Loss: 2.9303 | LR: 1.19e-04 | 1283.9ms/step | 15.6 samples/s +Step 29960/44000 | Loss: 2.9116 | LR: 1.19e-04 | 1283.4ms/step | 15.6 samples/s +Step 29970/44000 | Loss: 2.9248 | LR: 1.19e-04 | 1283.6ms/step | 15.6 samples/s +Step 29980/44000 | Loss: 2.9219 | LR: 1.19e-04 | 1282.6ms/step | 15.6 samples/s +Step 29990/44000 | Loss: 2.9366 | LR: 1.19e-04 | 1284.4ms/step | 15.6 samples/s +Step 30000/44000 | Loss: 2.9138 | LR: 1.19e-04 | 1286.3ms/step | 15.5 samples/s +Validation at step 30000: Loss = 2.9467, Perplexity = 19.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-30000 +Step 30010/44000 | Loss: 2.9434 | LR: 1.18e-04 | 3732.6ms/step | 5.4 samples/s +Step 30020/44000 | Loss: 2.9546 | LR: 1.18e-04 | 1284.6ms/step | 15.6 samples/s +Step 30030/44000 | Loss: 2.9221 | LR: 1.18e-04 | 1286.8ms/step | 15.5 samples/s +Step 30040/44000 | Loss: 2.9399 | LR: 1.18e-04 | 1283.5ms/step | 15.6 samples/s +Step 30050/44000 | Loss: 2.9139 | LR: 1.18e-04 | 1283.9ms/step | 15.6 samples/s +Step 30060/44000 | Loss: 2.9393 | LR: 1.18e-04 | 1284.1ms/step | 15.6 samples/s +Step 30070/44000 | Loss: 2.8934 | LR: 1.17e-04 | 1284.6ms/step | 15.6 samples/s +Step 30080/44000 | Loss: 2.9645 | LR: 1.17e-04 | 1284.3ms/step | 15.6 samples/s +Step 30090/44000 | Loss: 2.9423 | LR: 1.17e-04 | 1284.2ms/step | 15.6 samples/s +Step 30100/44000 | Loss: 2.9419 | LR: 1.17e-04 | 1284.1ms/step | 15.6 samples/s +Step 30110/44000 | Loss: 2.9284 | LR: 1.17e-04 | 1284.8ms/step | 15.6 samples/s +Step 30120/44000 | Loss: 2.9169 | LR: 1.17e-04 | 1286.2ms/step | 15.5 samples/s +Step 30130/44000 | Loss: 2.9459 | LR: 1.17e-04 | 1284.5ms/step | 15.6 samples/s +Step 30140/44000 | Loss: 2.9361 | LR: 1.16e-04 | 1285.1ms/step | 15.6 samples/s +Step 30150/44000 | Loss: 2.9185 | LR: 1.16e-04 | 1286.6ms/step | 15.5 samples/s +Step 30160/44000 | Loss: 2.9041 | LR: 1.16e-04 | 1284.9ms/step | 15.6 samples/s +Step 30170/44000 | Loss: 2.9072 | LR: 1.16e-04 | 1283.4ms/step | 15.6 samples/s +Step 30180/44000 | Loss: 2.9433 | LR: 1.16e-04 | 1285.2ms/step | 15.6 samples/s +Step 30190/44000 | Loss: 2.9197 | LR: 1.16e-04 | 1292.3ms/step | 15.5 samples/s +Step 30200/44000 | Loss: 2.9275 | LR: 1.15e-04 | 1290.1ms/step | 15.5 samples/s +Step 30210/44000 | Loss: 2.9286 | LR: 1.15e-04 | 1288.9ms/step | 15.5 samples/s +Step 30220/44000 | Loss: 2.9041 | LR: 1.15e-04 | 1288.7ms/step | 15.5 samples/s +Step 30230/44000 | Loss: 2.9491 | LR: 1.15e-04 | 1287.3ms/step | 15.5 samples/s +Step 30240/44000 | Loss: 2.9322 | LR: 1.15e-04 | 1286.4ms/step | 15.5 samples/s +Step 30250/44000 | Loss: 2.9194 | LR: 1.15e-04 | 1284.5ms/step | 15.6 samples/s +Validation at step 30250: Loss = 2.9457, Perplexity = 19.02 +Step 30260/44000 | Loss: 2.9234 | LR: 1.15e-04 | 3439.1ms/step | 5.8 samples/s +Step 30270/44000 | Loss: 2.9114 | LR: 1.14e-04 | 1286.4ms/step | 15.5 samples/s +Step 30280/44000 | Loss: 2.9559 | LR: 1.14e-04 | 1286.4ms/step | 15.5 samples/s +Step 30290/44000 | Loss: 2.9134 | LR: 1.14e-04 | 1284.9ms/step | 15.6 samples/s +Step 30300/44000 | Loss: 2.8967 | LR: 1.14e-04 | 1284.3ms/step | 15.6 samples/s +Step 30310/44000 | Loss: 2.9072 | LR: 1.14e-04 | 1284.4ms/step | 15.6 samples/s +Step 30320/44000 | Loss: 2.9056 | LR: 1.14e-04 | 1283.8ms/step | 15.6 samples/s +Step 30330/44000 | Loss: 2.9236 | LR: 1.13e-04 | 1283.6ms/step | 15.6 samples/s +Step 30340/44000 | Loss: 2.9301 | LR: 1.13e-04 | 1283.4ms/step | 15.6 samples/s +Step 30350/44000 | Loss: 2.8946 | LR: 1.13e-04 | 1283.6ms/step | 15.6 samples/s +Step 30360/44000 | Loss: 2.9296 | LR: 1.13e-04 | 1283.6ms/step | 15.6 samples/s +Step 30370/44000 | Loss: 2.9421 | LR: 1.13e-04 | 1285.4ms/step | 15.6 samples/s +Step 30380/44000 | Loss: 2.9514 | LR: 1.13e-04 | 1286.2ms/step | 15.5 samples/s +Step 30390/44000 | Loss: 2.9230 | LR: 1.13e-04 | 1283.6ms/step | 15.6 samples/s +Step 30400/44000 | Loss: 2.9251 | LR: 1.12e-04 | 1283.3ms/step | 15.6 samples/s +Step 30410/44000 | Loss: 2.9335 | LR: 1.12e-04 | 1284.2ms/step | 15.6 samples/s +Step 30420/44000 | Loss: 2.9283 | LR: 1.12e-04 | 1285.9ms/step | 15.6 samples/s +Step 30430/44000 | Loss: 2.9297 | LR: 1.12e-04 | 1282.2ms/step | 15.6 samples/s +Step 30440/44000 | Loss: 2.9311 | LR: 1.12e-04 | 1281.8ms/step | 15.6 samples/s +Step 30450/44000 | Loss: 2.9011 | LR: 1.12e-04 | 1283.2ms/step | 15.6 samples/s +Step 30460/44000 | Loss: 2.9320 | LR: 1.11e-04 | 1283.4ms/step | 15.6 samples/s +Step 30470/44000 | Loss: 2.9433 | LR: 1.11e-04 | 1281.2ms/step | 15.6 samples/s +Step 30480/44000 | Loss: 2.9316 | LR: 1.11e-04 | 1284.3ms/step | 15.6 samples/s +Step 30490/44000 | Loss: 2.9248 | LR: 1.11e-04 | 1282.2ms/step | 15.6 samples/s +Step 30500/44000 | Loss: 2.9208 | LR: 1.11e-04 | 1279.7ms/step | 15.6 samples/s +Validation at step 30500: Loss = 2.9450, Perplexity = 19.01 +Step 30510/44000 | Loss: 2.9194 | LR: 1.11e-04 | 3431.5ms/step | 5.8 samples/s +Step 30520/44000 | Loss: 2.9256 | LR: 1.11e-04 | 1283.7ms/step | 15.6 samples/s +Step 30530/44000 | Loss: 2.9655 | LR: 1.10e-04 | 1284.2ms/step | 15.6 samples/s +Step 30540/44000 | Loss: 2.9623 | LR: 1.10e-04 | 1283.1ms/step | 15.6 samples/s +Step 30550/44000 | Loss: 2.9339 | LR: 1.10e-04 | 1284.3ms/step | 15.6 samples/s +Step 30560/44000 | Loss: 2.9327 | LR: 1.10e-04 | 1283.3ms/step | 15.6 samples/s +Step 30570/44000 | Loss: 2.9469 | LR: 1.10e-04 | 1280.4ms/step | 15.6 samples/s +Step 30580/44000 | Loss: 2.9416 | LR: 1.10e-04 | 1287.1ms/step | 15.5 samples/s +Step 30590/44000 | Loss: 2.9308 | LR: 1.10e-04 | 1284.3ms/step | 15.6 samples/s +Step 30600/44000 | Loss: 2.9260 | LR: 1.09e-04 | 1285.6ms/step | 15.6 samples/s +Step 30610/44000 | Loss: 2.9281 | LR: 1.09e-04 | 1284.7ms/step | 15.6 samples/s +Step 30620/44000 | Loss: 2.9398 | LR: 1.09e-04 | 1283.9ms/step | 15.6 samples/s +Step 30630/44000 | Loss: 2.9123 | LR: 1.09e-04 | 1288.1ms/step | 15.5 samples/s +Step 30640/44000 | Loss: 2.9403 | LR: 1.09e-04 | 1286.6ms/step | 15.5 samples/s +Step 30650/44000 | Loss: 2.9499 | LR: 1.09e-04 | 1286.1ms/step | 15.6 samples/s +Step 30660/44000 | Loss: 2.9677 | LR: 1.08e-04 | 1283.8ms/step | 15.6 samples/s +Step 30670/44000 | Loss: 2.9534 | LR: 1.08e-04 | 1285.9ms/step | 15.6 samples/s +Step 30680/44000 | Loss: 2.9366 | LR: 1.08e-04 | 1290.5ms/step | 15.5 samples/s +Step 30690/44000 | Loss: 2.9215 | LR: 1.08e-04 | 1294.2ms/step | 15.5 samples/s +Step 30700/44000 | Loss: 2.9437 | LR: 1.08e-04 | 1300.8ms/step | 15.4 samples/s +Step 30710/44000 | Loss: 2.9245 | LR: 1.08e-04 | 1294.5ms/step | 15.4 samples/s +Step 30720/44000 | Loss: 2.9413 | LR: 1.08e-04 | 1287.1ms/step | 15.5 samples/s +Step 30730/44000 | Loss: 2.9244 | LR: 1.07e-04 | 1283.1ms/step | 15.6 samples/s +Step 30740/44000 | Loss: 2.9428 | LR: 1.07e-04 | 1286.6ms/step | 15.5 samples/s +Step 30750/44000 | Loss: 2.9265 | LR: 1.07e-04 | 1284.5ms/step | 15.6 samples/s +Validation at step 30750: Loss = 2.9442, Perplexity = 19.00 +Step 30760/44000 | Loss: 2.9293 | LR: 1.07e-04 | 3429.5ms/step | 5.8 samples/s +Step 30770/44000 | Loss: 2.9724 | LR: 1.07e-04 | 1285.2ms/step | 15.6 samples/s +Step 30780/44000 | Loss: 2.9231 | LR: 1.07e-04 | 1283.8ms/step | 15.6 samples/s +Step 30790/44000 | Loss: 2.9296 | LR: 1.07e-04 | 1286.1ms/step | 15.6 samples/s +Step 30800/44000 | Loss: 2.9373 | LR: 1.06e-04 | 1285.6ms/step | 15.6 samples/s +Step 30810/44000 | Loss: 2.9045 | LR: 1.06e-04 | 1286.0ms/step | 15.6 samples/s +Step 30820/44000 | Loss: 2.9361 | LR: 1.06e-04 | 1285.0ms/step | 15.6 samples/s +Step 30830/44000 | Loss: 2.9743 | LR: 1.06e-04 | 1285.5ms/step | 15.6 samples/s +Step 30840/44000 | Loss: 2.9444 | LR: 1.06e-04 | 1287.2ms/step | 15.5 samples/s +Step 30850/44000 | Loss: 2.9172 | LR: 1.06e-04 | 1286.4ms/step | 15.5 samples/s +Step 30860/44000 | Loss: 2.9464 | LR: 1.05e-04 | 1285.5ms/step | 15.6 samples/s +Step 30870/44000 | Loss: 2.9059 | LR: 1.05e-04 | 1285.3ms/step | 15.6 samples/s +Step 30880/44000 | Loss: 2.9239 | LR: 1.05e-04 | 1287.2ms/step | 15.5 samples/s +Step 30890/44000 | Loss: 2.8779 | LR: 1.05e-04 | 1286.9ms/step | 15.5 samples/s +Step 30900/44000 | Loss: 2.9070 | LR: 1.05e-04 | 1285.3ms/step | 15.6 samples/s +Step 30910/44000 | Loss: 2.9188 | LR: 1.05e-04 | 1284.4ms/step | 15.6 samples/s +Step 30920/44000 | Loss: 2.9347 | LR: 1.05e-04 | 1287.5ms/step | 15.5 samples/s +Step 30930/44000 | Loss: 2.9205 | LR: 1.04e-04 | 1288.1ms/step | 15.5 samples/s +Step 30940/44000 | Loss: 2.9066 | LR: 1.04e-04 | 1290.0ms/step | 15.5 samples/s +Step 30950/44000 | Loss: 2.9300 | LR: 1.04e-04 | 1287.8ms/step | 15.5 samples/s +Step 30960/44000 | Loss: 2.9226 | LR: 1.04e-04 | 1282.1ms/step | 15.6 samples/s +Step 30970/44000 | Loss: 2.9631 | LR: 1.04e-04 | 1282.0ms/step | 15.6 samples/s +Step 30980/44000 | Loss: 2.9093 | LR: 1.04e-04 | 1283.7ms/step | 15.6 samples/s +Step 30990/44000 | Loss: 2.9144 | LR: 1.04e-04 | 1282.7ms/step | 15.6 samples/s +Step 31000/44000 | Loss: 2.9033 | LR: 1.03e-04 | 1284.1ms/step | 15.6 samples/s +Validation at step 31000: Loss = 2.9435, Perplexity = 18.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-31000 +Step 31010/44000 | Loss: 2.9128 | LR: 1.03e-04 | 3736.9ms/step | 5.4 samples/s +Step 31020/44000 | Loss: 2.9419 | LR: 1.03e-04 | 1283.1ms/step | 15.6 samples/s +Step 31030/44000 | Loss: 2.8950 | LR: 1.03e-04 | 1285.2ms/step | 15.6 samples/s +Step 31040/44000 | Loss: 2.9247 | LR: 1.03e-04 | 1287.5ms/step | 15.5 samples/s +Step 31050/44000 | Loss: 2.9363 | LR: 1.03e-04 | 1286.2ms/step | 15.5 samples/s +Step 31060/44000 | Loss: 2.9440 | LR: 1.03e-04 | 1278.1ms/step | 15.6 samples/s +Step 31070/44000 | Loss: 2.9326 | LR: 1.02e-04 | 1278.2ms/step | 15.6 samples/s +Step 31080/44000 | Loss: 2.9233 | LR: 1.02e-04 | 1278.3ms/step | 15.6 samples/s +Step 31090/44000 | Loss: 2.9202 | LR: 1.02e-04 | 1285.3ms/step | 15.6 samples/s +Step 31100/44000 | Loss: 2.9117 | LR: 1.02e-04 | 1279.7ms/step | 15.6 samples/s +Step 31110/44000 | Loss: 2.9149 | LR: 1.02e-04 | 1278.7ms/step | 15.6 samples/s +Step 31120/44000 | Loss: 2.9315 | LR: 1.02e-04 | 1280.2ms/step | 15.6 samples/s +Step 31130/44000 | Loss: 2.9620 | LR: 1.02e-04 | 1282.2ms/step | 15.6 samples/s +Step 31140/44000 | Loss: 2.9150 | LR: 1.01e-04 | 1281.5ms/step | 15.6 samples/s +Step 31150/44000 | Loss: 2.9605 | LR: 1.01e-04 | 1281.8ms/step | 15.6 samples/s +Step 31160/44000 | Loss: 2.9328 | LR: 1.01e-04 | 1283.0ms/step | 15.6 samples/s +Step 31170/44000 | Loss: 2.9245 | LR: 1.01e-04 | 1282.1ms/step | 15.6 samples/s +Step 31180/44000 | Loss: 2.9341 | LR: 1.01e-04 | 1282.1ms/step | 15.6 samples/s +Step 31190/44000 | Loss: 2.9052 | LR: 1.01e-04 | 1283.2ms/step | 15.6 samples/s +Step 31200/44000 | Loss: 2.9178 | LR: 1.00e-04 | 1285.2ms/step | 15.6 samples/s +Step 31210/44000 | Loss: 2.9365 | LR: 1.00e-04 | 1284.1ms/step | 15.6 samples/s +Step 31220/44000 | Loss: 2.9245 | LR: 1.00e-04 | 1286.2ms/step | 15.5 samples/s +Step 31230/44000 | Loss: 2.9589 | LR: 1.00e-04 | 1285.8ms/step | 15.6 samples/s +Step 31240/44000 | Loss: 2.9255 | LR: 9.99e-05 | 1284.6ms/step | 15.6 samples/s +Step 31250/44000 | Loss: 2.9095 | LR: 9.98e-05 | 1285.8ms/step | 15.6 samples/s +Validation at step 31250: Loss = 2.9428, Perplexity = 18.97 +Step 31260/44000 | Loss: 2.9403 | LR: 9.96e-05 | 3427.1ms/step | 5.8 samples/s +Step 31270/44000 | Loss: 2.9531 | LR: 9.95e-05 | 1283.0ms/step | 15.6 samples/s +Step 31280/44000 | Loss: 2.9143 | LR: 9.93e-05 | 1283.9ms/step | 15.6 samples/s +Step 31290/44000 | Loss: 2.9417 | LR: 9.92e-05 | 1279.5ms/step | 15.6 samples/s +Step 31300/44000 | Loss: 2.9393 | LR: 9.90e-05 | 1279.2ms/step | 15.6 samples/s +Step 31310/44000 | Loss: 2.9285 | LR: 9.89e-05 | 1279.4ms/step | 15.6 samples/s +Step 31320/44000 | Loss: 2.9239 | LR: 9.88e-05 | 1279.6ms/step | 15.6 samples/s +Step 31330/44000 | Loss: 2.9528 | LR: 9.86e-05 | 1279.7ms/step | 15.6 samples/s +Step 31340/44000 | Loss: 2.9194 | LR: 9.85e-05 | 1285.3ms/step | 15.6 samples/s +Step 31350/44000 | Loss: 2.8991 | LR: 9.83e-05 | 1284.5ms/step | 15.6 samples/s +Step 31360/44000 | Loss: 2.9077 | LR: 9.82e-05 | 1283.2ms/step | 15.6 samples/s +Step 31370/44000 | Loss: 2.9402 | LR: 9.80e-05 | 1284.4ms/step | 15.6 samples/s +Step 31380/44000 | Loss: 2.9549 | LR: 9.79e-05 | 1283.3ms/step | 15.6 samples/s +Step 31390/44000 | Loss: 2.8938 | LR: 9.77e-05 | 1286.0ms/step | 15.6 samples/s +Step 31400/44000 | Loss: 2.9261 | LR: 9.76e-05 | 1284.5ms/step | 15.6 samples/s +Step 31410/44000 | Loss: 2.8977 | LR: 9.75e-05 | 1286.5ms/step | 15.5 samples/s +Step 31420/44000 | Loss: 2.9180 | LR: 9.73e-05 | 1282.7ms/step | 15.6 samples/s +Step 31430/44000 | Loss: 2.9391 | LR: 9.72e-05 | 1284.3ms/step | 15.6 samples/s +Step 31440/44000 | Loss: 2.9238 | LR: 9.70e-05 | 1285.8ms/step | 15.6 samples/s +Step 31450/44000 | Loss: 2.9355 | LR: 9.69e-05 | 1281.2ms/step | 15.6 samples/s +Step 31460/44000 | Loss: 2.9146 | LR: 9.67e-05 | 1281.8ms/step | 15.6 samples/s +Step 31470/44000 | Loss: 2.9419 | LR: 9.66e-05 | 1283.3ms/step | 15.6 samples/s +Step 31480/44000 | Loss: 2.9060 | LR: 9.65e-05 | 1282.0ms/step | 15.6 samples/s +Step 31490/44000 | Loss: 2.9157 | LR: 9.63e-05 | 1282.4ms/step | 15.6 samples/s +Step 31500/44000 | Loss: 2.9284 | LR: 9.62e-05 | 1284.0ms/step | 15.6 samples/s +Validation at step 31500: Loss = 2.9421, Perplexity = 18.96 +Step 31510/44000 | Loss: 2.9089 | LR: 9.60e-05 | 3427.6ms/step | 5.8 samples/s +Step 31520/44000 | Loss: 2.9208 | LR: 9.59e-05 | 1283.7ms/step | 15.6 samples/s +Step 31530/44000 | Loss: 2.9330 | LR: 9.57e-05 | 1284.3ms/step | 15.6 samples/s +Step 31540/44000 | Loss: 2.9144 | LR: 9.56e-05 | 1287.7ms/step | 15.5 samples/s +Step 31550/44000 | Loss: 2.9398 | LR: 9.54e-05 | 1287.9ms/step | 15.5 samples/s +Step 31560/44000 | Loss: 2.9251 | LR: 9.53e-05 | 1281.3ms/step | 15.6 samples/s +Step 31570/44000 | Loss: 2.9164 | LR: 9.52e-05 | 1284.6ms/step | 15.6 samples/s +Step 31580/44000 | Loss: 2.9393 | LR: 9.50e-05 | 1291.7ms/step | 15.5 samples/s +Step 31590/44000 | Loss: 2.9415 | LR: 9.49e-05 | 1291.4ms/step | 15.5 samples/s +Step 31600/44000 | Loss: 2.9239 | LR: 9.47e-05 | 1291.1ms/step | 15.5 samples/s +Step 31610/44000 | Loss: 2.9122 | LR: 9.46e-05 | 1290.1ms/step | 15.5 samples/s +Step 31620/44000 | Loss: 2.9369 | LR: 9.45e-05 | 1288.2ms/step | 15.5 samples/s +Step 31630/44000 | Loss: 2.9185 | LR: 9.43e-05 | 1292.1ms/step | 15.5 samples/s +Step 31640/44000 | Loss: 2.8934 | LR: 9.42e-05 | 1288.1ms/step | 15.5 samples/s +Step 31650/44000 | Loss: 2.8972 | LR: 9.40e-05 | 1285.4ms/step | 15.6 samples/s +Step 31660/44000 | Loss: 2.9153 | LR: 9.39e-05 | 1285.1ms/step | 15.6 samples/s +Step 31670/44000 | Loss: 2.9078 | LR: 9.37e-05 | 1283.2ms/step | 15.6 samples/s +Step 31680/44000 | Loss: 2.9204 | LR: 9.36e-05 | 1281.5ms/step | 15.6 samples/s +Step 31690/44000 | Loss: 2.9295 | LR: 9.35e-05 | 1282.0ms/step | 15.6 samples/s +Step 31700/44000 | Loss: 2.9452 | LR: 9.33e-05 | 1281.3ms/step | 15.6 samples/s +Step 31710/44000 | Loss: 2.9204 | LR: 9.32e-05 | 1281.2ms/step | 15.6 samples/s +Step 31720/44000 | Loss: 2.9199 | LR: 9.30e-05 | 1278.9ms/step | 15.6 samples/s +Step 31730/44000 | Loss: 2.8939 | LR: 9.29e-05 | 1280.8ms/step | 15.6 samples/s +Step 31740/44000 | Loss: 2.9279 | LR: 9.28e-05 | 1282.2ms/step | 15.6 samples/s +Step 31750/44000 | Loss: 2.9365 | LR: 9.26e-05 | 1284.0ms/step | 15.6 samples/s +Validation at step 31750: Loss = 2.9415, Perplexity = 18.94 +Step 31760/44000 | Loss: 2.9106 | LR: 9.25e-05 | 3430.3ms/step | 5.8 samples/s +Step 31770/44000 | Loss: 2.9355 | LR: 9.23e-05 | 1284.7ms/step | 15.6 samples/s +Step 31780/44000 | Loss: 2.9277 | LR: 9.22e-05 | 1285.0ms/step | 15.6 samples/s +Step 31790/44000 | Loss: 2.9287 | LR: 9.20e-05 | 1284.1ms/step | 15.6 samples/s +Step 31800/44000 | Loss: 2.8991 | LR: 9.19e-05 | 1284.0ms/step | 15.6 samples/s +Step 31810/44000 | Loss: 2.8996 | LR: 9.18e-05 | 1283.5ms/step | 15.6 samples/s +Step 31820/44000 | Loss: 2.9165 | LR: 9.16e-05 | 1285.9ms/step | 15.6 samples/s +Step 31830/44000 | Loss: 2.9217 | LR: 9.15e-05 | 1285.8ms/step | 15.6 samples/s +Step 31840/44000 | Loss: 2.9347 | LR: 9.13e-05 | 1284.6ms/step | 15.6 samples/s +Step 31850/44000 | Loss: 2.9364 | LR: 9.12e-05 | 1283.8ms/step | 15.6 samples/s +Step 31860/44000 | Loss: 2.9444 | LR: 9.11e-05 | 1283.7ms/step | 15.6 samples/s +Step 31870/44000 | Loss: 2.9212 | LR: 9.09e-05 | 1282.2ms/step | 15.6 samples/s +Step 31880/44000 | Loss: 2.9558 | LR: 9.08e-05 | 1284.7ms/step | 15.6 samples/s +Step 31890/44000 | Loss: 2.8963 | LR: 9.06e-05 | 1281.2ms/step | 15.6 samples/s +Step 31900/44000 | Loss: 2.9150 | LR: 9.05e-05 | 1282.5ms/step | 15.6 samples/s +Step 31910/44000 | Loss: 2.9477 | LR: 9.04e-05 | 1284.9ms/step | 15.6 samples/s +Step 31920/44000 | Loss: 2.8748 | LR: 9.02e-05 | 1285.1ms/step | 15.6 samples/s +Step 31930/44000 | Loss: 2.9314 | LR: 9.01e-05 | 1285.5ms/step | 15.6 samples/s +Step 31940/44000 | Loss: 2.9392 | LR: 8.99e-05 | 1283.9ms/step | 15.6 samples/s +Step 31950/44000 | Loss: 2.9279 | LR: 8.98e-05 | 1283.0ms/step | 15.6 samples/s +Step 31960/44000 | Loss: 2.9429 | LR: 8.97e-05 | 1285.6ms/step | 15.6 samples/s +Step 31970/44000 | Loss: 2.9110 | LR: 8.95e-05 | 1285.2ms/step | 15.6 samples/s +Step 31980/44000 | Loss: 2.9223 | LR: 8.94e-05 | 1289.5ms/step | 15.5 samples/s +Step 31990/44000 | Loss: 2.8902 | LR: 8.92e-05 | 1288.0ms/step | 15.5 samples/s +Step 32000/44000 | Loss: 2.9134 | LR: 8.91e-05 | 1288.1ms/step | 15.5 samples/s +Validation at step 32000: Loss = 2.9409, Perplexity = 18.93 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-32000 +Step 32010/44000 | Loss: 2.9345 | LR: 8.90e-05 | 3734.6ms/step | 5.4 samples/s +Step 32020/44000 | Loss: 2.9241 | LR: 8.88e-05 | 1296.2ms/step | 15.4 samples/s +Step 32030/44000 | Loss: 2.9325 | LR: 8.87e-05 | 1288.8ms/step | 15.5 samples/s +Step 32040/44000 | Loss: 2.9346 | LR: 8.86e-05 | 1282.7ms/step | 15.6 samples/s +Step 32050/44000 | Loss: 2.9090 | LR: 8.84e-05 | 1288.4ms/step | 15.5 samples/s +Step 32060/44000 | Loss: 2.8948 | LR: 8.83e-05 | 1288.2ms/step | 15.5 samples/s +Step 32070/44000 | Loss: 2.9197 | LR: 8.81e-05 | 1281.8ms/step | 15.6 samples/s +Step 32080/44000 | Loss: 2.8944 | LR: 8.80e-05 | 1280.5ms/step | 15.6 samples/s +Step 32090/44000 | Loss: 2.9289 | LR: 8.79e-05 | 1282.8ms/step | 15.6 samples/s +Step 32100/44000 | Loss: 2.9300 | LR: 8.77e-05 | 1280.8ms/step | 15.6 samples/s +Step 32110/44000 | Loss: 2.9134 | LR: 8.76e-05 | 1285.5ms/step | 15.6 samples/s +Step 32120/44000 | Loss: 2.9174 | LR: 8.74e-05 | 1286.1ms/step | 15.6 samples/s +Step 32130/44000 | Loss: 2.9106 | LR: 8.73e-05 | 1283.6ms/step | 15.6 samples/s +Step 32140/44000 | Loss: 2.9076 | LR: 8.72e-05 | 1282.4ms/step | 15.6 samples/s +Step 32150/44000 | Loss: 2.9258 | LR: 8.70e-05 | 1283.3ms/step | 15.6 samples/s +Step 32160/44000 | Loss: 2.8965 | LR: 8.69e-05 | 1280.6ms/step | 15.6 samples/s +Step 32170/44000 | Loss: 2.9203 | LR: 8.68e-05 | 1277.6ms/step | 15.7 samples/s +Step 32180/44000 | Loss: 2.9531 | LR: 8.66e-05 | 1276.9ms/step | 15.7 samples/s +Step 32190/44000 | Loss: 2.9251 | LR: 8.65e-05 | 1278.9ms/step | 15.6 samples/s +Step 32200/44000 | Loss: 2.9385 | LR: 8.63e-05 | 1278.1ms/step | 15.6 samples/s +Step 32210/44000 | Loss: 2.9438 | LR: 8.62e-05 | 1278.3ms/step | 15.6 samples/s +Step 32220/44000 | Loss: 2.8989 | LR: 8.61e-05 | 1283.8ms/step | 15.6 samples/s +Step 32230/44000 | Loss: 2.9280 | LR: 8.59e-05 | 1283.7ms/step | 15.6 samples/s +Step 32240/44000 | Loss: 2.9150 | LR: 8.58e-05 | 1284.3ms/step | 15.6 samples/s +Step 32250/44000 | Loss: 2.9365 | LR: 8.57e-05 | 1283.2ms/step | 15.6 samples/s +Validation at step 32250: Loss = 2.9404, Perplexity = 18.92 +Step 32260/44000 | Loss: 2.9495 | LR: 8.55e-05 | 3419.6ms/step | 5.8 samples/s +Step 32270/44000 | Loss: 2.9237 | LR: 8.54e-05 | 1278.8ms/step | 15.6 samples/s +Step 32280/44000 | Loss: 2.9462 | LR: 8.53e-05 | 1279.2ms/step | 15.6 samples/s +Step 32290/44000 | Loss: 2.9131 | LR: 8.51e-05 | 1277.0ms/step | 15.7 samples/s +Step 32300/44000 | Loss: 2.9094 | LR: 8.50e-05 | 1278.4ms/step | 15.6 samples/s +Step 32310/44000 | Loss: 2.9327 | LR: 8.48e-05 | 1282.1ms/step | 15.6 samples/s +Step 32320/44000 | Loss: 2.9506 | LR: 8.47e-05 | 1284.8ms/step | 15.6 samples/s +Step 32330/44000 | Loss: 2.9204 | LR: 8.46e-05 | 1286.0ms/step | 15.6 samples/s +Step 32340/44000 | Loss: 2.9172 | LR: 8.44e-05 | 1284.3ms/step | 15.6 samples/s +Step 32350/44000 | Loss: 2.9397 | LR: 8.43e-05 | 1280.7ms/step | 15.6 samples/s +Step 32360/44000 | Loss: 2.9184 | LR: 8.42e-05 | 1280.2ms/step | 15.6 samples/s +Step 32370/44000 | Loss: 2.9177 | LR: 8.40e-05 | 1280.4ms/step | 15.6 samples/s +Step 32380/44000 | Loss: 2.9190 | LR: 8.39e-05 | 1280.0ms/step | 15.6 samples/s +Step 32390/44000 | Loss: 2.9198 | LR: 8.38e-05 | 1279.4ms/step | 15.6 samples/s +Step 32400/44000 | Loss: 2.9259 | LR: 8.36e-05 | 1280.2ms/step | 15.6 samples/s +Step 32410/44000 | Loss: 2.9217 | LR: 8.35e-05 | 1279.6ms/step | 15.6 samples/s +Step 32420/44000 | Loss: 2.9186 | LR: 8.33e-05 | 1279.9ms/step | 15.6 samples/s +Step 32430/44000 | Loss: 2.9664 | LR: 8.32e-05 | 1277.1ms/step | 15.7 samples/s +Step 32440/44000 | Loss: 2.9331 | LR: 8.31e-05 | 1279.5ms/step | 15.6 samples/s +Step 32450/44000 | Loss: 2.9258 | LR: 8.29e-05 | 1282.0ms/step | 15.6 samples/s +Step 32460/44000 | Loss: 2.9465 | LR: 8.28e-05 | 1282.0ms/step | 15.6 samples/s +Step 32470/44000 | Loss: 2.9030 | LR: 8.27e-05 | 1283.5ms/step | 15.6 samples/s +Step 32480/44000 | Loss: 2.9357 | LR: 8.25e-05 | 1284.7ms/step | 15.6 samples/s +Step 32490/44000 | Loss: 2.9124 | LR: 8.24e-05 | 1285.4ms/step | 15.6 samples/s +Step 32500/44000 | Loss: 2.9307 | LR: 8.23e-05 | 1281.1ms/step | 15.6 samples/s +Validation at step 32500: Loss = 2.9399, Perplexity = 18.91 +Step 32510/44000 | Loss: 2.8984 | LR: 8.21e-05 | 3423.0ms/step | 5.8 samples/s +Step 32520/44000 | Loss: 2.9237 | LR: 8.20e-05 | 1277.9ms/step | 15.7 samples/s +Step 32530/44000 | Loss: 2.9535 | LR: 8.19e-05 | 1276.8ms/step | 15.7 samples/s +Step 32540/44000 | Loss: 2.9092 | LR: 8.17e-05 | 1275.6ms/step | 15.7 samples/s +Step 32550/44000 | Loss: 2.8790 | LR: 8.16e-05 | 1278.5ms/step | 15.6 samples/s +Step 32560/44000 | Loss: 2.9454 | LR: 8.15e-05 | 1278.8ms/step | 15.6 samples/s +Step 32570/44000 | Loss: 2.9254 | LR: 8.13e-05 | 1281.7ms/step | 15.6 samples/s +Step 32580/44000 | Loss: 2.9313 | LR: 8.12e-05 | 1281.0ms/step | 15.6 samples/s +Step 32590/44000 | Loss: 2.9545 | LR: 8.11e-05 | 1281.7ms/step | 15.6 samples/s +Step 32600/44000 | Loss: 2.9308 | LR: 8.09e-05 | 1278.7ms/step | 15.6 samples/s +Step 32610/44000 | Loss: 2.9328 | LR: 8.08e-05 | 1273.7ms/step | 15.7 samples/s +Step 32620/44000 | Loss: 2.9407 | LR: 8.07e-05 | 1275.6ms/step | 15.7 samples/s +Step 32630/44000 | Loss: 2.9005 | LR: 8.05e-05 | 1278.3ms/step | 15.6 samples/s +Step 32640/44000 | Loss: 2.9624 | LR: 8.04e-05 | 1278.1ms/step | 15.6 samples/s +Step 32650/44000 | Loss: 2.9392 | LR: 8.03e-05 | 1274.6ms/step | 15.7 samples/s +Step 32660/44000 | Loss: 2.9273 | LR: 8.01e-05 | 1274.3ms/step | 15.7 samples/s +Step 32670/44000 | Loss: 2.9643 | LR: 8.00e-05 | 1274.3ms/step | 15.7 samples/s +Step 32680/44000 | Loss: 2.9217 | LR: 7.99e-05 | 1274.2ms/step | 15.7 samples/s +Step 32690/44000 | Loss: 2.9321 | LR: 7.97e-05 | 1273.1ms/step | 15.7 samples/s +Step 32700/44000 | Loss: 2.9120 | LR: 7.96e-05 | 1277.0ms/step | 15.7 samples/s +Step 32710/44000 | Loss: 2.9405 | LR: 7.95e-05 | 1279.4ms/step | 15.6 samples/s +Step 32720/44000 | Loss: 2.9393 | LR: 7.93e-05 | 1280.3ms/step | 15.6 samples/s +Step 32730/44000 | Loss: 2.9177 | LR: 7.92e-05 | 1281.9ms/step | 15.6 samples/s +Step 32740/44000 | Loss: 2.9063 | LR: 7.91e-05 | 1280.6ms/step | 15.6 samples/s +Step 32750/44000 | Loss: 2.8978 | LR: 7.89e-05 | 1281.3ms/step | 15.6 samples/s +Validation at step 32750: Loss = 2.9394, Perplexity = 18.90 +Step 32760/44000 | Loss: 2.9324 | LR: 7.88e-05 | 3428.4ms/step | 5.8 samples/s +Step 32770/44000 | Loss: 2.9292 | LR: 7.87e-05 | 1283.1ms/step | 15.6 samples/s +Step 32780/44000 | Loss: 2.9546 | LR: 7.85e-05 | 1284.2ms/step | 15.6 samples/s +Step 32790/44000 | Loss: 2.9598 | LR: 7.84e-05 | 1284.7ms/step | 15.6 samples/s +Step 32800/44000 | Loss: 2.9151 | LR: 7.83e-05 | 1284.4ms/step | 15.6 samples/s +Step 32810/44000 | Loss: 2.9249 | LR: 7.81e-05 | 1285.2ms/step | 15.6 samples/s +Step 32820/44000 | Loss: 2.9433 | LR: 7.80e-05 | 1285.1ms/step | 15.6 samples/s +Step 32830/44000 | Loss: 2.9392 | LR: 7.79e-05 | 1285.1ms/step | 15.6 samples/s +Step 32840/44000 | Loss: 2.8909 | LR: 7.77e-05 | 1285.6ms/step | 15.6 samples/s +Step 32850/44000 | Loss: 2.9214 | LR: 7.76e-05 | 1284.3ms/step | 15.6 samples/s +Step 32860/44000 | Loss: 2.9200 | LR: 7.75e-05 | 1285.4ms/step | 15.6 samples/s +Step 32870/44000 | Loss: 2.9367 | LR: 7.73e-05 | 1287.6ms/step | 15.5 samples/s +Step 32880/44000 | Loss: 2.8981 | LR: 7.72e-05 | 1290.9ms/step | 15.5 samples/s +Step 32890/44000 | Loss: 2.9348 | LR: 7.71e-05 | 1291.0ms/step | 15.5 samples/s +Step 32900/44000 | Loss: 2.9300 | LR: 7.70e-05 | 1295.4ms/step | 15.4 samples/s +Step 32910/44000 | Loss: 2.9203 | LR: 7.68e-05 | 1295.1ms/step | 15.4 samples/s +Step 32920/44000 | Loss: 2.9458 | LR: 7.67e-05 | 1290.3ms/step | 15.5 samples/s +Step 32930/44000 | Loss: 2.9174 | LR: 7.66e-05 | 1292.5ms/step | 15.5 samples/s +Step 32940/44000 | Loss: 2.9029 | LR: 7.64e-05 | 1291.6ms/step | 15.5 samples/s +Step 32950/44000 | Loss: 2.9097 | LR: 7.63e-05 | 1285.6ms/step | 15.6 samples/s +Step 32960/44000 | Loss: 2.9180 | LR: 7.62e-05 | 1287.9ms/step | 15.5 samples/s +Step 32970/44000 | Loss: 2.9221 | LR: 7.60e-05 | 1289.1ms/step | 15.5 samples/s +Step 32980/44000 | Loss: 2.9249 | LR: 7.59e-05 | 1292.2ms/step | 15.5 samples/s +Step 32990/44000 | Loss: 2.9281 | LR: 7.58e-05 | 1292.4ms/step | 15.5 samples/s +Step 33000/44000 | Loss: 2.9493 | LR: 7.56e-05 | 1292.0ms/step | 15.5 samples/s +Validation at step 33000: Loss = 2.9389, Perplexity = 18.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-33000 +Step 33010/44000 | Loss: 2.9375 | LR: 7.55e-05 | 3737.2ms/step | 5.4 samples/s +Step 33020/44000 | Loss: 2.9015 | LR: 7.54e-05 | 1282.2ms/step | 15.6 samples/s +Step 33030/44000 | Loss: 2.9453 | LR: 7.53e-05 | 1283.4ms/step | 15.6 samples/s +Step 33040/44000 | Loss: 2.9285 | LR: 7.51e-05 | 1284.2ms/step | 15.6 samples/s +Step 33050/44000 | Loss: 2.9390 | LR: 7.50e-05 | 1286.4ms/step | 15.5 samples/s +Step 33060/44000 | Loss: 2.9081 | LR: 7.49e-05 | 1286.5ms/step | 15.5 samples/s +Step 33070/44000 | Loss: 2.9384 | LR: 7.47e-05 | 1286.9ms/step | 15.5 samples/s +Step 33080/44000 | Loss: 2.9394 | LR: 7.46e-05 | 1286.5ms/step | 15.5 samples/s +Step 33090/44000 | Loss: 2.9282 | LR: 7.45e-05 | 1290.1ms/step | 15.5 samples/s +Step 33100/44000 | Loss: 2.9011 | LR: 7.44e-05 | 1287.2ms/step | 15.5 samples/s +Step 33110/44000 | Loss: 2.8788 | LR: 7.42e-05 | 1291.9ms/step | 15.5 samples/s +Step 33120/44000 | Loss: 2.9107 | LR: 7.41e-05 | 1293.2ms/step | 15.5 samples/s +Step 33130/44000 | Loss: 2.9274 | LR: 7.40e-05 | 1286.0ms/step | 15.6 samples/s +Step 33140/44000 | Loss: 2.9447 | LR: 7.38e-05 | 1285.6ms/step | 15.6 samples/s +Step 33150/44000 | Loss: 2.9366 | LR: 7.37e-05 | 1292.9ms/step | 15.5 samples/s +Step 33160/44000 | Loss: 2.9131 | LR: 7.36e-05 | 1285.9ms/step | 15.6 samples/s +Step 33170/44000 | Loss: 2.9182 | LR: 7.34e-05 | 1285.4ms/step | 15.6 samples/s +Step 33180/44000 | Loss: 2.9428 | LR: 7.33e-05 | 1285.8ms/step | 15.6 samples/s +Step 33190/44000 | Loss: 2.9418 | LR: 7.32e-05 | 1286.6ms/step | 15.5 samples/s +Step 33200/44000 | Loss: 2.9319 | LR: 7.31e-05 | 1286.8ms/step | 15.5 samples/s +Step 33210/44000 | Loss: 2.9192 | LR: 7.29e-05 | 1289.0ms/step | 15.5 samples/s +Step 33220/44000 | Loss: 2.9376 | LR: 7.28e-05 | 1289.3ms/step | 15.5 samples/s +Step 33230/44000 | Loss: 2.9202 | LR: 7.27e-05 | 1292.2ms/step | 15.5 samples/s +Step 33240/44000 | Loss: 2.9142 | LR: 7.26e-05 | 1290.4ms/step | 15.5 samples/s +Step 33250/44000 | Loss: 2.9242 | LR: 7.24e-05 | 1289.8ms/step | 15.5 samples/s +Validation at step 33250: Loss = 2.9386, Perplexity = 18.89 +Step 33260/44000 | Loss: 2.9365 | LR: 7.23e-05 | 3445.2ms/step | 5.8 samples/s +Step 33270/44000 | Loss: 2.9172 | LR: 7.22e-05 | 1291.0ms/step | 15.5 samples/s +Step 33280/44000 | Loss: 2.9358 | LR: 7.20e-05 | 1291.2ms/step | 15.5 samples/s +Step 33290/44000 | Loss: 2.8830 | LR: 7.19e-05 | 1291.3ms/step | 15.5 samples/s +Step 33300/44000 | Loss: 2.9261 | LR: 7.18e-05 | 1290.2ms/step | 15.5 samples/s +Step 33310/44000 | Loss: 2.9412 | LR: 7.17e-05 | 1285.2ms/step | 15.6 samples/s +Step 33320/44000 | Loss: 2.9238 | LR: 7.15e-05 | 1283.1ms/step | 15.6 samples/s +Step 33330/44000 | Loss: 2.9138 | LR: 7.14e-05 | 1287.9ms/step | 15.5 samples/s +Step 33340/44000 | Loss: 2.9148 | LR: 7.13e-05 | 1287.5ms/step | 15.5 samples/s +Step 33350/44000 | Loss: 2.9255 | LR: 7.11e-05 | 1287.1ms/step | 15.5 samples/s +Step 33360/44000 | Loss: 2.9369 | LR: 7.10e-05 | 1290.9ms/step | 15.5 samples/s +Step 33370/44000 | Loss: 2.8968 | LR: 7.09e-05 | 1282.6ms/step | 15.6 samples/s +Step 33380/44000 | Loss: 2.9135 | LR: 7.08e-05 | 1289.6ms/step | 15.5 samples/s +Step 33390/44000 | Loss: 2.9442 | LR: 7.06e-05 | 1290.9ms/step | 15.5 samples/s +Step 33400/44000 | Loss: 2.8909 | LR: 7.05e-05 | 1294.2ms/step | 15.5 samples/s +Step 33410/44000 | Loss: 2.9380 | LR: 7.04e-05 | 1293.3ms/step | 15.5 samples/s +Step 33420/44000 | Loss: 2.9156 | LR: 7.03e-05 | 1290.8ms/step | 15.5 samples/s +Step 33430/44000 | Loss: 2.9126 | LR: 7.01e-05 | 1285.6ms/step | 15.6 samples/s +Step 33440/44000 | Loss: 2.9408 | LR: 7.00e-05 | 1279.6ms/step | 15.6 samples/s +Step 33450/44000 | Loss: 2.9357 | LR: 6.99e-05 | 1275.9ms/step | 15.7 samples/s +Step 33460/44000 | Loss: 2.9282 | LR: 6.98e-05 | 1278.0ms/step | 15.6 samples/s +Step 33470/44000 | Loss: 2.9209 | LR: 6.96e-05 | 1280.3ms/step | 15.6 samples/s +Step 33480/44000 | Loss: 2.9072 | LR: 6.95e-05 | 1279.4ms/step | 15.6 samples/s +Step 33490/44000 | Loss: 2.9380 | LR: 6.94e-05 | 1277.3ms/step | 15.7 samples/s +Step 33500/44000 | Loss: 2.8972 | LR: 6.93e-05 | 1277.9ms/step | 15.7 samples/s +Validation at step 33500: Loss = 2.9383, Perplexity = 18.88 +Step 33510/44000 | Loss: 2.9020 | LR: 6.91e-05 | 3423.5ms/step | 5.8 samples/s +Step 33520/44000 | Loss: 2.9421 | LR: 6.90e-05 | 1279.5ms/step | 15.6 samples/s +Step 33530/44000 | Loss: 2.9035 | LR: 6.89e-05 | 1278.7ms/step | 15.6 samples/s +Step 33540/44000 | Loss: 2.9120 | LR: 6.88e-05 | 1279.2ms/step | 15.6 samples/s +Step 33550/44000 | Loss: 2.9206 | LR: 6.86e-05 | 1278.4ms/step | 15.6 samples/s +Step 33560/44000 | Loss: 2.9009 | LR: 6.85e-05 | 1279.1ms/step | 15.6 samples/s +Step 33570/44000 | Loss: 2.8953 | LR: 6.84e-05 | 1281.2ms/step | 15.6 samples/s +Step 33580/44000 | Loss: 2.9409 | LR: 6.83e-05 | 1280.0ms/step | 15.6 samples/s +Step 33590/44000 | Loss: 2.9373 | LR: 6.81e-05 | 1279.9ms/step | 15.6 samples/s +Step 33600/44000 | Loss: 2.9449 | LR: 6.80e-05 | 1280.9ms/step | 15.6 samples/s +Step 33610/44000 | Loss: 2.8990 | LR: 6.79e-05 | 1282.7ms/step | 15.6 samples/s +Step 33620/44000 | Loss: 2.9152 | LR: 6.78e-05 | 1283.8ms/step | 15.6 samples/s +Step 33630/44000 | Loss: 2.9034 | LR: 6.76e-05 | 1282.8ms/step | 15.6 samples/s +Step 33640/44000 | Loss: 2.9168 | LR: 6.75e-05 | 1282.2ms/step | 15.6 samples/s +Step 33650/44000 | Loss: 2.9251 | LR: 6.74e-05 | 1280.7ms/step | 15.6 samples/s +Step 33660/44000 | Loss: 2.9300 | LR: 6.73e-05 | 1280.3ms/step | 15.6 samples/s +Step 33670/44000 | Loss: 2.9380 | LR: 6.71e-05 | 1284.3ms/step | 15.6 samples/s +Step 33680/44000 | Loss: 2.9228 | LR: 6.70e-05 | 1280.5ms/step | 15.6 samples/s +Step 33690/44000 | Loss: 2.9199 | LR: 6.69e-05 | 1281.4ms/step | 15.6 samples/s +Step 33700/44000 | Loss: 2.9366 | LR: 6.68e-05 | 1280.3ms/step | 15.6 samples/s +Step 33710/44000 | Loss: 2.9500 | LR: 6.66e-05 | 1282.0ms/step | 15.6 samples/s +Step 33720/44000 | Loss: 2.9271 | LR: 6.65e-05 | 1281.1ms/step | 15.6 samples/s +Step 33730/44000 | Loss: 2.9115 | LR: 6.64e-05 | 1286.7ms/step | 15.5 samples/s +Step 33740/44000 | Loss: 2.9262 | LR: 6.63e-05 | 1285.4ms/step | 15.6 samples/s +Step 33750/44000 | Loss: 2.9321 | LR: 6.61e-05 | 1286.5ms/step | 15.5 samples/s +Validation at step 33750: Loss = 2.9378, Perplexity = 18.87 +Step 33760/44000 | Loss: 2.8967 | LR: 6.60e-05 | 3435.0ms/step | 5.8 samples/s +Step 33770/44000 | Loss: 2.9452 | LR: 6.59e-05 | 1283.5ms/step | 15.6 samples/s +Step 33780/44000 | Loss: 2.9480 | LR: 6.58e-05 | 1285.1ms/step | 15.6 samples/s +Step 33790/44000 | Loss: 2.9016 | LR: 6.57e-05 | 1283.5ms/step | 15.6 samples/s +Step 33800/44000 | Loss: 2.9341 | LR: 6.55e-05 | 1282.7ms/step | 15.6 samples/s +Step 33810/44000 | Loss: 2.9506 | LR: 6.54e-05 | 1282.3ms/step | 15.6 samples/s +Step 33820/44000 | Loss: 2.9232 | LR: 6.53e-05 | 1282.6ms/step | 15.6 samples/s +Step 33830/44000 | Loss: 2.9440 | LR: 6.52e-05 | 1281.6ms/step | 15.6 samples/s +Step 33840/44000 | Loss: 2.9248 | LR: 6.50e-05 | 1281.4ms/step | 15.6 samples/s +Step 33850/44000 | Loss: 2.9348 | LR: 6.49e-05 | 1282.0ms/step | 15.6 samples/s +Step 33860/44000 | Loss: 2.9192 | LR: 6.48e-05 | 1279.7ms/step | 15.6 samples/s +Step 33870/44000 | Loss: 2.9290 | LR: 6.47e-05 | 1280.2ms/step | 15.6 samples/s +Step 33880/44000 | Loss: 2.9586 | LR: 6.46e-05 | 1280.3ms/step | 15.6 samples/s +Step 33890/44000 | Loss: 2.9166 | LR: 6.44e-05 | 1281.9ms/step | 15.6 samples/s +Step 33900/44000 | Loss: 2.9414 | LR: 6.43e-05 | 1281.0ms/step | 15.6 samples/s +Step 33910/44000 | Loss: 2.9316 | LR: 6.42e-05 | 1283.9ms/step | 15.6 samples/s +Step 33920/44000 | Loss: 2.9244 | LR: 6.41e-05 | 1279.4ms/step | 15.6 samples/s +Step 33930/44000 | Loss: 2.8831 | LR: 6.40e-05 | 1280.0ms/step | 15.6 samples/s +Step 33940/44000 | Loss: 2.9196 | LR: 6.38e-05 | 1279.7ms/step | 15.6 samples/s +Step 33950/44000 | Loss: 2.9159 | LR: 6.37e-05 | 1280.4ms/step | 15.6 samples/s +Step 33960/44000 | Loss: 2.9046 | LR: 6.36e-05 | 1281.6ms/step | 15.6 samples/s +Step 33970/44000 | Loss: 2.9136 | LR: 6.35e-05 | 1281.4ms/step | 15.6 samples/s +Step 33980/44000 | Loss: 2.9468 | LR: 6.33e-05 | 1284.3ms/step | 15.6 samples/s +Step 33990/44000 | Loss: 2.9393 | LR: 6.32e-05 | 1283.8ms/step | 15.6 samples/s +Step 34000/44000 | Loss: 2.9425 | LR: 6.31e-05 | 1282.3ms/step | 15.6 samples/s +Validation at step 34000: Loss = 2.9375, Perplexity = 18.87 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-34000 +Step 34010/44000 | Loss: 2.9216 | LR: 6.30e-05 | 3734.0ms/step | 5.4 samples/s +Step 34020/44000 | Loss: 2.9158 | LR: 6.29e-05 | 1282.9ms/step | 15.6 samples/s +Step 34030/44000 | Loss: 2.9635 | LR: 6.27e-05 | 1284.3ms/step | 15.6 samples/s +Step 34040/44000 | Loss: 2.9157 | LR: 6.26e-05 | 1283.2ms/step | 15.6 samples/s +Step 34050/44000 | Loss: 2.9169 | LR: 6.25e-05 | 1283.9ms/step | 15.6 samples/s +Step 34060/44000 | Loss: 2.9220 | LR: 6.24e-05 | 1284.1ms/step | 15.6 samples/s +Step 34070/44000 | Loss: 2.9457 | LR: 6.23e-05 | 1284.1ms/step | 15.6 samples/s +Step 34080/44000 | Loss: 2.9288 | LR: 6.21e-05 | 1283.1ms/step | 15.6 samples/s +Step 34090/44000 | Loss: 2.9110 | LR: 6.20e-05 | 1282.9ms/step | 15.6 samples/s +Step 34100/44000 | Loss: 2.9262 | LR: 6.19e-05 | 1284.2ms/step | 15.6 samples/s +Step 34110/44000 | Loss: 2.8943 | LR: 6.18e-05 | 1283.4ms/step | 15.6 samples/s +Step 34120/44000 | Loss: 2.9188 | LR: 6.17e-05 | 1285.2ms/step | 15.6 samples/s +Step 34130/44000 | Loss: 2.9224 | LR: 6.15e-05 | 1286.2ms/step | 15.5 samples/s +Step 34140/44000 | Loss: 2.9321 | LR: 6.14e-05 | 1286.5ms/step | 15.5 samples/s +Step 34150/44000 | Loss: 2.9271 | LR: 6.13e-05 | 1286.5ms/step | 15.5 samples/s +Step 34160/44000 | Loss: 2.9552 | LR: 6.12e-05 | 1288.0ms/step | 15.5 samples/s +Step 34170/44000 | Loss: 2.9332 | LR: 6.11e-05 | 1284.7ms/step | 15.6 samples/s +Step 34180/44000 | Loss: 2.9085 | LR: 6.09e-05 | 1284.4ms/step | 15.6 samples/s +Step 34190/44000 | Loss: 2.9318 | LR: 6.08e-05 | 1286.0ms/step | 15.6 samples/s +Step 34200/44000 | Loss: 2.9513 | LR: 6.07e-05 | 1281.7ms/step | 15.6 samples/s +Step 34210/44000 | Loss: 2.9314 | LR: 6.06e-05 | 1283.4ms/step | 15.6 samples/s +Step 34220/44000 | Loss: 2.9232 | LR: 6.05e-05 | 1281.3ms/step | 15.6 samples/s +Step 34230/44000 | Loss: 2.9394 | LR: 6.04e-05 | 1282.7ms/step | 15.6 samples/s +Step 34240/44000 | Loss: 2.9568 | LR: 6.02e-05 | 1283.4ms/step | 15.6 samples/s +Step 34250/44000 | Loss: 2.9344 | LR: 6.01e-05 | 1280.4ms/step | 15.6 samples/s +Validation at step 34250: Loss = 2.9371, Perplexity = 18.86 +Step 34260/44000 | Loss: 2.9318 | LR: 6.00e-05 | 3447.8ms/step | 5.8 samples/s +Step 34270/44000 | Loss: 2.9054 | LR: 5.99e-05 | 1285.9ms/step | 15.6 samples/s +Step 34280/44000 | Loss: 2.9485 | LR: 5.98e-05 | 1286.5ms/step | 15.5 samples/s +Step 34290/44000 | Loss: 2.9088 | LR: 5.96e-05 | 1282.8ms/step | 15.6 samples/s +Step 34300/44000 | Loss: 2.9069 | LR: 5.95e-05 | 1279.2ms/step | 15.6 samples/s +Step 34310/44000 | Loss: 2.9218 | LR: 5.94e-05 | 1278.6ms/step | 15.6 samples/s +Step 34320/44000 | Loss: 2.9713 | LR: 5.93e-05 | 1277.3ms/step | 15.7 samples/s +Step 34330/44000 | Loss: 2.8926 | LR: 5.92e-05 | 1275.4ms/step | 15.7 samples/s +Step 34340/44000 | Loss: 2.8963 | LR: 5.91e-05 | 1277.1ms/step | 15.7 samples/s +Step 34350/44000 | Loss: 2.9166 | LR: 5.89e-05 | 1276.6ms/step | 15.7 samples/s +Step 34360/44000 | Loss: 2.9288 | LR: 5.88e-05 | 1279.4ms/step | 15.6 samples/s +Step 34370/44000 | Loss: 2.9163 | LR: 5.87e-05 | 1277.1ms/step | 15.7 samples/s +Step 34380/44000 | Loss: 2.9154 | LR: 5.86e-05 | 1276.3ms/step | 15.7 samples/s +Step 34390/44000 | Loss: 2.9277 | LR: 5.85e-05 | 1279.4ms/step | 15.6 samples/s +Step 34400/44000 | Loss: 2.9261 | LR: 5.84e-05 | 1284.3ms/step | 15.6 samples/s +Step 34410/44000 | Loss: 2.9446 | LR: 5.82e-05 | 1283.9ms/step | 15.6 samples/s +Step 34420/44000 | Loss: 2.9237 | LR: 5.81e-05 | 1285.6ms/step | 15.6 samples/s +Step 34430/44000 | Loss: 2.9204 | LR: 5.80e-05 | 1287.9ms/step | 15.5 samples/s +Step 34440/44000 | Loss: 2.9323 | LR: 5.79e-05 | 1285.7ms/step | 15.6 samples/s +Step 34450/44000 | Loss: 2.9110 | LR: 5.78e-05 | 1285.6ms/step | 15.6 samples/s +Step 34460/44000 | Loss: 2.9400 | LR: 5.77e-05 | 1290.7ms/step | 15.5 samples/s +Step 34470/44000 | Loss: 2.9224 | LR: 5.75e-05 | 1288.8ms/step | 15.5 samples/s +Step 34480/44000 | Loss: 2.9287 | LR: 5.74e-05 | 1290.6ms/step | 15.5 samples/s +Step 34490/44000 | Loss: 2.8920 | LR: 5.73e-05 | 1286.8ms/step | 15.5 samples/s +Step 34500/44000 | Loss: 2.9400 | LR: 5.72e-05 | 1287.9ms/step | 15.5 samples/s +Validation at step 34500: Loss = 2.9368, Perplexity = 18.85 +Step 34510/44000 | Loss: 2.9234 | LR: 5.71e-05 | 3446.7ms/step | 5.8 samples/s +Step 34520/44000 | Loss: 2.9163 | LR: 5.70e-05 | 1287.8ms/step | 15.5 samples/s +Step 34530/44000 | Loss: 2.9071 | LR: 5.69e-05 | 1282.6ms/step | 15.6 samples/s +Step 34540/44000 | Loss: 2.9269 | LR: 5.67e-05 | 1282.0ms/step | 15.6 samples/s +Step 34550/44000 | Loss: 2.9447 | LR: 5.66e-05 | 1280.5ms/step | 15.6 samples/s +Step 34560/44000 | Loss: 2.8937 | LR: 5.65e-05 | 1278.2ms/step | 15.6 samples/s +Step 34570/44000 | Loss: 2.9103 | LR: 5.64e-05 | 1280.7ms/step | 15.6 samples/s +Step 34580/44000 | Loss: 2.9036 | LR: 5.63e-05 | 1278.9ms/step | 15.6 samples/s +Step 34590/44000 | Loss: 2.9154 | LR: 5.62e-05 | 1280.2ms/step | 15.6 samples/s +Step 34600/44000 | Loss: 2.9275 | LR: 5.60e-05 | 1280.0ms/step | 15.6 samples/s +Step 34610/44000 | Loss: 2.9293 | LR: 5.59e-05 | 1279.3ms/step | 15.6 samples/s +Step 34620/44000 | Loss: 2.9227 | LR: 5.58e-05 | 1279.4ms/step | 15.6 samples/s +Step 34630/44000 | Loss: 2.9120 | LR: 5.57e-05 | 1286.8ms/step | 15.5 samples/s +Step 34640/44000 | Loss: 2.9195 | LR: 5.56e-05 | 1290.7ms/step | 15.5 samples/s +Step 34650/44000 | Loss: 2.9352 | LR: 5.55e-05 | 1293.8ms/step | 15.5 samples/s +Step 34660/44000 | Loss: 2.9406 | LR: 5.54e-05 | 1287.6ms/step | 15.5 samples/s +Step 34670/44000 | Loss: 2.8914 | LR: 5.52e-05 | 1289.4ms/step | 15.5 samples/s +Step 34680/44000 | Loss: 2.9242 | LR: 5.51e-05 | 1285.2ms/step | 15.6 samples/s +Step 34690/44000 | Loss: 2.9085 | LR: 5.50e-05 | 1285.9ms/step | 15.6 samples/s +Step 34700/44000 | Loss: 2.8989 | LR: 5.49e-05 | 1289.7ms/step | 15.5 samples/s +Step 34710/44000 | Loss: 2.9361 | LR: 5.48e-05 | 1294.8ms/step | 15.4 samples/s +Step 34720/44000 | Loss: 2.9101 | LR: 5.47e-05 | 1290.8ms/step | 15.5 samples/s +Step 34730/44000 | Loss: 2.9366 | LR: 5.46e-05 | 1284.4ms/step | 15.6 samples/s +Step 34740/44000 | Loss: 2.9094 | LR: 5.45e-05 | 1287.2ms/step | 15.5 samples/s +Step 34750/44000 | Loss: 2.9184 | LR: 5.43e-05 | 1289.7ms/step | 15.5 samples/s +Validation at step 34750: Loss = 2.9365, Perplexity = 18.85 +Step 34760/44000 | Loss: 2.9055 | LR: 5.42e-05 | 3460.0ms/step | 5.8 samples/s +Step 34770/44000 | Loss: 2.9110 | LR: 5.41e-05 | 1281.1ms/step | 15.6 samples/s +Step 34780/44000 | Loss: 2.9506 | LR: 5.40e-05 | 1280.6ms/step | 15.6 samples/s +Step 34790/44000 | Loss: 2.9461 | LR: 5.39e-05 | 1280.4ms/step | 15.6 samples/s +Step 34800/44000 | Loss: 2.9269 | LR: 5.38e-05 | 1280.0ms/step | 15.6 samples/s +Step 34810/44000 | Loss: 2.9431 | LR: 5.37e-05 | 1282.8ms/step | 15.6 samples/s +Step 34820/44000 | Loss: 2.9226 | LR: 5.36e-05 | 1282.6ms/step | 15.6 samples/s +Step 34830/44000 | Loss: 2.9213 | LR: 5.34e-05 | 1279.7ms/step | 15.6 samples/s +Step 34840/44000 | Loss: 2.9305 | LR: 5.33e-05 | 1280.8ms/step | 15.6 samples/s +Step 34850/44000 | Loss: 2.9404 | LR: 5.32e-05 | 1285.3ms/step | 15.6 samples/s +Step 34860/44000 | Loss: 2.9303 | LR: 5.31e-05 | 1288.9ms/step | 15.5 samples/s +Step 34870/44000 | Loss: 2.9110 | LR: 5.30e-05 | 1287.4ms/step | 15.5 samples/s +Step 34880/44000 | Loss: 2.9201 | LR: 5.29e-05 | 1282.5ms/step | 15.6 samples/s +Step 34890/44000 | Loss: 2.9164 | LR: 5.28e-05 | 1281.9ms/step | 15.6 samples/s +Step 34900/44000 | Loss: 2.9326 | LR: 5.27e-05 | 1281.6ms/step | 15.6 samples/s +Step 34910/44000 | Loss: 2.9295 | LR: 5.25e-05 | 1279.2ms/step | 15.6 samples/s +Step 34920/44000 | Loss: 2.9162 | LR: 5.24e-05 | 1276.5ms/step | 15.7 samples/s +Step 34930/44000 | Loss: 2.9031 | LR: 5.23e-05 | 1285.6ms/step | 15.6 samples/s +Step 34940/44000 | Loss: 2.8775 | LR: 5.22e-05 | 1285.9ms/step | 15.6 samples/s +Step 34950/44000 | Loss: 2.9343 | LR: 5.21e-05 | 1287.2ms/step | 15.5 samples/s +Step 34960/44000 | Loss: 2.9252 | LR: 5.20e-05 | 1284.6ms/step | 15.6 samples/s +Step 34970/44000 | Loss: 2.9068 | LR: 5.19e-05 | 1286.4ms/step | 15.5 samples/s +Step 34980/44000 | Loss: 2.9292 | LR: 5.18e-05 | 1286.7ms/step | 15.5 samples/s +Step 34990/44000 | Loss: 2.9323 | LR: 5.17e-05 | 1284.4ms/step | 15.6 samples/s +Step 35000/44000 | Loss: 2.9002 | LR: 5.15e-05 | 1283.7ms/step | 15.6 samples/s +Validation at step 35000: Loss = 2.9363, Perplexity = 18.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-35000 +Step 35010/44000 | Loss: 2.9248 | LR: 5.14e-05 | 3749.6ms/step | 5.3 samples/s +Step 35020/44000 | Loss: 2.8940 | LR: 5.13e-05 | 1286.6ms/step | 15.5 samples/s +Step 35030/44000 | Loss: 2.9014 | LR: 5.12e-05 | 1285.6ms/step | 15.6 samples/s +Step 35040/44000 | Loss: 2.9132 | LR: 5.11e-05 | 1284.7ms/step | 15.6 samples/s +Step 35050/44000 | Loss: 2.9041 | LR: 5.10e-05 | 1281.4ms/step | 15.6 samples/s +Step 35060/44000 | Loss: 2.9138 | LR: 5.09e-05 | 1283.6ms/step | 15.6 samples/s +Step 35070/44000 | Loss: 2.8824 | LR: 5.08e-05 | 1283.5ms/step | 15.6 samples/s +Step 35080/44000 | Loss: 2.9182 | LR: 5.07e-05 | 1283.9ms/step | 15.6 samples/s +Step 35090/44000 | Loss: 2.9056 | LR: 5.06e-05 | 1282.7ms/step | 15.6 samples/s +Step 35100/44000 | Loss: 2.9480 | LR: 5.04e-05 | 1283.8ms/step | 15.6 samples/s +Step 35110/44000 | Loss: 2.9105 | LR: 5.03e-05 | 1283.3ms/step | 15.6 samples/s +Step 35120/44000 | Loss: 2.9284 | LR: 5.02e-05 | 1284.0ms/step | 15.6 samples/s +Step 35130/44000 | Loss: 2.9249 | LR: 5.01e-05 | 1283.5ms/step | 15.6 samples/s +Step 35140/44000 | Loss: 2.8933 | LR: 5.00e-05 | 1281.6ms/step | 15.6 samples/s +Step 35150/44000 | Loss: 2.9335 | LR: 4.99e-05 | 1281.4ms/step | 15.6 samples/s +Step 35160/44000 | Loss: 2.9544 | LR: 4.98e-05 | 1282.2ms/step | 15.6 samples/s +Step 35170/44000 | Loss: 2.9313 | LR: 4.97e-05 | 1284.3ms/step | 15.6 samples/s +Step 35180/44000 | Loss: 2.9385 | LR: 4.96e-05 | 1283.3ms/step | 15.6 samples/s +Step 35190/44000 | Loss: 2.9069 | LR: 4.95e-05 | 1284.8ms/step | 15.6 samples/s +Step 35200/44000 | Loss: 2.9370 | LR: 4.94e-05 | 1286.8ms/step | 15.5 samples/s +Step 35210/44000 | Loss: 2.9339 | LR: 4.93e-05 | 1286.1ms/step | 15.6 samples/s +Step 35220/44000 | Loss: 2.9106 | LR: 4.91e-05 | 1287.3ms/step | 15.5 samples/s +Step 35230/44000 | Loss: 2.9192 | LR: 4.90e-05 | 1285.9ms/step | 15.6 samples/s +Step 35240/44000 | Loss: 2.9075 | LR: 4.89e-05 | 1286.9ms/step | 15.5 samples/s +Step 35250/44000 | Loss: 2.9557 | LR: 4.88e-05 | 1283.0ms/step | 15.6 samples/s +Validation at step 35250: Loss = 2.9360, Perplexity = 18.84 +Step 35260/44000 | Loss: 2.9121 | LR: 4.87e-05 | 3429.2ms/step | 5.8 samples/s +Step 35270/44000 | Loss: 2.9434 | LR: 4.86e-05 | 1279.2ms/step | 15.6 samples/s +Step 35280/44000 | Loss: 2.8997 | LR: 4.85e-05 | 1279.2ms/step | 15.6 samples/s +Step 35290/44000 | Loss: 2.9086 | LR: 4.84e-05 | 1280.1ms/step | 15.6 samples/s +Step 35300/44000 | Loss: 2.9099 | LR: 4.83e-05 | 1280.1ms/step | 15.6 samples/s +Step 35310/44000 | Loss: 2.9292 | LR: 4.82e-05 | 1280.0ms/step | 15.6 samples/s +Step 35320/44000 | Loss: 2.8928 | LR: 4.81e-05 | 1280.5ms/step | 15.6 samples/s +Step 35330/44000 | Loss: 2.9260 | LR: 4.80e-05 | 1275.7ms/step | 15.7 samples/s +Step 35340/44000 | Loss: 2.9228 | LR: 4.79e-05 | 1276.5ms/step | 15.7 samples/s +Step 35350/44000 | Loss: 2.9488 | LR: 4.77e-05 | 1281.9ms/step | 15.6 samples/s +Step 35360/44000 | Loss: 2.9052 | LR: 4.76e-05 | 1286.4ms/step | 15.5 samples/s +Step 35370/44000 | Loss: 2.9164 | LR: 4.75e-05 | 1283.9ms/step | 15.6 samples/s +Step 35380/44000 | Loss: 2.9050 | LR: 4.74e-05 | 1283.1ms/step | 15.6 samples/s +Step 35390/44000 | Loss: 2.9103 | LR: 4.73e-05 | 1280.2ms/step | 15.6 samples/s +Step 35400/44000 | Loss: 2.8988 | LR: 4.72e-05 | 1281.3ms/step | 15.6 samples/s +Step 35410/44000 | Loss: 2.9337 | LR: 4.71e-05 | 1281.1ms/step | 15.6 samples/s +Step 35420/44000 | Loss: 2.9036 | LR: 4.70e-05 | 1281.9ms/step | 15.6 samples/s +Step 35430/44000 | Loss: 2.9628 | LR: 4.69e-05 | 1284.2ms/step | 15.6 samples/s +Step 35440/44000 | Loss: 2.9425 | LR: 4.68e-05 | 1281.4ms/step | 15.6 samples/s +Step 35450/44000 | Loss: 2.9250 | LR: 4.67e-05 | 1282.1ms/step | 15.6 samples/s +Step 35460/44000 | Loss: 2.9454 | LR: 4.66e-05 | 1282.3ms/step | 15.6 samples/s +Step 35470/44000 | Loss: 2.9161 | LR: 4.65e-05 | 1281.0ms/step | 15.6 samples/s +Step 35480/44000 | Loss: 2.9255 | LR: 4.64e-05 | 1281.0ms/step | 15.6 samples/s +Step 35490/44000 | Loss: 2.9314 | LR: 4.63e-05 | 1282.5ms/step | 15.6 samples/s +Step 35500/44000 | Loss: 2.9027 | LR: 4.62e-05 | 1281.1ms/step | 15.6 samples/s +Validation at step 35500: Loss = 2.9359, Perplexity = 18.84 +Step 35510/44000 | Loss: 2.9152 | LR: 4.61e-05 | 3433.0ms/step | 5.8 samples/s +Step 35520/44000 | Loss: 2.9231 | LR: 4.59e-05 | 1283.2ms/step | 15.6 samples/s +Step 35530/44000 | Loss: 2.9226 | LR: 4.58e-05 | 1284.0ms/step | 15.6 samples/s +Step 35540/44000 | Loss: 2.9422 | LR: 4.57e-05 | 1281.1ms/step | 15.6 samples/s +Step 35550/44000 | Loss: 2.9440 | LR: 4.56e-05 | 1282.7ms/step | 15.6 samples/s +Step 35560/44000 | Loss: 2.9338 | LR: 4.55e-05 | 1283.3ms/step | 15.6 samples/s +Step 35570/44000 | Loss: 2.9421 | LR: 4.54e-05 | 1284.3ms/step | 15.6 samples/s +Step 35580/44000 | Loss: 2.9259 | LR: 4.53e-05 | 1280.0ms/step | 15.6 samples/s +Step 35590/44000 | Loss: 2.9365 | LR: 4.52e-05 | 1282.9ms/step | 15.6 samples/s +Step 35600/44000 | Loss: 2.9006 | LR: 4.51e-05 | 1281.4ms/step | 15.6 samples/s +Step 35610/44000 | Loss: 2.9388 | LR: 4.50e-05 | 1286.7ms/step | 15.5 samples/s +Step 35620/44000 | Loss: 2.9162 | LR: 4.49e-05 | 1285.3ms/step | 15.6 samples/s +Step 35630/44000 | Loss: 2.9268 | LR: 4.48e-05 | 1287.7ms/step | 15.5 samples/s +Step 35640/44000 | Loss: 2.9043 | LR: 4.47e-05 | 1284.5ms/step | 15.6 samples/s +Step 35650/44000 | Loss: 2.9124 | LR: 4.46e-05 | 1281.7ms/step | 15.6 samples/s +Step 35660/44000 | Loss: 2.9265 | LR: 4.45e-05 | 1287.2ms/step | 15.5 samples/s +Step 35670/44000 | Loss: 2.9173 | LR: 4.44e-05 | 1284.8ms/step | 15.6 samples/s +Step 35680/44000 | Loss: 2.9206 | LR: 4.43e-05 | 1286.0ms/step | 15.6 samples/s +Step 35690/44000 | Loss: 2.9312 | LR: 4.42e-05 | 1284.9ms/step | 15.6 samples/s +Step 35700/44000 | Loss: 2.9082 | LR: 4.41e-05 | 1288.0ms/step | 15.5 samples/s +Step 35710/44000 | Loss: 2.9240 | LR: 4.40e-05 | 1286.2ms/step | 15.5 samples/s +Step 35720/44000 | Loss: 2.9050 | LR: 4.39e-05 | 1285.0ms/step | 15.6 samples/s +Step 35730/44000 | Loss: 2.8968 | LR: 4.38e-05 | 1281.7ms/step | 15.6 samples/s +Step 35740/44000 | Loss: 2.9288 | LR: 4.37e-05 | 1290.6ms/step | 15.5 samples/s +Step 35750/44000 | Loss: 2.9090 | LR: 4.36e-05 | 1285.7ms/step | 15.6 samples/s +Validation at step 35750: Loss = 2.9356, Perplexity = 18.83 +Step 35760/44000 | Loss: 2.9394 | LR: 4.35e-05 | 3433.1ms/step | 5.8 samples/s +Step 35770/44000 | Loss: 2.9135 | LR: 4.34e-05 | 1288.9ms/step | 15.5 samples/s +Step 35780/44000 | Loss: 2.9036 | LR: 4.33e-05 | 1282.6ms/step | 15.6 samples/s +Step 35790/44000 | Loss: 2.9279 | LR: 4.32e-05 | 1286.4ms/step | 15.5 samples/s +Step 35800/44000 | Loss: 2.9378 | LR: 4.31e-05 | 1282.5ms/step | 15.6 samples/s +Step 35810/44000 | Loss: 2.9340 | LR: 4.29e-05 | 1282.2ms/step | 15.6 samples/s +Step 35820/44000 | Loss: 2.9354 | LR: 4.28e-05 | 1284.0ms/step | 15.6 samples/s +Step 35830/44000 | Loss: 2.9547 | LR: 4.27e-05 | 1285.6ms/step | 15.6 samples/s +Step 35840/44000 | Loss: 2.9144 | LR: 4.26e-05 | 1287.3ms/step | 15.5 samples/s +Step 35850/44000 | Loss: 2.9236 | LR: 4.25e-05 | 1287.9ms/step | 15.5 samples/s +Step 35860/44000 | Loss: 2.9471 | LR: 4.24e-05 | 1281.3ms/step | 15.6 samples/s +Step 35870/44000 | Loss: 2.9229 | LR: 4.23e-05 | 1281.8ms/step | 15.6 samples/s +Step 35880/44000 | Loss: 2.9404 | LR: 4.22e-05 | 1279.3ms/step | 15.6 samples/s +Step 35890/44000 | Loss: 2.9043 | LR: 4.21e-05 | 1280.4ms/step | 15.6 samples/s +Step 35900/44000 | Loss: 2.9261 | LR: 4.20e-05 | 1280.2ms/step | 15.6 samples/s +Step 35910/44000 | Loss: 2.9096 | LR: 4.19e-05 | 1279.2ms/step | 15.6 samples/s +Step 35920/44000 | Loss: 2.9217 | LR: 4.18e-05 | 1282.9ms/step | 15.6 samples/s +Step 35930/44000 | Loss: 2.9107 | LR: 4.17e-05 | 1281.6ms/step | 15.6 samples/s +Step 35940/44000 | Loss: 2.9114 | LR: 4.16e-05 | 1282.0ms/step | 15.6 samples/s +Step 35950/44000 | Loss: 2.8994 | LR: 4.15e-05 | 1281.9ms/step | 15.6 samples/s +Step 35960/44000 | Loss: 2.9034 | LR: 4.14e-05 | 1282.2ms/step | 15.6 samples/s +Step 35970/44000 | Loss: 2.8959 | LR: 4.13e-05 | 1284.3ms/step | 15.6 samples/s +Step 35980/44000 | Loss: 2.9488 | LR: 4.12e-05 | 1283.4ms/step | 15.6 samples/s +Step 35990/44000 | Loss: 2.9195 | LR: 4.11e-05 | 1284.8ms/step | 15.6 samples/s +Step 36000/44000 | Loss: 2.9193 | LR: 4.10e-05 | 1286.3ms/step | 15.5 samples/s +Validation at step 36000: Loss = 2.9355, Perplexity = 18.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-36000 +Step 36010/44000 | Loss: 2.9357 | LR: 4.09e-05 | 3726.9ms/step | 5.4 samples/s +Step 36020/44000 | Loss: 2.9132 | LR: 4.08e-05 | 1282.2ms/step | 15.6 samples/s +Step 36030/44000 | Loss: 2.9013 | LR: 4.07e-05 | 1288.6ms/step | 15.5 samples/s +Step 36040/44000 | Loss: 2.9084 | LR: 4.06e-05 | 1283.6ms/step | 15.6 samples/s +Step 36050/44000 | Loss: 2.9057 | LR: 4.05e-05 | 1288.4ms/step | 15.5 samples/s +Step 36060/44000 | Loss: 2.9342 | LR: 4.04e-05 | 1295.3ms/step | 15.4 samples/s +Step 36070/44000 | Loss: 2.9245 | LR: 4.03e-05 | 1289.8ms/step | 15.5 samples/s +Step 36080/44000 | Loss: 2.9389 | LR: 4.02e-05 | 1284.5ms/step | 15.6 samples/s +Step 36090/44000 | Loss: 2.9456 | LR: 4.01e-05 | 1285.7ms/step | 15.6 samples/s +Step 36100/44000 | Loss: 2.9237 | LR: 4.00e-05 | 1287.1ms/step | 15.5 samples/s +Step 36110/44000 | Loss: 2.9394 | LR: 3.99e-05 | 1289.2ms/step | 15.5 samples/s +Step 36120/44000 | Loss: 2.9258 | LR: 3.98e-05 | 1288.6ms/step | 15.5 samples/s +Step 36130/44000 | Loss: 2.9279 | LR: 3.97e-05 | 1292.8ms/step | 15.5 samples/s +Step 36140/44000 | Loss: 2.9206 | LR: 3.97e-05 | 1290.5ms/step | 15.5 samples/s +Step 36150/44000 | Loss: 2.9305 | LR: 3.96e-05 | 1282.9ms/step | 15.6 samples/s +Step 36160/44000 | Loss: 2.9223 | LR: 3.95e-05 | 1284.0ms/step | 15.6 samples/s +Step 36170/44000 | Loss: 2.8812 | LR: 3.94e-05 | 1288.2ms/step | 15.5 samples/s +Step 36180/44000 | Loss: 2.9221 | LR: 3.93e-05 | 1285.3ms/step | 15.6 samples/s +Step 36190/44000 | Loss: 2.9184 | LR: 3.92e-05 | 1284.3ms/step | 15.6 samples/s +Step 36200/44000 | Loss: 2.9120 | LR: 3.91e-05 | 1285.5ms/step | 15.6 samples/s +Step 36210/44000 | Loss: 2.9204 | LR: 3.90e-05 | 1284.0ms/step | 15.6 samples/s +Step 36220/44000 | Loss: 2.9155 | LR: 3.89e-05 | 1283.7ms/step | 15.6 samples/s +Step 36230/44000 | Loss: 2.9168 | LR: 3.88e-05 | 1284.8ms/step | 15.6 samples/s +Step 36240/44000 | Loss: 2.9265 | LR: 3.87e-05 | 1283.5ms/step | 15.6 samples/s +Step 36250/44000 | Loss: 2.9420 | LR: 3.86e-05 | 1281.6ms/step | 15.6 samples/s +Validation at step 36250: Loss = 2.9353, Perplexity = 18.83 +Step 36260/44000 | Loss: 2.9326 | LR: 3.85e-05 | 3426.1ms/step | 5.8 samples/s +Step 36270/44000 | Loss: 2.9425 | LR: 3.84e-05 | 1282.7ms/step | 15.6 samples/s +Step 36280/44000 | Loss: 2.9142 | LR: 3.83e-05 | 1284.3ms/step | 15.6 samples/s +Step 36290/44000 | Loss: 2.9292 | LR: 3.82e-05 | 1285.3ms/step | 15.6 samples/s +Step 36300/44000 | Loss: 2.9135 | LR: 3.81e-05 | 1283.5ms/step | 15.6 samples/s +Step 36310/44000 | Loss: 2.9154 | LR: 3.80e-05 | 1282.1ms/step | 15.6 samples/s +Step 36320/44000 | Loss: 2.8916 | LR: 3.79e-05 | 1283.3ms/step | 15.6 samples/s +Step 36330/44000 | Loss: 2.9087 | LR: 3.78e-05 | 1285.7ms/step | 15.6 samples/s +Step 36340/44000 | Loss: 2.9266 | LR: 3.77e-05 | 1279.7ms/step | 15.6 samples/s +Step 36350/44000 | Loss: 2.8974 | LR: 3.76e-05 | 1279.2ms/step | 15.6 samples/s +Step 36360/44000 | Loss: 2.9058 | LR: 3.75e-05 | 1280.9ms/step | 15.6 samples/s +Step 36370/44000 | Loss: 2.9163 | LR: 3.74e-05 | 1279.1ms/step | 15.6 samples/s +Step 36380/44000 | Loss: 2.9150 | LR: 3.73e-05 | 1278.8ms/step | 15.6 samples/s +Step 36390/44000 | Loss: 2.9371 | LR: 3.72e-05 | 1279.5ms/step | 15.6 samples/s +Step 36400/44000 | Loss: 2.9145 | LR: 3.71e-05 | 1279.6ms/step | 15.6 samples/s +Step 36410/44000 | Loss: 2.9252 | LR: 3.70e-05 | 1280.7ms/step | 15.6 samples/s +Step 36420/44000 | Loss: 2.8931 | LR: 3.69e-05 | 1279.6ms/step | 15.6 samples/s +Step 36430/44000 | Loss: 2.9010 | LR: 3.69e-05 | 1283.3ms/step | 15.6 samples/s +Step 36440/44000 | Loss: 2.9103 | LR: 3.68e-05 | 1282.8ms/step | 15.6 samples/s +Step 36450/44000 | Loss: 2.9508 | LR: 3.67e-05 | 1281.8ms/step | 15.6 samples/s +Step 36460/44000 | Loss: 2.9258 | LR: 3.66e-05 | 1280.2ms/step | 15.6 samples/s +Step 36470/44000 | Loss: 2.8996 | LR: 3.65e-05 | 1279.9ms/step | 15.6 samples/s +Step 36480/44000 | Loss: 2.9239 | LR: 3.64e-05 | 1279.5ms/step | 15.6 samples/s +Step 36490/44000 | Loss: 2.9450 | LR: 3.63e-05 | 1279.9ms/step | 15.6 samples/s +Step 36500/44000 | Loss: 2.8938 | LR: 3.62e-05 | 1283.4ms/step | 15.6 samples/s +Validation at step 36500: Loss = 2.9352, Perplexity = 18.82 +Step 36510/44000 | Loss: 2.9344 | LR: 3.61e-05 | 3433.5ms/step | 5.8 samples/s +Step 36520/44000 | Loss: 2.9084 | LR: 3.60e-05 | 1284.7ms/step | 15.6 samples/s +Step 36530/44000 | Loss: 2.9156 | LR: 3.59e-05 | 1282.9ms/step | 15.6 samples/s +Step 36540/44000 | Loss: 2.9047 | LR: 3.58e-05 | 1284.5ms/step | 15.6 samples/s +Step 36550/44000 | Loss: 2.9117 | LR: 3.57e-05 | 1284.0ms/step | 15.6 samples/s +Step 36560/44000 | Loss: 2.9331 | LR: 3.56e-05 | 1285.9ms/step | 15.6 samples/s +Step 36570/44000 | Loss: 2.8936 | LR: 3.55e-05 | 1285.6ms/step | 15.6 samples/s +Step 36580/44000 | Loss: 2.9099 | LR: 3.54e-05 | 1285.0ms/step | 15.6 samples/s +Step 36590/44000 | Loss: 2.9210 | LR: 3.53e-05 | 1284.5ms/step | 15.6 samples/s +Step 36600/44000 | Loss: 2.8936 | LR: 3.53e-05 | 1284.0ms/step | 15.6 samples/s +Step 36610/44000 | Loss: 2.9167 | LR: 3.52e-05 | 1284.2ms/step | 15.6 samples/s +Step 36620/44000 | Loss: 2.9067 | LR: 3.51e-05 | 1285.0ms/step | 15.6 samples/s +Step 36630/44000 | Loss: 2.9212 | LR: 3.50e-05 | 1284.6ms/step | 15.6 samples/s +Step 36640/44000 | Loss: 2.9190 | LR: 3.49e-05 | 1284.5ms/step | 15.6 samples/s +Step 36650/44000 | Loss: 2.9152 | LR: 3.48e-05 | 1285.7ms/step | 15.6 samples/s +Step 36660/44000 | Loss: 2.9082 | LR: 3.47e-05 | 1284.2ms/step | 15.6 samples/s +Step 36670/44000 | Loss: 2.9478 | LR: 3.46e-05 | 1283.7ms/step | 15.6 samples/s +Step 36680/44000 | Loss: 2.9348 | LR: 3.45e-05 | 1281.1ms/step | 15.6 samples/s +Step 36690/44000 | Loss: 2.9116 | LR: 3.44e-05 | 1281.2ms/step | 15.6 samples/s +Step 36700/44000 | Loss: 2.9131 | LR: 3.43e-05 | 1284.5ms/step | 15.6 samples/s +Step 36710/44000 | Loss: 2.9046 | LR: 3.42e-05 | 1286.1ms/step | 15.6 samples/s +Step 36720/44000 | Loss: 2.9171 | LR: 3.41e-05 | 1284.2ms/step | 15.6 samples/s +Step 36730/44000 | Loss: 2.9080 | LR: 3.41e-05 | 1285.5ms/step | 15.6 samples/s +Step 36740/44000 | Loss: 2.9321 | LR: 3.40e-05 | 1285.8ms/step | 15.6 samples/s +Step 36750/44000 | Loss: 2.9013 | LR: 3.39e-05 | 1284.7ms/step | 15.6 samples/s +Validation at step 36750: Loss = 2.9351, Perplexity = 18.82 +Step 36760/44000 | Loss: 2.9097 | LR: 3.38e-05 | 3432.7ms/step | 5.8 samples/s +Step 36770/44000 | Loss: 2.9212 | LR: 3.37e-05 | 1283.8ms/step | 15.6 samples/s +Step 36780/44000 | Loss: 2.9305 | LR: 3.36e-05 | 1282.3ms/step | 15.6 samples/s +Step 36790/44000 | Loss: 2.9274 | LR: 3.35e-05 | 1283.4ms/step | 15.6 samples/s +Step 36800/44000 | Loss: 2.9192 | LR: 3.34e-05 | 1284.5ms/step | 15.6 samples/s +Step 36810/44000 | Loss: 2.9145 | LR: 3.33e-05 | 1286.1ms/step | 15.6 samples/s +Step 36820/44000 | Loss: 2.9143 | LR: 3.32e-05 | 1286.1ms/step | 15.6 samples/s +Step 36830/44000 | Loss: 2.9151 | LR: 3.31e-05 | 1285.7ms/step | 15.6 samples/s +Step 36840/44000 | Loss: 2.9199 | LR: 3.31e-05 | 1283.9ms/step | 15.6 samples/s +Step 36850/44000 | Loss: 2.9212 | LR: 3.30e-05 | 1282.9ms/step | 15.6 samples/s +Step 36860/44000 | Loss: 2.9021 | LR: 3.29e-05 | 1282.6ms/step | 15.6 samples/s +Step 36870/44000 | Loss: 2.9221 | LR: 3.28e-05 | 1283.0ms/step | 15.6 samples/s +Step 36880/44000 | Loss: 2.8951 | LR: 3.27e-05 | 1283.9ms/step | 15.6 samples/s +Step 36890/44000 | Loss: 2.8920 | LR: 3.26e-05 | 1283.0ms/step | 15.6 samples/s +Step 36900/44000 | Loss: 2.8967 | LR: 3.25e-05 | 1282.8ms/step | 15.6 samples/s +Step 36910/44000 | Loss: 2.9127 | LR: 3.24e-05 | 1281.1ms/step | 15.6 samples/s +Step 36920/44000 | Loss: 2.9117 | LR: 3.23e-05 | 1282.7ms/step | 15.6 samples/s +Step 36930/44000 | Loss: 2.8901 | LR: 3.22e-05 | 1283.8ms/step | 15.6 samples/s +Step 36940/44000 | Loss: 2.9298 | LR: 3.22e-05 | 1285.0ms/step | 15.6 samples/s +Step 36950/44000 | Loss: 2.9246 | LR: 3.21e-05 | 1284.1ms/step | 15.6 samples/s +Step 36960/44000 | Loss: 2.9083 | LR: 3.20e-05 | 1284.0ms/step | 15.6 samples/s +Step 36970/44000 | Loss: 2.8899 | LR: 3.19e-05 | 1288.1ms/step | 15.5 samples/s +Step 36980/44000 | Loss: 2.9119 | LR: 3.18e-05 | 1286.6ms/step | 15.5 samples/s +Step 36990/44000 | Loss: 2.9242 | LR: 3.17e-05 | 1289.0ms/step | 15.5 samples/s +Step 37000/44000 | Loss: 2.9336 | LR: 3.16e-05 | 1290.3ms/step | 15.5 samples/s +Validation at step 37000: Loss = 2.9349, Perplexity = 18.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-37000 +Step 37010/44000 | Loss: 2.9306 | LR: 3.15e-05 | 3800.0ms/step | 5.3 samples/s +Step 37020/44000 | Loss: 2.9693 | LR: 3.15e-05 | 1294.3ms/step | 15.5 samples/s +Step 37030/44000 | Loss: 2.9348 | LR: 3.14e-05 | 1292.1ms/step | 15.5 samples/s +Step 37040/44000 | Loss: 2.9312 | LR: 3.13e-05 | 1290.8ms/step | 15.5 samples/s +Step 37050/44000 | Loss: 2.9226 | LR: 3.12e-05 | 1290.7ms/step | 15.5 samples/s +Step 37060/44000 | Loss: 2.8915 | LR: 3.11e-05 | 1284.6ms/step | 15.6 samples/s +Step 37070/44000 | Loss: 2.9221 | LR: 3.10e-05 | 1283.9ms/step | 15.6 samples/s +Step 37080/44000 | Loss: 2.9522 | LR: 3.09e-05 | 1284.2ms/step | 15.6 samples/s +Step 37090/44000 | Loss: 2.9311 | LR: 3.08e-05 | 1285.9ms/step | 15.6 samples/s +Step 37100/44000 | Loss: 2.9146 | LR: 3.07e-05 | 1284.5ms/step | 15.6 samples/s +Step 37110/44000 | Loss: 2.9091 | LR: 3.07e-05 | 1285.2ms/step | 15.6 samples/s +Step 37120/44000 | Loss: 2.9321 | LR: 3.06e-05 | 1283.3ms/step | 15.6 samples/s +Step 37130/44000 | Loss: 2.9232 | LR: 3.05e-05 | 1282.7ms/step | 15.6 samples/s +Step 37140/44000 | Loss: 2.9159 | LR: 3.04e-05 | 1284.5ms/step | 15.6 samples/s +Step 37150/44000 | Loss: 2.8788 | LR: 3.03e-05 | 1281.9ms/step | 15.6 samples/s +Step 37160/44000 | Loss: 2.9281 | LR: 3.02e-05 | 1280.9ms/step | 15.6 samples/s +Step 37170/44000 | Loss: 2.9597 | LR: 3.01e-05 | 1284.0ms/step | 15.6 samples/s +Step 37180/44000 | Loss: 2.9127 | LR: 3.01e-05 | 1283.2ms/step | 15.6 samples/s +Step 37190/44000 | Loss: 2.9168 | LR: 3.00e-05 | 1280.8ms/step | 15.6 samples/s +Step 37200/44000 | Loss: 2.9220 | LR: 2.99e-05 | 1280.0ms/step | 15.6 samples/s +Step 37210/44000 | Loss: 2.9356 | LR: 2.98e-05 | 1282.1ms/step | 15.6 samples/s +Step 37220/44000 | Loss: 2.9161 | LR: 2.97e-05 | 1281.9ms/step | 15.6 samples/s +Step 37230/44000 | Loss: 2.9444 | LR: 2.96e-05 | 1281.0ms/step | 15.6 samples/s +Step 37240/44000 | Loss: 2.9206 | LR: 2.95e-05 | 1278.5ms/step | 15.6 samples/s +Step 37250/44000 | Loss: 2.9124 | LR: 2.95e-05 | 1280.1ms/step | 15.6 samples/s +Validation at step 37250: Loss = 2.9348, Perplexity = 18.82 +Step 37260/44000 | Loss: 2.9262 | LR: 2.94e-05 | 3426.3ms/step | 5.8 samples/s +Step 37270/44000 | Loss: 2.9378 | LR: 2.93e-05 | 1281.2ms/step | 15.6 samples/s +Step 37280/44000 | Loss: 2.9179 | LR: 2.92e-05 | 1283.1ms/step | 15.6 samples/s +Step 37290/44000 | Loss: 2.9273 | LR: 2.91e-05 | 1283.5ms/step | 15.6 samples/s +Step 37300/44000 | Loss: 2.9129 | LR: 2.90e-05 | 1280.4ms/step | 15.6 samples/s +Step 37310/44000 | Loss: 2.9163 | LR: 2.89e-05 | 1279.4ms/step | 15.6 samples/s +Step 37320/44000 | Loss: 2.9205 | LR: 2.89e-05 | 1281.3ms/step | 15.6 samples/s +Step 37330/44000 | Loss: 2.9449 | LR: 2.88e-05 | 1282.0ms/step | 15.6 samples/s +Step 37340/44000 | Loss: 2.9172 | LR: 2.87e-05 | 1281.3ms/step | 15.6 samples/s +Step 37350/44000 | Loss: 2.9356 | LR: 2.86e-05 | 1281.7ms/step | 15.6 samples/s +Step 37360/44000 | Loss: 2.9213 | LR: 2.85e-05 | 1280.5ms/step | 15.6 samples/s +Step 37370/44000 | Loss: 2.9228 | LR: 2.84e-05 | 1280.8ms/step | 15.6 samples/s +Step 37380/44000 | Loss: 2.9060 | LR: 2.84e-05 | 1280.2ms/step | 15.6 samples/s +Step 37390/44000 | Loss: 2.9036 | LR: 2.83e-05 | 1280.4ms/step | 15.6 samples/s +Step 37400/44000 | Loss: 2.9083 | LR: 2.82e-05 | 1282.6ms/step | 15.6 samples/s +Step 37410/44000 | Loss: 2.9034 | LR: 2.81e-05 | 1285.9ms/step | 15.6 samples/s +Step 37420/44000 | Loss: 2.9337 | LR: 2.80e-05 | 1282.3ms/step | 15.6 samples/s +Step 37430/44000 | Loss: 2.8944 | LR: 2.79e-05 | 1285.9ms/step | 15.6 samples/s +Step 37440/44000 | Loss: 2.9073 | LR: 2.78e-05 | 1283.4ms/step | 15.6 samples/s +Step 37450/44000 | Loss: 2.9211 | LR: 2.78e-05 | 1285.9ms/step | 15.6 samples/s +Step 37460/44000 | Loss: 2.9344 | LR: 2.77e-05 | 1286.7ms/step | 15.5 samples/s +Step 37470/44000 | Loss: 2.9185 | LR: 2.76e-05 | 1281.0ms/step | 15.6 samples/s +Step 37480/44000 | Loss: 2.9294 | LR: 2.75e-05 | 1282.9ms/step | 15.6 samples/s +Step 37490/44000 | Loss: 2.9142 | LR: 2.74e-05 | 1287.0ms/step | 15.5 samples/s +Step 37500/44000 | Loss: 2.9185 | LR: 2.74e-05 | 1291.5ms/step | 15.5 samples/s +Validation at step 37500: Loss = 2.9348, Perplexity = 18.82 +Step 37510/44000 | Loss: 2.9392 | LR: 2.73e-05 | 3434.3ms/step | 5.8 samples/s +Step 37520/44000 | Loss: 2.8930 | LR: 2.72e-05 | 1285.3ms/step | 15.6 samples/s +Step 37530/44000 | Loss: 2.9580 | LR: 2.71e-05 | 1283.8ms/step | 15.6 samples/s +Step 37540/44000 | Loss: 2.9274 | LR: 2.70e-05 | 1286.9ms/step | 15.5 samples/s +Step 37550/44000 | Loss: 2.9203 | LR: 2.69e-05 | 1287.1ms/step | 15.5 samples/s +Step 37560/44000 | Loss: 2.9385 | LR: 2.69e-05 | 1284.7ms/step | 15.6 samples/s +Step 37570/44000 | Loss: 2.9190 | LR: 2.68e-05 | 1286.0ms/step | 15.6 samples/s +Step 37580/44000 | Loss: 2.9101 | LR: 2.67e-05 | 1285.1ms/step | 15.6 samples/s +Step 37590/44000 | Loss: 2.9117 | LR: 2.66e-05 | 1282.1ms/step | 15.6 samples/s +Step 37600/44000 | Loss: 2.9175 | LR: 2.65e-05 | 1285.1ms/step | 15.6 samples/s +Step 37610/44000 | Loss: 2.9073 | LR: 2.65e-05 | 1283.7ms/step | 15.6 samples/s +Step 37620/44000 | Loss: 2.9307 | LR: 2.64e-05 | 1283.9ms/step | 15.6 samples/s +Step 37630/44000 | Loss: 2.8857 | LR: 2.63e-05 | 1285.7ms/step | 15.6 samples/s +Step 37640/44000 | Loss: 2.9354 | LR: 2.62e-05 | 1286.0ms/step | 15.6 samples/s +Step 37650/44000 | Loss: 2.9055 | LR: 2.61e-05 | 1286.8ms/step | 15.5 samples/s +Step 37660/44000 | Loss: 2.9445 | LR: 2.60e-05 | 1285.8ms/step | 15.6 samples/s +Step 37670/44000 | Loss: 2.9252 | LR: 2.60e-05 | 1284.2ms/step | 15.6 samples/s +Step 37680/44000 | Loss: 2.9370 | LR: 2.59e-05 | 1283.6ms/step | 15.6 samples/s +Step 37690/44000 | Loss: 2.9120 | LR: 2.58e-05 | 1284.4ms/step | 15.6 samples/s +Step 37700/44000 | Loss: 2.9098 | LR: 2.57e-05 | 1282.7ms/step | 15.6 samples/s +Step 37710/44000 | Loss: 2.9013 | LR: 2.56e-05 | 1284.5ms/step | 15.6 samples/s +Step 37720/44000 | Loss: 2.9359 | LR: 2.56e-05 | 1285.7ms/step | 15.6 samples/s +Step 37730/44000 | Loss: 2.9079 | LR: 2.55e-05 | 1285.8ms/step | 15.6 samples/s +Step 37740/44000 | Loss: 2.9195 | LR: 2.54e-05 | 1284.3ms/step | 15.6 samples/s +Step 37750/44000 | Loss: 2.9033 | LR: 2.53e-05 | 1285.8ms/step | 15.6 samples/s +Validation at step 37750: Loss = 2.9347, Perplexity = 18.82 +Step 37760/44000 | Loss: 2.9254 | LR: 2.52e-05 | 3428.2ms/step | 5.8 samples/s +Step 37770/44000 | Loss: 2.8999 | LR: 2.52e-05 | 1284.8ms/step | 15.6 samples/s +Step 37780/44000 | Loss: 2.9056 | LR: 2.51e-05 | 1283.2ms/step | 15.6 samples/s +Step 37790/44000 | Loss: 2.9398 | LR: 2.50e-05 | 1284.1ms/step | 15.6 samples/s +Step 37800/44000 | Loss: 2.9106 | LR: 2.49e-05 | 1284.5ms/step | 15.6 samples/s +Step 37810/44000 | Loss: 2.8833 | LR: 2.48e-05 | 1283.4ms/step | 15.6 samples/s +Step 37820/44000 | Loss: 2.9428 | LR: 2.48e-05 | 1282.4ms/step | 15.6 samples/s +Step 37830/44000 | Loss: 2.9415 | LR: 2.47e-05 | 1282.9ms/step | 15.6 samples/s +Step 37840/44000 | Loss: 2.9265 | LR: 2.46e-05 | 1282.6ms/step | 15.6 samples/s +Step 37850/44000 | Loss: 2.9074 | LR: 2.45e-05 | 1283.0ms/step | 15.6 samples/s +Step 37860/44000 | Loss: 2.8859 | LR: 2.45e-05 | 1283.5ms/step | 15.6 samples/s +Step 37870/44000 | Loss: 2.9109 | LR: 2.44e-05 | 1287.6ms/step | 15.5 samples/s +Step 37880/44000 | Loss: 2.9081 | LR: 2.43e-05 | 1291.6ms/step | 15.5 samples/s +Step 37890/44000 | Loss: 2.9305 | LR: 2.42e-05 | 1289.0ms/step | 15.5 samples/s +Step 37900/44000 | Loss: 2.9340 | LR: 2.41e-05 | 1285.1ms/step | 15.6 samples/s +Step 37910/44000 | Loss: 2.9186 | LR: 2.41e-05 | 1283.3ms/step | 15.6 samples/s +Step 37920/44000 | Loss: 2.9385 | LR: 2.40e-05 | 1283.3ms/step | 15.6 samples/s +Step 37930/44000 | Loss: 2.9156 | LR: 2.39e-05 | 1283.7ms/step | 15.6 samples/s +Step 37940/44000 | Loss: 2.9263 | LR: 2.38e-05 | 1281.5ms/step | 15.6 samples/s +Step 37950/44000 | Loss: 2.9412 | LR: 2.38e-05 | 1279.6ms/step | 15.6 samples/s +Step 37960/44000 | Loss: 2.9579 | LR: 2.37e-05 | 1282.0ms/step | 15.6 samples/s +Step 37970/44000 | Loss: 2.9170 | LR: 2.36e-05 | 1284.8ms/step | 15.6 samples/s +Step 37980/44000 | Loss: 2.9480 | LR: 2.35e-05 | 1282.3ms/step | 15.6 samples/s +Step 37990/44000 | Loss: 2.9157 | LR: 2.34e-05 | 1284.2ms/step | 15.6 samples/s +Step 38000/44000 | Loss: 2.9204 | LR: 2.34e-05 | 1283.6ms/step | 15.6 samples/s +Validation at step 38000: Loss = 2.9346, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-38000 +Step 38010/44000 | Loss: 2.8718 | LR: 2.33e-05 | 3729.5ms/step | 5.4 samples/s +Step 38020/44000 | Loss: 2.9046 | LR: 2.32e-05 | 1284.4ms/step | 15.6 samples/s +Step 38030/44000 | Loss: 2.8731 | LR: 2.31e-05 | 1283.1ms/step | 15.6 samples/s +Step 38040/44000 | Loss: 2.9437 | LR: 2.31e-05 | 1283.6ms/step | 15.6 samples/s +Step 38050/44000 | Loss: 2.9277 | LR: 2.30e-05 | 1281.1ms/step | 15.6 samples/s +Step 38060/44000 | Loss: 2.9319 | LR: 2.29e-05 | 1282.9ms/step | 15.6 samples/s +Step 38070/44000 | Loss: 2.9051 | LR: 2.28e-05 | 1282.5ms/step | 15.6 samples/s +Step 38080/44000 | Loss: 2.9186 | LR: 2.28e-05 | 1285.5ms/step | 15.6 samples/s +Step 38090/44000 | Loss: 2.9301 | LR: 2.27e-05 | 1283.3ms/step | 15.6 samples/s +Step 38100/44000 | Loss: 2.9137 | LR: 2.26e-05 | 1285.2ms/step | 15.6 samples/s +Step 38110/44000 | Loss: 2.9252 | LR: 2.25e-05 | 1285.6ms/step | 15.6 samples/s +Step 38120/44000 | Loss: 2.9361 | LR: 2.25e-05 | 1278.7ms/step | 15.6 samples/s +Step 38130/44000 | Loss: 2.9126 | LR: 2.24e-05 | 1280.2ms/step | 15.6 samples/s +Step 38140/44000 | Loss: 2.9371 | LR: 2.23e-05 | 1278.9ms/step | 15.6 samples/s +Step 38150/44000 | Loss: 2.9563 | LR: 2.22e-05 | 1280.6ms/step | 15.6 samples/s +Step 38160/44000 | Loss: 2.9611 | LR: 2.22e-05 | 1278.3ms/step | 15.6 samples/s +Step 38170/44000 | Loss: 2.9242 | LR: 2.21e-05 | 1280.2ms/step | 15.6 samples/s +Step 38180/44000 | Loss: 2.9517 | LR: 2.20e-05 | 1281.9ms/step | 15.6 samples/s +Step 38190/44000 | Loss: 2.8945 | LR: 2.19e-05 | 1282.6ms/step | 15.6 samples/s +Step 38200/44000 | Loss: 2.9242 | LR: 2.19e-05 | 1283.8ms/step | 15.6 samples/s +Step 38210/44000 | Loss: 2.9232 | LR: 2.18e-05 | 1285.5ms/step | 15.6 samples/s +Step 38220/44000 | Loss: 2.9408 | LR: 2.17e-05 | 1284.5ms/step | 15.6 samples/s +Step 38230/44000 | Loss: 2.9274 | LR: 2.16e-05 | 1283.0ms/step | 15.6 samples/s +Step 38240/44000 | Loss: 2.9279 | LR: 2.16e-05 | 1283.0ms/step | 15.6 samples/s +Step 38250/44000 | Loss: 2.9281 | LR: 2.15e-05 | 1283.3ms/step | 15.6 samples/s +Validation at step 38250: Loss = 2.9346, Perplexity = 18.81 +Step 38260/44000 | Loss: 2.9012 | LR: 2.14e-05 | 3430.6ms/step | 5.8 samples/s +Step 38270/44000 | Loss: 2.9322 | LR: 2.13e-05 | 1283.9ms/step | 15.6 samples/s +Step 38280/44000 | Loss: 2.8912 | LR: 2.13e-05 | 1284.0ms/step | 15.6 samples/s +Step 38290/44000 | Loss: 2.8980 | LR: 2.12e-05 | 1285.3ms/step | 15.6 samples/s +Step 38300/44000 | Loss: 2.9334 | LR: 2.11e-05 | 1285.1ms/step | 15.6 samples/s +Step 38310/44000 | Loss: 2.9047 | LR: 2.11e-05 | 1285.5ms/step | 15.6 samples/s +Step 38320/44000 | Loss: 2.9193 | LR: 2.10e-05 | 1294.1ms/step | 15.5 samples/s +Step 38330/44000 | Loss: 2.9392 | LR: 2.09e-05 | 1294.1ms/step | 15.5 samples/s +Step 38340/44000 | Loss: 2.8875 | LR: 2.08e-05 | 1297.4ms/step | 15.4 samples/s +Step 38350/44000 | Loss: 2.9177 | LR: 2.08e-05 | 1293.5ms/step | 15.5 samples/s +Step 38360/44000 | Loss: 2.9240 | LR: 2.07e-05 | 1283.0ms/step | 15.6 samples/s +Step 38370/44000 | Loss: 2.9327 | LR: 2.06e-05 | 1285.9ms/step | 15.6 samples/s +Step 38380/44000 | Loss: 2.9357 | LR: 2.05e-05 | 1288.2ms/step | 15.5 samples/s +Step 38390/44000 | Loss: 2.9150 | LR: 2.05e-05 | 1284.5ms/step | 15.6 samples/s +Step 38400/44000 | Loss: 2.9358 | LR: 2.04e-05 | 1282.5ms/step | 15.6 samples/s +Step 38410/44000 | Loss: 2.9177 | LR: 2.03e-05 | 1285.5ms/step | 15.6 samples/s +Step 38420/44000 | Loss: 2.9280 | LR: 2.03e-05 | 1283.0ms/step | 15.6 samples/s +Step 38430/44000 | Loss: 2.9320 | LR: 2.02e-05 | 1282.0ms/step | 15.6 samples/s +Step 38440/44000 | Loss: 2.8858 | LR: 2.01e-05 | 1282.0ms/step | 15.6 samples/s +Step 38450/44000 | Loss: 2.9217 | LR: 2.00e-05 | 1279.7ms/step | 15.6 samples/s +Step 38460/44000 | Loss: 2.9212 | LR: 2.00e-05 | 1280.9ms/step | 15.6 samples/s +Step 38470/44000 | Loss: 2.9163 | LR: 1.99e-05 | 1278.8ms/step | 15.6 samples/s +Step 38480/44000 | Loss: 2.9409 | LR: 1.98e-05 | 1279.8ms/step | 15.6 samples/s +Step 38490/44000 | Loss: 2.9275 | LR: 1.98e-05 | 1279.7ms/step | 15.6 samples/s +Step 38500/44000 | Loss: 2.9283 | LR: 1.97e-05 | 1279.4ms/step | 15.6 samples/s +Validation at step 38500: Loss = 2.9345, Perplexity = 18.81 +Step 38510/44000 | Loss: 2.9212 | LR: 1.96e-05 | 3426.2ms/step | 5.8 samples/s +Step 38520/44000 | Loss: 2.8982 | LR: 1.95e-05 | 1281.0ms/step | 15.6 samples/s +Step 38530/44000 | Loss: 2.9212 | LR: 1.95e-05 | 1279.9ms/step | 15.6 samples/s +Step 38540/44000 | Loss: 2.9322 | LR: 1.94e-05 | 1284.1ms/step | 15.6 samples/s +Step 38550/44000 | Loss: 2.9411 | LR: 1.93e-05 | 1283.3ms/step | 15.6 samples/s +Step 38560/44000 | Loss: 2.9241 | LR: 1.93e-05 | 1283.1ms/step | 15.6 samples/s +Step 38570/44000 | Loss: 2.9161 | LR: 1.92e-05 | 1284.8ms/step | 15.6 samples/s +Step 38580/44000 | Loss: 2.8975 | LR: 1.91e-05 | 1287.1ms/step | 15.5 samples/s +Step 38590/44000 | Loss: 2.9269 | LR: 1.91e-05 | 1287.2ms/step | 15.5 samples/s +Step 38600/44000 | Loss: 2.9037 | LR: 1.90e-05 | 1287.2ms/step | 15.5 samples/s +Step 38610/44000 | Loss: 2.9085 | LR: 1.89e-05 | 1288.0ms/step | 15.5 samples/s +Step 38620/44000 | Loss: 2.9288 | LR: 1.88e-05 | 1287.2ms/step | 15.5 samples/s +Step 38630/44000 | Loss: 2.9045 | LR: 1.88e-05 | 1287.9ms/step | 15.5 samples/s +Step 38640/44000 | Loss: 2.9448 | LR: 1.87e-05 | 1288.0ms/step | 15.5 samples/s +Step 38650/44000 | Loss: 2.8998 | LR: 1.86e-05 | 1288.7ms/step | 15.5 samples/s +Step 38660/44000 | Loss: 2.9101 | LR: 1.86e-05 | 1287.1ms/step | 15.5 samples/s +Step 38670/44000 | Loss: 2.9248 | LR: 1.85e-05 | 1286.1ms/step | 15.6 samples/s +Step 38680/44000 | Loss: 2.9480 | LR: 1.84e-05 | 1283.1ms/step | 15.6 samples/s +Step 38690/44000 | Loss: 2.9225 | LR: 1.84e-05 | 1284.0ms/step | 15.6 samples/s +Step 38700/44000 | Loss: 2.9401 | LR: 1.83e-05 | 1283.9ms/step | 15.6 samples/s +Step 38710/44000 | Loss: 2.9251 | LR: 1.82e-05 | 1285.1ms/step | 15.6 samples/s +Step 38720/44000 | Loss: 2.8983 | LR: 1.82e-05 | 1283.7ms/step | 15.6 samples/s +Step 38730/44000 | Loss: 2.9143 | LR: 1.81e-05 | 1284.9ms/step | 15.6 samples/s +Step 38740/44000 | Loss: 2.9354 | LR: 1.80e-05 | 1286.1ms/step | 15.6 samples/s +Step 38750/44000 | Loss: 2.9179 | LR: 1.80e-05 | 1286.7ms/step | 15.5 samples/s +Validation at step 38750: Loss = 2.9344, Perplexity = 18.81 +Step 38760/44000 | Loss: 2.9278 | LR: 1.79e-05 | 3453.3ms/step | 5.8 samples/s +Step 38770/44000 | Loss: 2.9162 | LR: 1.78e-05 | 1285.8ms/step | 15.6 samples/s +Step 38780/44000 | Loss: 2.9074 | LR: 1.78e-05 | 1282.5ms/step | 15.6 samples/s +Step 38790/44000 | Loss: 2.9392 | LR: 1.77e-05 | 1284.1ms/step | 15.6 samples/s +Step 38800/44000 | Loss: 2.9060 | LR: 1.76e-05 | 1284.3ms/step | 15.6 samples/s +Step 38810/44000 | Loss: 2.8898 | LR: 1.76e-05 | 1282.0ms/step | 15.6 samples/s +Step 38820/44000 | Loss: 2.9263 | LR: 1.75e-05 | 1285.3ms/step | 15.6 samples/s +Step 38830/44000 | Loss: 2.9258 | LR: 1.74e-05 | 1284.7ms/step | 15.6 samples/s +Step 38840/44000 | Loss: 2.8906 | LR: 1.74e-05 | 1292.5ms/step | 15.5 samples/s +Step 38850/44000 | Loss: 2.9278 | LR: 1.73e-05 | 1289.9ms/step | 15.5 samples/s +Step 38860/44000 | Loss: 2.9159 | LR: 1.72e-05 | 1286.8ms/step | 15.5 samples/s +Step 38870/44000 | Loss: 2.9333 | LR: 1.72e-05 | 1283.9ms/step | 15.6 samples/s +Step 38880/44000 | Loss: 2.9327 | LR: 1.71e-05 | 1282.2ms/step | 15.6 samples/s +Step 38890/44000 | Loss: 2.9200 | LR: 1.70e-05 | 1282.4ms/step | 15.6 samples/s +Step 38900/44000 | Loss: 2.9173 | LR: 1.70e-05 | 1284.8ms/step | 15.6 samples/s +Step 38910/44000 | Loss: 2.9395 | LR: 1.69e-05 | 1280.4ms/step | 15.6 samples/s +Step 38920/44000 | Loss: 2.9210 | LR: 1.68e-05 | 1281.2ms/step | 15.6 samples/s +Step 38930/44000 | Loss: 2.8997 | LR: 1.68e-05 | 1285.2ms/step | 15.6 samples/s +Step 38940/44000 | Loss: 2.9063 | LR: 1.67e-05 | 1285.6ms/step | 15.6 samples/s +Step 38950/44000 | Loss: 2.8938 | LR: 1.66e-05 | 1286.1ms/step | 15.6 samples/s +Step 38960/44000 | Loss: 2.9110 | LR: 1.66e-05 | 1284.8ms/step | 15.6 samples/s +Step 38970/44000 | Loss: 2.9262 | LR: 1.65e-05 | 1284.8ms/step | 15.6 samples/s +Step 38980/44000 | Loss: 2.9005 | LR: 1.64e-05 | 1282.6ms/step | 15.6 samples/s +Step 38990/44000 | Loss: 2.9314 | LR: 1.64e-05 | 1286.5ms/step | 15.5 samples/s +Step 39000/44000 | Loss: 2.9191 | LR: 1.63e-05 | 1284.8ms/step | 15.6 samples/s +Validation at step 39000: Loss = 2.9344, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-39000 +Step 39010/44000 | Loss: 2.9561 | LR: 1.62e-05 | 3726.8ms/step | 5.4 samples/s +Step 39020/44000 | Loss: 2.9090 | LR: 1.62e-05 | 1285.1ms/step | 15.6 samples/s +Step 39030/44000 | Loss: 2.9469 | LR: 1.61e-05 | 1283.5ms/step | 15.6 samples/s +Step 39040/44000 | Loss: 2.9002 | LR: 1.61e-05 | 1284.3ms/step | 15.6 samples/s +Step 39050/44000 | Loss: 2.9086 | LR: 1.60e-05 | 1284.9ms/step | 15.6 samples/s +Step 39060/44000 | Loss: 2.9208 | LR: 1.59e-05 | 1284.2ms/step | 15.6 samples/s +Step 39070/44000 | Loss: 2.8981 | LR: 1.59e-05 | 1284.1ms/step | 15.6 samples/s +Step 39080/44000 | Loss: 2.9283 | LR: 1.58e-05 | 1282.9ms/step | 15.6 samples/s +Step 39090/44000 | Loss: 2.8898 | LR: 1.57e-05 | 1282.8ms/step | 15.6 samples/s +Step 39100/44000 | Loss: 2.8949 | LR: 1.57e-05 | 1281.9ms/step | 15.6 samples/s +Step 39110/44000 | Loss: 2.9324 | LR: 1.56e-05 | 1283.1ms/step | 15.6 samples/s +Step 39120/44000 | Loss: 2.9243 | LR: 1.55e-05 | 1281.0ms/step | 15.6 samples/s +Step 39130/44000 | Loss: 2.8939 | LR: 1.55e-05 | 1280.7ms/step | 15.6 samples/s +Step 39140/44000 | Loss: 2.9129 | LR: 1.54e-05 | 1281.0ms/step | 15.6 samples/s +Step 39150/44000 | Loss: 2.9261 | LR: 1.54e-05 | 1281.4ms/step | 15.6 samples/s +Step 39160/44000 | Loss: 2.9534 | LR: 1.53e-05 | 1280.4ms/step | 15.6 samples/s +Step 39170/44000 | Loss: 2.9289 | LR: 1.52e-05 | 1283.6ms/step | 15.6 samples/s +Step 39180/44000 | Loss: 2.9314 | LR: 1.52e-05 | 1285.6ms/step | 15.6 samples/s +Step 39190/44000 | Loss: 2.8739 | LR: 1.51e-05 | 1283.2ms/step | 15.6 samples/s +Step 39200/44000 | Loss: 2.8978 | LR: 1.50e-05 | 1283.9ms/step | 15.6 samples/s +Step 39210/44000 | Loss: 2.8866 | LR: 1.50e-05 | 1282.2ms/step | 15.6 samples/s +Step 39220/44000 | Loss: 2.9248 | LR: 1.49e-05 | 1283.6ms/step | 15.6 samples/s +Step 39230/44000 | Loss: 2.9100 | LR: 1.49e-05 | 1285.1ms/step | 15.6 samples/s +Step 39240/44000 | Loss: 2.9129 | LR: 1.48e-05 | 1285.1ms/step | 15.6 samples/s +Step 39250/44000 | Loss: 2.9108 | LR: 1.47e-05 | 1284.9ms/step | 15.6 samples/s +Validation at step 39250: Loss = 2.9344, Perplexity = 18.81 +Step 39260/44000 | Loss: 2.9099 | LR: 1.47e-05 | 3431.5ms/step | 5.8 samples/s +Step 39270/44000 | Loss: 2.9029 | LR: 1.46e-05 | 1284.7ms/step | 15.6 samples/s +Step 39280/44000 | Loss: 2.9151 | LR: 1.46e-05 | 1284.6ms/step | 15.6 samples/s +Step 39290/44000 | Loss: 2.9134 | LR: 1.45e-05 | 1284.4ms/step | 15.6 samples/s +Step 39300/44000 | Loss: 2.9128 | LR: 1.44e-05 | 1282.4ms/step | 15.6 samples/s +Step 39310/44000 | Loss: 2.9039 | LR: 1.44e-05 | 1280.9ms/step | 15.6 samples/s +Step 39320/44000 | Loss: 2.8811 | LR: 1.43e-05 | 1279.9ms/step | 15.6 samples/s +Step 39330/44000 | Loss: 2.9244 | LR: 1.42e-05 | 1282.3ms/step | 15.6 samples/s +Step 39340/44000 | Loss: 2.9217 | LR: 1.42e-05 | 1284.6ms/step | 15.6 samples/s +Step 39350/44000 | Loss: 2.9147 | LR: 1.41e-05 | 1283.9ms/step | 15.6 samples/s +Step 39360/44000 | Loss: 2.9540 | LR: 1.41e-05 | 1282.5ms/step | 15.6 samples/s +Step 39370/44000 | Loss: 2.8982 | LR: 1.40e-05 | 1283.0ms/step | 15.6 samples/s +Step 39380/44000 | Loss: 2.9366 | LR: 1.39e-05 | 1282.8ms/step | 15.6 samples/s +Step 39390/44000 | Loss: 2.9206 | LR: 1.39e-05 | 1283.4ms/step | 15.6 samples/s +Step 39400/44000 | Loss: 2.9091 | LR: 1.38e-05 | 1284.5ms/step | 15.6 samples/s +Step 39410/44000 | Loss: 2.9197 | LR: 1.38e-05 | 1288.5ms/step | 15.5 samples/s +Step 39420/44000 | Loss: 2.9435 | LR: 1.37e-05 | 1287.9ms/step | 15.5 samples/s +Step 39430/44000 | Loss: 2.9384 | LR: 1.36e-05 | 1287.0ms/step | 15.5 samples/s +Step 39440/44000 | Loss: 2.9058 | LR: 1.36e-05 | 1286.9ms/step | 15.5 samples/s +Step 39450/44000 | Loss: 2.9400 | LR: 1.35e-05 | 1285.8ms/step | 15.6 samples/s +Step 39460/44000 | Loss: 2.9012 | LR: 1.35e-05 | 1287.4ms/step | 15.5 samples/s +Step 39470/44000 | Loss: 2.9067 | LR: 1.34e-05 | 1285.6ms/step | 15.6 samples/s +Step 39480/44000 | Loss: 2.9297 | LR: 1.34e-05 | 1284.5ms/step | 15.6 samples/s +Step 39490/44000 | Loss: 2.9300 | LR: 1.33e-05 | 1290.8ms/step | 15.5 samples/s +Step 39500/44000 | Loss: 2.9128 | LR: 1.32e-05 | 1290.8ms/step | 15.5 samples/s +Validation at step 39500: Loss = 2.9343, Perplexity = 18.81 +Step 39510/44000 | Loss: 2.8970 | LR: 1.32e-05 | 3432.3ms/step | 5.8 samples/s +Step 39520/44000 | Loss: 2.8968 | LR: 1.31e-05 | 1281.9ms/step | 15.6 samples/s +Step 39530/44000 | Loss: 2.9175 | LR: 1.31e-05 | 1284.4ms/step | 15.6 samples/s +Step 39540/44000 | Loss: 2.9205 | LR: 1.30e-05 | 1281.7ms/step | 15.6 samples/s +Step 39550/44000 | Loss: 2.9134 | LR: 1.29e-05 | 1281.9ms/step | 15.6 samples/s +Step 39560/44000 | Loss: 2.9268 | LR: 1.29e-05 | 1283.2ms/step | 15.6 samples/s +Step 39570/44000 | Loss: 2.8723 | LR: 1.28e-05 | 1282.1ms/step | 15.6 samples/s +Step 39580/44000 | Loss: 2.9150 | LR: 1.28e-05 | 1281.7ms/step | 15.6 samples/s +Step 39590/44000 | Loss: 2.9040 | LR: 1.27e-05 | 1283.7ms/step | 15.6 samples/s +Step 39600/44000 | Loss: 2.9110 | LR: 1.27e-05 | 1283.3ms/step | 15.6 samples/s +Step 39610/44000 | Loss: 2.9148 | LR: 1.26e-05 | 1282.9ms/step | 15.6 samples/s +Step 39620/44000 | Loss: 2.9216 | LR: 1.25e-05 | 1285.7ms/step | 15.6 samples/s +Step 39630/44000 | Loss: 2.9211 | LR: 1.25e-05 | 1282.5ms/step | 15.6 samples/s +Step 39640/44000 | Loss: 2.9241 | LR: 1.24e-05 | 1282.2ms/step | 15.6 samples/s +Step 39650/44000 | Loss: 2.9236 | LR: 1.24e-05 | 1282.7ms/step | 15.6 samples/s +Step 39660/44000 | Loss: 2.9102 | LR: 1.23e-05 | 1283.1ms/step | 15.6 samples/s +Step 39670/44000 | Loss: 2.9005 | LR: 1.23e-05 | 1284.8ms/step | 15.6 samples/s +Step 39680/44000 | Loss: 2.9322 | LR: 1.22e-05 | 1291.2ms/step | 15.5 samples/s +Step 39690/44000 | Loss: 2.9322 | LR: 1.22e-05 | 1293.6ms/step | 15.5 samples/s +Step 39700/44000 | Loss: 2.9237 | LR: 1.21e-05 | 1293.4ms/step | 15.5 samples/s +Step 39710/44000 | Loss: 2.9591 | LR: 1.20e-05 | 1290.2ms/step | 15.5 samples/s +Step 39720/44000 | Loss: 2.9300 | LR: 1.20e-05 | 1292.1ms/step | 15.5 samples/s +Step 39730/44000 | Loss: 2.9263 | LR: 1.19e-05 | 1289.8ms/step | 15.5 samples/s +Step 39740/44000 | Loss: 2.9301 | LR: 1.19e-05 | 1287.1ms/step | 15.5 samples/s +Step 39750/44000 | Loss: 2.9465 | LR: 1.18e-05 | 1298.4ms/step | 15.4 samples/s +Validation at step 39750: Loss = 2.9343, Perplexity = 18.81 +Step 39760/44000 | Loss: 2.9496 | LR: 1.18e-05 | 3480.1ms/step | 5.7 samples/s +Step 39770/44000 | Loss: 2.9148 | LR: 1.17e-05 | 1288.2ms/step | 15.5 samples/s +Step 39780/44000 | Loss: 2.9164 | LR: 1.17e-05 | 1287.8ms/step | 15.5 samples/s +Step 39790/44000 | Loss: 2.9030 | LR: 1.16e-05 | 1285.6ms/step | 15.6 samples/s +Step 39800/44000 | Loss: 2.9189 | LR: 1.15e-05 | 1285.4ms/step | 15.6 samples/s +Step 39810/44000 | Loss: 2.9041 | LR: 1.15e-05 | 1284.5ms/step | 15.6 samples/s +Step 39820/44000 | Loss: 2.9150 | LR: 1.14e-05 | 1280.7ms/step | 15.6 samples/s +Step 39830/44000 | Loss: 2.9249 | LR: 1.14e-05 | 1281.6ms/step | 15.6 samples/s +Step 39840/44000 | Loss: 2.9389 | LR: 1.13e-05 | 1279.9ms/step | 15.6 samples/s +Step 39850/44000 | Loss: 2.9308 | LR: 1.13e-05 | 1280.0ms/step | 15.6 samples/s +Step 39860/44000 | Loss: 2.9218 | LR: 1.12e-05 | 1280.5ms/step | 15.6 samples/s +Step 39870/44000 | Loss: 2.9142 | LR: 1.12e-05 | 1281.9ms/step | 15.6 samples/s +Step 39880/44000 | Loss: 2.8921 | LR: 1.11e-05 | 1282.3ms/step | 15.6 samples/s +Step 39890/44000 | Loss: 2.9077 | LR: 1.11e-05 | 1280.1ms/step | 15.6 samples/s +Step 39900/44000 | Loss: 2.9219 | LR: 1.10e-05 | 1279.0ms/step | 15.6 samples/s +Step 39910/44000 | Loss: 2.8901 | LR: 1.10e-05 | 1282.5ms/step | 15.6 samples/s +Step 39920/44000 | Loss: 2.9481 | LR: 1.09e-05 | 1283.9ms/step | 15.6 samples/s +Step 39930/44000 | Loss: 2.9198 | LR: 1.08e-05 | 1282.9ms/step | 15.6 samples/s +Step 39940/44000 | Loss: 2.9432 | LR: 1.08e-05 | 1283.6ms/step | 15.6 samples/s +Step 39950/44000 | Loss: 2.9044 | LR: 1.07e-05 | 1281.1ms/step | 15.6 samples/s +Step 39960/44000 | Loss: 2.9220 | LR: 1.07e-05 | 1279.9ms/step | 15.6 samples/s +Step 39970/44000 | Loss: 2.9191 | LR: 1.06e-05 | 1281.0ms/step | 15.6 samples/s +Step 39980/44000 | Loss: 2.9080 | LR: 1.06e-05 | 1282.2ms/step | 15.6 samples/s +Step 39990/44000 | Loss: 2.9324 | LR: 1.05e-05 | 1284.1ms/step | 15.6 samples/s +Step 40000/44000 | Loss: 2.9018 | LR: 1.05e-05 | 1280.7ms/step | 15.6 samples/s +Validation at step 40000: Loss = 2.9343, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-40000 +Step 40010/44000 | Loss: 2.9220 | LR: 1.04e-05 | 3723.9ms/step | 5.4 samples/s +Step 40020/44000 | Loss: 2.9222 | LR: 1.04e-05 | 1282.5ms/step | 15.6 samples/s +Step 40030/44000 | Loss: 2.9328 | LR: 1.03e-05 | 1280.7ms/step | 15.6 samples/s +Step 40040/44000 | Loss: 2.9129 | LR: 1.03e-05 | 1280.1ms/step | 15.6 samples/s +Step 40050/44000 | Loss: 2.8966 | LR: 1.02e-05 | 1279.4ms/step | 15.6 samples/s +Step 40060/44000 | Loss: 2.9031 | LR: 1.02e-05 | 1280.5ms/step | 15.6 samples/s +Step 40070/44000 | Loss: 2.9265 | LR: 1.01e-05 | 1283.3ms/step | 15.6 samples/s +Step 40080/44000 | Loss: 2.9292 | LR: 1.01e-05 | 1283.6ms/step | 15.6 samples/s +Step 40090/44000 | Loss: 2.9104 | LR: 1.00e-05 | 1282.4ms/step | 15.6 samples/s +Step 40100/44000 | Loss: 2.9193 | LR: 9.96e-06 | 1280.0ms/step | 15.6 samples/s +Step 40110/44000 | Loss: 2.9188 | LR: 9.91e-06 | 1285.4ms/step | 15.6 samples/s +Step 40120/44000 | Loss: 2.9331 | LR: 9.86e-06 | 1282.8ms/step | 15.6 samples/s +Step 40130/44000 | Loss: 2.9262 | LR: 9.81e-06 | 1280.1ms/step | 15.6 samples/s +Step 40140/44000 | Loss: 2.9315 | LR: 9.76e-06 | 1284.8ms/step | 15.6 samples/s +Step 40150/44000 | Loss: 2.8912 | LR: 9.71e-06 | 1286.8ms/step | 15.5 samples/s +Step 40160/44000 | Loss: 2.8987 | LR: 9.66e-06 | 1285.9ms/step | 15.6 samples/s +Step 40170/44000 | Loss: 2.9042 | LR: 9.61e-06 | 1286.9ms/step | 15.5 samples/s +Step 40180/44000 | Loss: 2.9432 | LR: 9.56e-06 | 1287.0ms/step | 15.5 samples/s +Step 40190/44000 | Loss: 2.9187 | LR: 9.51e-06 | 1286.8ms/step | 15.5 samples/s +Step 40200/44000 | Loss: 2.9504 | LR: 9.46e-06 | 1294.4ms/step | 15.5 samples/s +Step 40210/44000 | Loss: 2.9165 | LR: 9.41e-06 | 1288.7ms/step | 15.5 samples/s +Step 40220/44000 | Loss: 2.9356 | LR: 9.36e-06 | 1289.7ms/step | 15.5 samples/s +Step 40230/44000 | Loss: 2.9080 | LR: 9.32e-06 | 1285.3ms/step | 15.6 samples/s +Step 40240/44000 | Loss: 2.9177 | LR: 9.27e-06 | 1277.8ms/step | 15.7 samples/s +Step 40250/44000 | Loss: 2.8806 | LR: 9.22e-06 | 1277.8ms/step | 15.7 samples/s +Validation at step 40250: Loss = 2.9343, Perplexity = 18.81 +Step 40260/44000 | Loss: 2.9101 | LR: 9.17e-06 | 3415.8ms/step | 5.9 samples/s +Step 40270/44000 | Loss: 2.9268 | LR: 9.12e-06 | 1279.8ms/step | 15.6 samples/s +Step 40280/44000 | Loss: 2.9233 | LR: 9.07e-06 | 1278.0ms/step | 15.6 samples/s +Step 40290/44000 | Loss: 2.8982 | LR: 9.02e-06 | 1279.0ms/step | 15.6 samples/s +Step 40300/44000 | Loss: 2.9240 | LR: 8.97e-06 | 1280.1ms/step | 15.6 samples/s +Step 40310/44000 | Loss: 2.9123 | LR: 8.93e-06 | 1285.5ms/step | 15.6 samples/s +Step 40320/44000 | Loss: 2.9407 | LR: 8.88e-06 | 1286.3ms/step | 15.5 samples/s +Step 40330/44000 | Loss: 2.9220 | LR: 8.83e-06 | 1285.8ms/step | 15.6 samples/s +Step 40340/44000 | Loss: 2.9184 | LR: 8.78e-06 | 1284.0ms/step | 15.6 samples/s +Step 40350/44000 | Loss: 2.9102 | LR: 8.74e-06 | 1283.4ms/step | 15.6 samples/s +Step 40360/44000 | Loss: 2.9237 | LR: 8.69e-06 | 1284.1ms/step | 15.6 samples/s +Step 40370/44000 | Loss: 2.9008 | LR: 8.64e-06 | 1281.5ms/step | 15.6 samples/s +Step 40380/44000 | Loss: 2.9214 | LR: 8.59e-06 | 1277.9ms/step | 15.7 samples/s +Step 40390/44000 | Loss: 2.9368 | LR: 8.55e-06 | 1280.3ms/step | 15.6 samples/s +Step 40400/44000 | Loss: 2.9054 | LR: 8.50e-06 | 1278.8ms/step | 15.6 samples/s +Step 40410/44000 | Loss: 2.8908 | LR: 8.45e-06 | 1279.4ms/step | 15.6 samples/s +Step 40420/44000 | Loss: 2.9146 | LR: 8.41e-06 | 1278.9ms/step | 15.6 samples/s +Step 40430/44000 | Loss: 2.9234 | LR: 8.36e-06 | 1278.6ms/step | 15.6 samples/s +Step 40440/44000 | Loss: 2.9018 | LR: 8.31e-06 | 1279.7ms/step | 15.6 samples/s +Step 40450/44000 | Loss: 2.9125 | LR: 8.27e-06 | 1278.6ms/step | 15.6 samples/s +Step 40460/44000 | Loss: 2.8816 | LR: 8.22e-06 | 1280.5ms/step | 15.6 samples/s +Step 40470/44000 | Loss: 2.9367 | LR: 8.17e-06 | 1280.3ms/step | 15.6 samples/s +Step 40480/44000 | Loss: 2.9246 | LR: 8.13e-06 | 1281.8ms/step | 15.6 samples/s +Step 40490/44000 | Loss: 2.9245 | LR: 8.08e-06 | 1279.8ms/step | 15.6 samples/s +Step 40500/44000 | Loss: 2.9184 | LR: 8.04e-06 | 1280.9ms/step | 15.6 samples/s +Validation at step 40500: Loss = 2.9343, Perplexity = 18.81 +Step 40510/44000 | Loss: 2.9061 | LR: 7.99e-06 | 3426.4ms/step | 5.8 samples/s +Step 40520/44000 | Loss: 2.9541 | LR: 7.94e-06 | 1282.6ms/step | 15.6 samples/s +Step 40530/44000 | Loss: 2.9412 | LR: 7.90e-06 | 1280.0ms/step | 15.6 samples/s +Step 40540/44000 | Loss: 2.9337 | LR: 7.85e-06 | 1282.9ms/step | 15.6 samples/s +Step 40550/44000 | Loss: 2.8914 | LR: 7.81e-06 | 1281.1ms/step | 15.6 samples/s +Step 40560/44000 | Loss: 2.8911 | LR: 7.76e-06 | 1279.7ms/step | 15.6 samples/s +Step 40570/44000 | Loss: 2.9267 | LR: 7.72e-06 | 1283.8ms/step | 15.6 samples/s +Step 40580/44000 | Loss: 2.9186 | LR: 7.67e-06 | 1281.3ms/step | 15.6 samples/s +Step 40590/44000 | Loss: 2.8952 | LR: 7.63e-06 | 1281.0ms/step | 15.6 samples/s +Step 40600/44000 | Loss: 2.9217 | LR: 7.59e-06 | 1281.6ms/step | 15.6 samples/s +Step 40610/44000 | Loss: 2.9526 | LR: 7.54e-06 | 1278.9ms/step | 15.6 samples/s +Step 40620/44000 | Loss: 2.9185 | LR: 7.50e-06 | 1282.4ms/step | 15.6 samples/s +Step 40630/44000 | Loss: 2.9326 | LR: 7.45e-06 | 1281.2ms/step | 15.6 samples/s +Step 40640/44000 | Loss: 2.9385 | LR: 7.41e-06 | 1282.7ms/step | 15.6 samples/s +Step 40650/44000 | Loss: 2.9365 | LR: 7.37e-06 | 1283.5ms/step | 15.6 samples/s +Step 40660/44000 | Loss: 2.9276 | LR: 7.32e-06 | 1284.9ms/step | 15.6 samples/s +Step 40670/44000 | Loss: 2.9360 | LR: 7.28e-06 | 1283.9ms/step | 15.6 samples/s +Step 40680/44000 | Loss: 2.9209 | LR: 7.23e-06 | 1281.7ms/step | 15.6 samples/s +Step 40690/44000 | Loss: 2.9315 | LR: 7.19e-06 | 1282.3ms/step | 15.6 samples/s +Step 40700/44000 | Loss: 2.9098 | LR: 7.15e-06 | 1282.8ms/step | 15.6 samples/s +Step 40710/44000 | Loss: 2.9244 | LR: 7.10e-06 | 1281.9ms/step | 15.6 samples/s +Step 40720/44000 | Loss: 2.9329 | LR: 7.06e-06 | 1279.8ms/step | 15.6 samples/s +Step 40730/44000 | Loss: 2.9408 | LR: 7.02e-06 | 1280.0ms/step | 15.6 samples/s +Step 40740/44000 | Loss: 2.9116 | LR: 6.98e-06 | 1279.5ms/step | 15.6 samples/s +Step 40750/44000 | Loss: 2.9087 | LR: 6.93e-06 | 1277.5ms/step | 15.7 samples/s +Validation at step 40750: Loss = 2.9343, Perplexity = 18.81 +Step 40760/44000 | Loss: 2.9363 | LR: 6.89e-06 | 3412.8ms/step | 5.9 samples/s +Step 40770/44000 | Loss: 2.9158 | LR: 6.85e-06 | 1278.7ms/step | 15.6 samples/s +Step 40780/44000 | Loss: 2.9365 | LR: 6.81e-06 | 1278.7ms/step | 15.6 samples/s +Step 40790/44000 | Loss: 2.9388 | LR: 6.77e-06 | 1278.6ms/step | 15.6 samples/s +Step 40800/44000 | Loss: 2.9423 | LR: 6.72e-06 | 1278.9ms/step | 15.6 samples/s +Step 40810/44000 | Loss: 2.9221 | LR: 6.68e-06 | 1278.1ms/step | 15.6 samples/s +Step 40820/44000 | Loss: 2.8997 | LR: 6.64e-06 | 1275.4ms/step | 15.7 samples/s +Step 40830/44000 | Loss: 2.8973 | LR: 6.60e-06 | 1274.2ms/step | 15.7 samples/s +Step 40840/44000 | Loss: 2.9390 | LR: 6.56e-06 | 1276.3ms/step | 15.7 samples/s +Step 40850/44000 | Loss: 2.9266 | LR: 6.52e-06 | 1274.4ms/step | 15.7 samples/s +Step 40860/44000 | Loss: 2.9396 | LR: 6.47e-06 | 1281.4ms/step | 15.6 samples/s +Step 40870/44000 | Loss: 2.9424 | LR: 6.43e-06 | 1281.8ms/step | 15.6 samples/s +Step 40880/44000 | Loss: 2.9144 | LR: 6.39e-06 | 1281.7ms/step | 15.6 samples/s +Step 40890/44000 | Loss: 2.9322 | LR: 6.35e-06 | 1283.7ms/step | 15.6 samples/s +Step 40900/44000 | Loss: 2.9074 | LR: 6.31e-06 | 1282.7ms/step | 15.6 samples/s +Step 40910/44000 | Loss: 2.9779 | LR: 6.27e-06 | 1282.5ms/step | 15.6 samples/s +Step 40920/44000 | Loss: 2.9248 | LR: 6.23e-06 | 1284.3ms/step | 15.6 samples/s +Step 40930/44000 | Loss: 2.9338 | LR: 6.19e-06 | 1282.2ms/step | 15.6 samples/s +Step 40940/44000 | Loss: 2.9362 | LR: 6.15e-06 | 1281.1ms/step | 15.6 samples/s +Step 40950/44000 | Loss: 2.8979 | LR: 6.11e-06 | 1284.0ms/step | 15.6 samples/s +Step 40960/44000 | Loss: 2.8851 | LR: 6.07e-06 | 1283.3ms/step | 15.6 samples/s +Step 40970/44000 | Loss: 2.9144 | LR: 6.03e-06 | 1282.4ms/step | 15.6 samples/s +Step 40980/44000 | Loss: 2.9018 | LR: 5.99e-06 | 1279.4ms/step | 15.6 samples/s +Step 40990/44000 | Loss: 2.9251 | LR: 5.95e-06 | 1279.9ms/step | 15.6 samples/s +Step 41000/44000 | Loss: 2.9289 | LR: 5.91e-06 | 1277.8ms/step | 15.7 samples/s +Validation at step 41000: Loss = 2.9343, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-41000 +Step 41010/44000 | Loss: 2.9377 | LR: 5.87e-06 | 3723.1ms/step | 5.4 samples/s +Step 41020/44000 | Loss: 2.9435 | LR: 5.83e-06 | 1290.9ms/step | 15.5 samples/s +Step 41030/44000 | Loss: 2.9185 | LR: 5.80e-06 | 1292.0ms/step | 15.5 samples/s +Step 41040/44000 | Loss: 2.9188 | LR: 5.76e-06 | 1285.9ms/step | 15.6 samples/s +Step 41050/44000 | Loss: 2.9165 | LR: 5.72e-06 | 1287.9ms/step | 15.5 samples/s +Step 41060/44000 | Loss: 2.8874 | LR: 5.68e-06 | 1291.2ms/step | 15.5 samples/s +Step 41070/44000 | Loss: 2.9158 | LR: 5.64e-06 | 1286.1ms/step | 15.6 samples/s +Step 41080/44000 | Loss: 2.9470 | LR: 5.60e-06 | 1286.6ms/step | 15.5 samples/s +Step 41090/44000 | Loss: 2.9630 | LR: 5.56e-06 | 1289.5ms/step | 15.5 samples/s +Step 41100/44000 | Loss: 2.9410 | LR: 5.53e-06 | 1284.5ms/step | 15.6 samples/s +Step 41110/44000 | Loss: 2.9643 | LR: 5.49e-06 | 1283.9ms/step | 15.6 samples/s +Step 41120/44000 | Loss: 2.9509 | LR: 5.45e-06 | 1288.1ms/step | 15.5 samples/s +Step 41130/44000 | Loss: 2.9196 | LR: 5.41e-06 | 1279.8ms/step | 15.6 samples/s +Step 41140/44000 | Loss: 2.9010 | LR: 5.38e-06 | 1278.7ms/step | 15.6 samples/s +Step 41150/44000 | Loss: 2.9408 | LR: 5.34e-06 | 1278.3ms/step | 15.6 samples/s +Step 41160/44000 | Loss: 2.9140 | LR: 5.30e-06 | 1280.3ms/step | 15.6 samples/s +Step 41170/44000 | Loss: 2.9248 | LR: 5.26e-06 | 1281.6ms/step | 15.6 samples/s +Step 41180/44000 | Loss: 2.9006 | LR: 5.23e-06 | 1279.7ms/step | 15.6 samples/s +Step 41190/44000 | Loss: 2.8836 | LR: 5.19e-06 | 1279.0ms/step | 15.6 samples/s +Step 41200/44000 | Loss: 2.9242 | LR: 5.15e-06 | 1281.9ms/step | 15.6 samples/s +Step 41210/44000 | Loss: 2.9331 | LR: 5.12e-06 | 1283.1ms/step | 15.6 samples/s +Step 41220/44000 | Loss: 2.9408 | LR: 5.08e-06 | 1282.7ms/step | 15.6 samples/s +Step 41230/44000 | Loss: 2.9138 | LR: 5.04e-06 | 1283.2ms/step | 15.6 samples/s +Step 41240/44000 | Loss: 2.9032 | LR: 5.01e-06 | 1280.6ms/step | 15.6 samples/s +Step 41250/44000 | Loss: 2.9200 | LR: 4.97e-06 | 1279.4ms/step | 15.6 samples/s +Validation at step 41250: Loss = 2.9343, Perplexity = 18.81 +Step 41260/44000 | Loss: 2.9456 | LR: 4.94e-06 | 3431.0ms/step | 5.8 samples/s +Step 41270/44000 | Loss: 2.9267 | LR: 4.90e-06 | 1279.1ms/step | 15.6 samples/s +Step 41280/44000 | Loss: 2.9231 | LR: 4.86e-06 | 1284.0ms/step | 15.6 samples/s +Step 41290/44000 | Loss: 2.9106 | LR: 4.83e-06 | 1284.9ms/step | 15.6 samples/s +Step 41300/44000 | Loss: 2.9158 | LR: 4.79e-06 | 1281.8ms/step | 15.6 samples/s +Step 41310/44000 | Loss: 2.9031 | LR: 4.76e-06 | 1280.5ms/step | 15.6 samples/s +Step 41320/44000 | Loss: 2.9288 | LR: 4.72e-06 | 1280.7ms/step | 15.6 samples/s +Step 41330/44000 | Loss: 2.9138 | LR: 4.69e-06 | 1279.9ms/step | 15.6 samples/s +Step 41340/44000 | Loss: 2.9362 | LR: 4.65e-06 | 1280.3ms/step | 15.6 samples/s +Step 41350/44000 | Loss: 2.9008 | LR: 4.62e-06 | 1280.2ms/step | 15.6 samples/s +Step 41360/44000 | Loss: 2.9491 | LR: 4.58e-06 | 1275.0ms/step | 15.7 samples/s +Step 41370/44000 | Loss: 2.9313 | LR: 4.55e-06 | 1274.0ms/step | 15.7 samples/s +Step 41380/44000 | Loss: 2.9258 | LR: 4.51e-06 | 1273.4ms/step | 15.7 samples/s +Step 41390/44000 | Loss: 2.9038 | LR: 4.48e-06 | 1273.7ms/step | 15.7 samples/s +Step 41400/44000 | Loss: 2.8906 | LR: 4.45e-06 | 1274.0ms/step | 15.7 samples/s +Step 41410/44000 | Loss: 2.8978 | LR: 4.41e-06 | 1275.2ms/step | 15.7 samples/s +Step 41420/44000 | Loss: 2.8906 | LR: 4.38e-06 | 1279.5ms/step | 15.6 samples/s +Step 41430/44000 | Loss: 2.9160 | LR: 4.34e-06 | 1281.5ms/step | 15.6 samples/s +Step 41440/44000 | Loss: 2.9553 | LR: 4.31e-06 | 1279.9ms/step | 15.6 samples/s +Step 41450/44000 | Loss: 2.9213 | LR: 4.28e-06 | 1280.8ms/step | 15.6 samples/s +Step 41460/44000 | Loss: 2.9210 | LR: 4.24e-06 | 1282.7ms/step | 15.6 samples/s +Step 41470/44000 | Loss: 2.9075 | LR: 4.21e-06 | 1280.7ms/step | 15.6 samples/s +Step 41480/44000 | Loss: 2.9374 | LR: 4.18e-06 | 1283.9ms/step | 15.6 samples/s +Step 41490/44000 | Loss: 2.9066 | LR: 4.14e-06 | 1286.8ms/step | 15.5 samples/s +Step 41500/44000 | Loss: 2.9221 | LR: 4.11e-06 | 1287.0ms/step | 15.5 samples/s +Validation at step 41500: Loss = 2.9343, Perplexity = 18.81 +Step 41510/44000 | Loss: 2.9181 | LR: 4.08e-06 | 3474.7ms/step | 5.8 samples/s +Step 41520/44000 | Loss: 2.9191 | LR: 4.05e-06 | 1287.5ms/step | 15.5 samples/s +Step 41530/44000 | Loss: 2.9232 | LR: 4.01e-06 | 1288.1ms/step | 15.5 samples/s +Step 41540/44000 | Loss: 2.9458 | LR: 3.98e-06 | 1289.7ms/step | 15.5 samples/s +Step 41550/44000 | Loss: 2.9191 | LR: 3.95e-06 | 1293.9ms/step | 15.5 samples/s +Step 41560/44000 | Loss: 2.9317 | LR: 3.92e-06 | 1287.8ms/step | 15.5 samples/s +Step 41570/44000 | Loss: 2.9494 | LR: 3.88e-06 | 1288.6ms/step | 15.5 samples/s +Step 41580/44000 | Loss: 2.9050 | LR: 3.85e-06 | 1288.8ms/step | 15.5 samples/s +Step 41590/44000 | Loss: 2.9182 | LR: 3.82e-06 | 1286.2ms/step | 15.5 samples/s +Step 41600/44000 | Loss: 2.9306 | LR: 3.79e-06 | 1284.4ms/step | 15.6 samples/s +Step 41610/44000 | Loss: 2.9107 | LR: 3.76e-06 | 1284.7ms/step | 15.6 samples/s +Step 41620/44000 | Loss: 2.8864 | LR: 3.73e-06 | 1285.3ms/step | 15.6 samples/s +Step 41630/44000 | Loss: 2.9439 | LR: 3.70e-06 | 1284.2ms/step | 15.6 samples/s +Step 41640/44000 | Loss: 2.9436 | LR: 3.66e-06 | 1285.5ms/step | 15.6 samples/s +Step 41650/44000 | Loss: 2.9258 | LR: 3.63e-06 | 1286.4ms/step | 15.5 samples/s +Step 41660/44000 | Loss: 2.9446 | LR: 3.60e-06 | 1283.9ms/step | 15.6 samples/s +Step 41670/44000 | Loss: 2.9189 | LR: 3.57e-06 | 1285.7ms/step | 15.6 samples/s +Step 41680/44000 | Loss: 2.8910 | LR: 3.54e-06 | 1284.5ms/step | 15.6 samples/s +Step 41690/44000 | Loss: 2.9047 | LR: 3.51e-06 | 1286.1ms/step | 15.6 samples/s +Step 41700/44000 | Loss: 2.9376 | LR: 3.48e-06 | 1284.3ms/step | 15.6 samples/s +Step 41710/44000 | Loss: 2.9115 | LR: 3.45e-06 | 1283.2ms/step | 15.6 samples/s +Step 41720/44000 | Loss: 2.9146 | LR: 3.42e-06 | 1282.1ms/step | 15.6 samples/s +Step 41730/44000 | Loss: 2.9273 | LR: 3.39e-06 | 1283.7ms/step | 15.6 samples/s +Step 41740/44000 | Loss: 2.9378 | LR: 3.36e-06 | 1283.2ms/step | 15.6 samples/s +Step 41750/44000 | Loss: 2.9137 | LR: 3.33e-06 | 1285.3ms/step | 15.6 samples/s +Validation at step 41750: Loss = 2.9343, Perplexity = 18.81 +Step 41760/44000 | Loss: 2.9457 | LR: 3.30e-06 | 3431.6ms/step | 5.8 samples/s +Step 41770/44000 | Loss: 2.9199 | LR: 3.27e-06 | 1286.1ms/step | 15.6 samples/s +Step 41780/44000 | Loss: 2.9116 | LR: 3.24e-06 | 1286.9ms/step | 15.5 samples/s +Step 41790/44000 | Loss: 2.9549 | LR: 3.21e-06 | 1287.5ms/step | 15.5 samples/s +Step 41800/44000 | Loss: 2.9663 | LR: 3.19e-06 | 1284.9ms/step | 15.6 samples/s +Step 41810/44000 | Loss: 2.9309 | LR: 3.16e-06 | 1285.5ms/step | 15.6 samples/s +Step 41820/44000 | Loss: 2.9068 | LR: 3.13e-06 | 1281.4ms/step | 15.6 samples/s +Step 41830/44000 | Loss: 2.9079 | LR: 3.10e-06 | 1280.4ms/step | 15.6 samples/s +Step 41840/44000 | Loss: 2.8893 | LR: 3.07e-06 | 1280.3ms/step | 15.6 samples/s +Step 41850/44000 | Loss: 2.8958 | LR: 3.04e-06 | 1280.0ms/step | 15.6 samples/s +Step 41860/44000 | Loss: 2.9431 | LR: 3.01e-06 | 1279.4ms/step | 15.6 samples/s +Step 41870/44000 | Loss: 2.9537 | LR: 2.99e-06 | 1281.7ms/step | 15.6 samples/s +Step 41880/44000 | Loss: 2.9316 | LR: 2.96e-06 | 1287.8ms/step | 15.5 samples/s +Step 41890/44000 | Loss: 2.9141 | LR: 2.93e-06 | 1287.3ms/step | 15.5 samples/s +Step 41900/44000 | Loss: 2.9165 | LR: 2.90e-06 | 1286.1ms/step | 15.6 samples/s +Step 41910/44000 | Loss: 2.9096 | LR: 2.88e-06 | 1286.3ms/step | 15.5 samples/s +Step 41920/44000 | Loss: 2.9616 | LR: 2.85e-06 | 1284.9ms/step | 15.6 samples/s +Step 41930/44000 | Loss: 2.9131 | LR: 2.82e-06 | 1285.2ms/step | 15.6 samples/s +Step 41940/44000 | Loss: 2.9220 | LR: 2.79e-06 | 1283.3ms/step | 15.6 samples/s +Step 41950/44000 | Loss: 2.8837 | LR: 2.77e-06 | 1285.1ms/step | 15.6 samples/s +Step 41960/44000 | Loss: 2.9303 | LR: 2.74e-06 | 1284.8ms/step | 15.6 samples/s +Step 41970/44000 | Loss: 2.9248 | LR: 2.71e-06 | 1287.3ms/step | 15.5 samples/s +Step 41980/44000 | Loss: 2.9195 | LR: 2.69e-06 | 1284.8ms/step | 15.6 samples/s +Step 41990/44000 | Loss: 2.9221 | LR: 2.66e-06 | 1287.5ms/step | 15.5 samples/s +Step 42000/44000 | Loss: 2.9215 | LR: 2.63e-06 | 1290.3ms/step | 15.5 samples/s +Validation at step 42000: Loss = 2.9343, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-42000 +Step 42010/44000 | Loss: 2.9297 | LR: 2.61e-06 | 3726.4ms/step | 5.4 samples/s +Step 42020/44000 | Loss: 2.9384 | LR: 2.58e-06 | 1281.5ms/step | 15.6 samples/s +Step 42030/44000 | Loss: 2.9449 | LR: 2.56e-06 | 1281.1ms/step | 15.6 samples/s +Step 42040/44000 | Loss: 2.9428 | LR: 2.53e-06 | 1281.4ms/step | 15.6 samples/s +Step 42050/44000 | Loss: 2.8999 | LR: 2.50e-06 | 1282.1ms/step | 15.6 samples/s +Step 42060/44000 | Loss: 2.9368 | LR: 2.48e-06 | 1281.9ms/step | 15.6 samples/s +Step 42070/44000 | Loss: 2.9382 | LR: 2.45e-06 | 1281.7ms/step | 15.6 samples/s +Step 42080/44000 | Loss: 2.9178 | LR: 2.43e-06 | 1280.0ms/step | 15.6 samples/s +Step 42090/44000 | Loss: 2.9030 | LR: 2.40e-06 | 1275.6ms/step | 15.7 samples/s +Step 42100/44000 | Loss: 2.9242 | LR: 2.38e-06 | 1276.1ms/step | 15.7 samples/s +Step 42110/44000 | Loss: 2.9377 | LR: 2.35e-06 | 1275.3ms/step | 15.7 samples/s +Step 42120/44000 | Loss: 2.9235 | LR: 2.33e-06 | 1276.6ms/step | 15.7 samples/s +Step 42130/44000 | Loss: 2.9368 | LR: 2.30e-06 | 1275.8ms/step | 15.7 samples/s +Step 42140/44000 | Loss: 2.9324 | LR: 2.28e-06 | 1281.4ms/step | 15.6 samples/s +Step 42150/44000 | Loss: 2.9529 | LR: 2.25e-06 | 1282.0ms/step | 15.6 samples/s +Step 42160/44000 | Loss: 2.9203 | LR: 2.23e-06 | 1284.2ms/step | 15.6 samples/s +Step 42170/44000 | Loss: 2.9066 | LR: 2.21e-06 | 1288.5ms/step | 15.5 samples/s +Step 42180/44000 | Loss: 2.9341 | LR: 2.18e-06 | 1284.5ms/step | 15.6 samples/s +Step 42190/44000 | Loss: 2.9187 | LR: 2.16e-06 | 1284.1ms/step | 15.6 samples/s +Step 42200/44000 | Loss: 2.9047 | LR: 2.13e-06 | 1283.9ms/step | 15.6 samples/s +Step 42210/44000 | Loss: 2.9499 | LR: 2.11e-06 | 1284.4ms/step | 15.6 samples/s +Step 42220/44000 | Loss: 2.8897 | LR: 2.09e-06 | 1283.9ms/step | 15.6 samples/s +Step 42230/44000 | Loss: 2.9422 | LR: 2.06e-06 | 1285.8ms/step | 15.6 samples/s +Step 42240/44000 | Loss: 2.9198 | LR: 2.04e-06 | 1285.5ms/step | 15.6 samples/s +Step 42250/44000 | Loss: 2.9367 | LR: 2.02e-06 | 1286.0ms/step | 15.6 samples/s +Validation at step 42250: Loss = 2.9342, Perplexity = 18.81 +Step 42260/44000 | Loss: 2.9264 | LR: 1.99e-06 | 3428.0ms/step | 5.8 samples/s +Step 42270/44000 | Loss: 2.9206 | LR: 1.97e-06 | 1282.4ms/step | 15.6 samples/s +Step 42280/44000 | Loss: 2.9221 | LR: 1.95e-06 | 1281.8ms/step | 15.6 samples/s +Step 42290/44000 | Loss: 2.8913 | LR: 1.93e-06 | 1287.3ms/step | 15.5 samples/s +Step 42300/44000 | Loss: 2.9189 | LR: 1.90e-06 | 1288.7ms/step | 15.5 samples/s +Step 42310/44000 | Loss: 2.8923 | LR: 1.88e-06 | 1288.5ms/step | 15.5 samples/s +Step 42320/44000 | Loss: 2.9384 | LR: 1.86e-06 | 1289.9ms/step | 15.5 samples/s +Step 42330/44000 | Loss: 2.9059 | LR: 1.84e-06 | 1289.8ms/step | 15.5 samples/s +Step 42340/44000 | Loss: 2.9478 | LR: 1.82e-06 | 1286.8ms/step | 15.5 samples/s +Step 42350/44000 | Loss: 2.8805 | LR: 1.79e-06 | 1290.8ms/step | 15.5 samples/s +Step 42360/44000 | Loss: 2.9207 | LR: 1.77e-06 | 1289.0ms/step | 15.5 samples/s +Step 42370/44000 | Loss: 2.9213 | LR: 1.75e-06 | 1282.6ms/step | 15.6 samples/s +Step 42380/44000 | Loss: 2.9551 | LR: 1.73e-06 | 1286.9ms/step | 15.5 samples/s +Step 42390/44000 | Loss: 2.8869 | LR: 1.71e-06 | 1293.6ms/step | 15.5 samples/s +Step 42400/44000 | Loss: 2.9152 | LR: 1.69e-06 | 1286.3ms/step | 15.5 samples/s +Step 42410/44000 | Loss: 2.9189 | LR: 1.67e-06 | 1292.0ms/step | 15.5 samples/s +Step 42420/44000 | Loss: 2.9467 | LR: 1.64e-06 | 1297.7ms/step | 15.4 samples/s +Step 42430/44000 | Loss: 2.9489 | LR: 1.62e-06 | 1286.5ms/step | 15.5 samples/s +Step 42440/44000 | Loss: 2.9407 | LR: 1.60e-06 | 1290.0ms/step | 15.5 samples/s +Step 42450/44000 | Loss: 2.9099 | LR: 1.58e-06 | 1290.9ms/step | 15.5 samples/s +Step 42460/44000 | Loss: 2.9188 | LR: 1.56e-06 | 1290.4ms/step | 15.5 samples/s +Step 42470/44000 | Loss: 2.9539 | LR: 1.54e-06 | 1287.1ms/step | 15.5 samples/s +Step 42480/44000 | Loss: 2.9219 | LR: 1.52e-06 | 1284.7ms/step | 15.6 samples/s +Step 42490/44000 | Loss: 2.9290 | LR: 1.50e-06 | 1288.6ms/step | 15.5 samples/s +Step 42500/44000 | Loss: 2.8976 | LR: 1.48e-06 | 1288.3ms/step | 15.5 samples/s +Validation at step 42500: Loss = 2.9343, Perplexity = 18.81 +Step 42510/44000 | Loss: 2.9395 | LR: 1.46e-06 | 3431.6ms/step | 5.8 samples/s +Step 42520/44000 | Loss: 2.8998 | LR: 1.44e-06 | 1281.7ms/step | 15.6 samples/s +Step 42530/44000 | Loss: 2.9278 | LR: 1.42e-06 | 1279.3ms/step | 15.6 samples/s +Step 42540/44000 | Loss: 2.9150 | LR: 1.40e-06 | 1284.8ms/step | 15.6 samples/s +Step 42550/44000 | Loss: 2.9326 | LR: 1.39e-06 | 1285.6ms/step | 15.6 samples/s +Step 42560/44000 | Loss: 2.9045 | LR: 1.37e-06 | 1284.8ms/step | 15.6 samples/s +Step 42570/44000 | Loss: 2.9217 | LR: 1.35e-06 | 1287.3ms/step | 15.5 samples/s +Step 42580/44000 | Loss: 2.9218 | LR: 1.33e-06 | 1288.9ms/step | 15.5 samples/s +Step 42590/44000 | Loss: 2.9065 | LR: 1.31e-06 | 1287.4ms/step | 15.5 samples/s +Step 42600/44000 | Loss: 2.9017 | LR: 1.29e-06 | 1289.0ms/step | 15.5 samples/s +Step 42610/44000 | Loss: 2.9291 | LR: 1.27e-06 | 1293.8ms/step | 15.5 samples/s +Step 42620/44000 | Loss: 2.9372 | LR: 1.25e-06 | 1287.3ms/step | 15.5 samples/s +Step 42630/44000 | Loss: 2.9062 | LR: 1.24e-06 | 1286.2ms/step | 15.5 samples/s +Step 42640/44000 | Loss: 2.9149 | LR: 1.22e-06 | 1284.1ms/step | 15.6 samples/s +Step 42650/44000 | Loss: 2.9360 | LR: 1.20e-06 | 1283.4ms/step | 15.6 samples/s +Step 42660/44000 | Loss: 2.9076 | LR: 1.18e-06 | 1283.2ms/step | 15.6 samples/s +Step 42670/44000 | Loss: 2.9066 | LR: 1.17e-06 | 1284.4ms/step | 15.6 samples/s +Step 42680/44000 | Loss: 2.8994 | LR: 1.15e-06 | 1284.4ms/step | 15.6 samples/s +Step 42690/44000 | Loss: 2.8927 | LR: 1.13e-06 | 1284.4ms/step | 15.6 samples/s +Step 42700/44000 | Loss: 2.8969 | LR: 1.11e-06 | 1283.5ms/step | 15.6 samples/s +Step 42710/44000 | Loss: 2.9347 | LR: 1.10e-06 | 1283.6ms/step | 15.6 samples/s +Step 42720/44000 | Loss: 2.9271 | LR: 1.08e-06 | 1283.6ms/step | 15.6 samples/s +Step 42730/44000 | Loss: 2.9196 | LR: 1.06e-06 | 1282.8ms/step | 15.6 samples/s +Step 42740/44000 | Loss: 2.8944 | LR: 1.05e-06 | 1285.4ms/step | 15.6 samples/s +Step 42750/44000 | Loss: 2.9044 | LR: 1.03e-06 | 1285.4ms/step | 15.6 samples/s +Validation at step 42750: Loss = 2.9343, Perplexity = 18.81 +Step 42760/44000 | Loss: 2.9197 | LR: 1.01e-06 | 3427.3ms/step | 5.8 samples/s +Step 42770/44000 | Loss: 2.9343 | LR: 9.97e-07 | 1282.4ms/step | 15.6 samples/s +Step 42780/44000 | Loss: 2.9397 | LR: 9.81e-07 | 1282.9ms/step | 15.6 samples/s +Step 42790/44000 | Loss: 2.9126 | LR: 9.65e-07 | 1284.5ms/step | 15.6 samples/s +Step 42800/44000 | Loss: 2.9056 | LR: 9.49e-07 | 1282.2ms/step | 15.6 samples/s +Step 42810/44000 | Loss: 2.9005 | LR: 9.33e-07 | 1281.2ms/step | 15.6 samples/s +Step 42820/44000 | Loss: 2.9572 | LR: 9.18e-07 | 1281.3ms/step | 15.6 samples/s +Step 42830/44000 | Loss: 2.9203 | LR: 9.02e-07 | 1283.1ms/step | 15.6 samples/s +Step 42840/44000 | Loss: 2.9306 | LR: 8.87e-07 | 1282.9ms/step | 15.6 samples/s +Step 42850/44000 | Loss: 2.9293 | LR: 8.72e-07 | 1285.7ms/step | 15.6 samples/s +Step 42860/44000 | Loss: 2.9379 | LR: 8.57e-07 | 1291.0ms/step | 15.5 samples/s +Step 42870/44000 | Loss: 2.9361 | LR: 8.42e-07 | 1288.4ms/step | 15.5 samples/s +Step 42880/44000 | Loss: 2.9233 | LR: 8.27e-07 | 1288.4ms/step | 15.5 samples/s +Step 42890/44000 | Loss: 2.8757 | LR: 8.12e-07 | 1288.9ms/step | 15.5 samples/s +Step 42900/44000 | Loss: 2.9325 | LR: 7.98e-07 | 1287.3ms/step | 15.5 samples/s +Step 42910/44000 | Loss: 2.9252 | LR: 7.83e-07 | 1285.7ms/step | 15.6 samples/s +Step 42920/44000 | Loss: 2.9051 | LR: 7.69e-07 | 1282.1ms/step | 15.6 samples/s +Step 42930/44000 | Loss: 2.9014 | LR: 7.55e-07 | 1284.7ms/step | 15.6 samples/s +Step 42940/44000 | Loss: 2.9128 | LR: 7.41e-07 | 1279.5ms/step | 15.6 samples/s +Step 42950/44000 | Loss: 2.9284 | LR: 7.27e-07 | 1286.8ms/step | 15.5 samples/s +Step 42960/44000 | Loss: 2.9190 | LR: 7.13e-07 | 1285.5ms/step | 15.6 samples/s +Step 42970/44000 | Loss: 2.9138 | LR: 6.99e-07 | 1292.0ms/step | 15.5 samples/s +Step 42980/44000 | Loss: 2.9257 | LR: 6.86e-07 | 1292.6ms/step | 15.5 samples/s +Step 42990/44000 | Loss: 2.9203 | LR: 6.72e-07 | 1290.6ms/step | 15.5 samples/s +Step 43000/44000 | Loss: 2.9113 | LR: 6.59e-07 | 1286.5ms/step | 15.5 samples/s +Validation at step 43000: Loss = 2.9343, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-43000 +Step 43010/44000 | Loss: 2.9183 | LR: 6.46e-07 | 3749.8ms/step | 5.3 samples/s +Step 43020/44000 | Loss: 2.9050 | LR: 6.33e-07 | 1283.4ms/step | 15.6 samples/s +Step 43030/44000 | Loss: 2.9249 | LR: 6.20e-07 | 1283.9ms/step | 15.6 samples/s +Step 43040/44000 | Loss: 2.8892 | LR: 6.08e-07 | 1281.9ms/step | 15.6 samples/s +Step 43050/44000 | Loss: 2.9151 | LR: 5.95e-07 | 1281.9ms/step | 15.6 samples/s +Step 43060/44000 | Loss: 2.9248 | LR: 5.83e-07 | 1284.0ms/step | 15.6 samples/s +Step 43070/44000 | Loss: 2.9396 | LR: 5.70e-07 | 1283.4ms/step | 15.6 samples/s +Step 43080/44000 | Loss: 2.9148 | LR: 5.58e-07 | 1287.5ms/step | 15.5 samples/s +Step 43090/44000 | Loss: 2.8965 | LR: 5.46e-07 | 1283.8ms/step | 15.6 samples/s +Step 43100/44000 | Loss: 2.9105 | LR: 5.34e-07 | 1286.7ms/step | 15.5 samples/s +Step 43110/44000 | Loss: 2.9223 | LR: 5.22e-07 | 1285.4ms/step | 15.6 samples/s +Step 43120/44000 | Loss: 2.9333 | LR: 5.11e-07 | 1286.5ms/step | 15.5 samples/s +Step 43130/44000 | Loss: 2.9360 | LR: 4.99e-07 | 1284.5ms/step | 15.6 samples/s +Step 43140/44000 | Loss: 2.9233 | LR: 4.88e-07 | 1286.9ms/step | 15.5 samples/s +Step 43150/44000 | Loss: 2.9082 | LR: 4.76e-07 | 1284.8ms/step | 15.6 samples/s +Step 43160/44000 | Loss: 2.9059 | LR: 4.65e-07 | 1283.9ms/step | 15.6 samples/s +Step 43170/44000 | Loss: 2.8943 | LR: 4.54e-07 | 1281.1ms/step | 15.6 samples/s +Step 43180/44000 | Loss: 2.9062 | LR: 4.43e-07 | 1283.5ms/step | 15.6 samples/s +Step 43190/44000 | Loss: 2.8999 | LR: 4.33e-07 | 1283.9ms/step | 15.6 samples/s +Step 43200/44000 | Loss: 2.8900 | LR: 4.22e-07 | 1284.4ms/step | 15.6 samples/s +Step 43210/44000 | Loss: 2.9256 | LR: 4.12e-07 | 1284.3ms/step | 15.6 samples/s +Step 43220/44000 | Loss: 2.9125 | LR: 4.01e-07 | 1284.7ms/step | 15.6 samples/s +Step 43230/44000 | Loss: 2.9347 | LR: 3.91e-07 | 1285.5ms/step | 15.6 samples/s +Step 43240/44000 | Loss: 2.9315 | LR: 3.81e-07 | 1284.3ms/step | 15.6 samples/s +Step 43250/44000 | Loss: 2.9344 | LR: 3.71e-07 | 1283.0ms/step | 15.6 samples/s +Validation at step 43250: Loss = 2.9343, Perplexity = 18.81 +Step 43260/44000 | Loss: 2.9265 | LR: 3.61e-07 | 3430.5ms/step | 5.8 samples/s +Step 43270/44000 | Loss: 2.9149 | LR: 3.51e-07 | 1284.0ms/step | 15.6 samples/s +Step 43280/44000 | Loss: 2.9217 | LR: 3.42e-07 | 1286.9ms/step | 15.5 samples/s +Step 43290/44000 | Loss: 2.9055 | LR: 3.32e-07 | 1286.6ms/step | 15.5 samples/s +Step 43300/44000 | Loss: 2.9383 | LR: 3.23e-07 | 1286.4ms/step | 15.5 samples/s +Step 43310/44000 | Loss: 2.9419 | LR: 3.14e-07 | 1286.1ms/step | 15.6 samples/s +Step 43320/44000 | Loss: 2.9472 | LR: 3.05e-07 | 1285.3ms/step | 15.6 samples/s +Step 43330/44000 | Loss: 2.9083 | LR: 2.96e-07 | 1284.4ms/step | 15.6 samples/s +Step 43340/44000 | Loss: 2.9298 | LR: 2.87e-07 | 1289.3ms/step | 15.5 samples/s +Step 43350/44000 | Loss: 2.9050 | LR: 2.79e-07 | 1287.0ms/step | 15.5 samples/s +Step 43360/44000 | Loss: 2.9257 | LR: 2.70e-07 | 1285.9ms/step | 15.6 samples/s +Step 43370/44000 | Loss: 2.9303 | LR: 2.62e-07 | 1286.5ms/step | 15.5 samples/s +Step 43380/44000 | Loss: 2.9241 | LR: 2.53e-07 | 1289.7ms/step | 15.5 samples/s +Step 43390/44000 | Loss: 2.8983 | LR: 2.45e-07 | 1291.1ms/step | 15.5 samples/s +Step 43400/44000 | Loss: 2.9155 | LR: 2.37e-07 | 1284.5ms/step | 15.6 samples/s +Step 43410/44000 | Loss: 2.9129 | LR: 2.30e-07 | 1283.1ms/step | 15.6 samples/s +Step 43420/44000 | Loss: 2.8936 | LR: 2.22e-07 | 1281.7ms/step | 15.6 samples/s +Step 43430/44000 | Loss: 2.9087 | LR: 2.14e-07 | 1281.3ms/step | 15.6 samples/s +Step 43440/44000 | Loss: 2.9557 | LR: 2.07e-07 | 1283.7ms/step | 15.6 samples/s +Step 43450/44000 | Loss: 2.9368 | LR: 1.99e-07 | 1283.5ms/step | 15.6 samples/s +Step 43460/44000 | Loss: 2.9266 | LR: 1.92e-07 | 1284.1ms/step | 15.6 samples/s +Step 43470/44000 | Loss: 2.9273 | LR: 1.85e-07 | 1284.0ms/step | 15.6 samples/s +Step 43480/44000 | Loss: 2.8955 | LR: 1.78e-07 | 1282.1ms/step | 15.6 samples/s +Step 43490/44000 | Loss: 2.9145 | LR: 1.72e-07 | 1282.2ms/step | 15.6 samples/s +Step 43500/44000 | Loss: 2.9551 | LR: 1.65e-07 | 1279.6ms/step | 15.6 samples/s +Validation at step 43500: Loss = 2.9343, Perplexity = 18.81 +Step 43510/44000 | Loss: 2.9205 | LR: 1.58e-07 | 3423.9ms/step | 5.8 samples/s +Step 43520/44000 | Loss: 2.9401 | LR: 1.52e-07 | 1279.8ms/step | 15.6 samples/s +Step 43530/44000 | Loss: 2.9041 | LR: 1.46e-07 | 1280.1ms/step | 15.6 samples/s +Step 43540/44000 | Loss: 2.8918 | LR: 1.40e-07 | 1281.5ms/step | 15.6 samples/s +Step 43550/44000 | Loss: 2.9142 | LR: 1.34e-07 | 1281.6ms/step | 15.6 samples/s +Step 43560/44000 | Loss: 2.9144 | LR: 1.28e-07 | 1283.4ms/step | 15.6 samples/s +Step 43570/44000 | Loss: 2.9126 | LR: 1.22e-07 | 1283.0ms/step | 15.6 samples/s +Step 43580/44000 | Loss: 2.9387 | LR: 1.16e-07 | 1284.1ms/step | 15.6 samples/s +Step 43590/44000 | Loss: 2.9141 | LR: 1.11e-07 | 1285.3ms/step | 15.6 samples/s +Step 43600/44000 | Loss: 2.9256 | LR: 1.06e-07 | 1285.5ms/step | 15.6 samples/s +Step 43610/44000 | Loss: 2.8730 | LR: 1.00e-07 | 1286.2ms/step | 15.5 samples/s +Step 43620/44000 | Loss: 2.9111 | LR: 9.52e-08 | 1286.8ms/step | 15.5 samples/s +Step 43630/44000 | Loss: 2.9033 | LR: 9.03e-08 | 1286.5ms/step | 15.5 samples/s +Step 43640/44000 | Loss: 2.9196 | LR: 8.55e-08 | 1286.3ms/step | 15.5 samples/s +Step 43650/44000 | Loss: 2.9215 | LR: 8.08e-08 | 1282.7ms/step | 15.6 samples/s +Step 43660/44000 | Loss: 2.9237 | LR: 7.62e-08 | 1278.5ms/step | 15.6 samples/s +Step 43670/44000 | Loss: 2.9311 | LR: 7.18e-08 | 1277.8ms/step | 15.7 samples/s +Step 43680/44000 | Loss: 2.9385 | LR: 6.75e-08 | 1277.7ms/step | 15.7 samples/s +Step 43690/44000 | Loss: 2.8994 | LR: 6.34e-08 | 1277.9ms/step | 15.7 samples/s +Step 43700/44000 | Loss: 2.9189 | LR: 5.94e-08 | 1277.0ms/step | 15.7 samples/s +Step 43710/44000 | Loss: 2.9386 | LR: 5.55e-08 | 1278.5ms/step | 15.6 samples/s +Step 43720/44000 | Loss: 2.9052 | LR: 5.17e-08 | 1283.4ms/step | 15.6 samples/s +Step 43730/44000 | Loss: 2.9443 | LR: 4.81e-08 | 1279.8ms/step | 15.6 samples/s +Step 43740/44000 | Loss: 2.9511 | LR: 4.46e-08 | 1283.1ms/step | 15.6 samples/s +Step 43750/44000 | Loss: 2.9215 | LR: 4.12e-08 | 1291.0ms/step | 15.5 samples/s +Validation at step 43750: Loss = 2.9343, Perplexity = 18.81 +Step 43760/44000 | Loss: 2.9402 | LR: 3.80e-08 | 3463.4ms/step | 5.8 samples/s +Step 43770/44000 | Loss: 2.9387 | LR: 3.49e-08 | 1285.5ms/step | 15.6 samples/s +Step 43780/44000 | Loss: 2.9203 | LR: 3.19e-08 | 1283.9ms/step | 15.6 samples/s +Step 43790/44000 | Loss: 2.9091 | LR: 2.91e-08 | 1284.7ms/step | 15.6 samples/s +Step 43800/44000 | Loss: 2.9118 | LR: 2.64e-08 | 1281.5ms/step | 15.6 samples/s +Step 43810/44000 | Loss: 2.9193 | LR: 2.38e-08 | 1281.8ms/step | 15.6 samples/s +Step 43820/44000 | Loss: 2.9113 | LR: 2.14e-08 | 1281.3ms/step | 15.6 samples/s +Step 43830/44000 | Loss: 2.9354 | LR: 1.91e-08 | 1282.0ms/step | 15.6 samples/s +Step 43840/44000 | Loss: 2.9201 | LR: 1.69e-08 | 1275.7ms/step | 15.7 samples/s +Step 43850/44000 | Loss: 2.9130 | LR: 1.48e-08 | 1279.3ms/step | 15.6 samples/s +Step 43860/44000 | Loss: 2.9230 | LR: 1.29e-08 | 1279.2ms/step | 15.6 samples/s +Step 43870/44000 | Loss: 2.9378 | LR: 1.11e-08 | 1280.0ms/step | 15.6 samples/s +Step 43880/44000 | Loss: 2.9542 | LR: 9.50e-09 | 1277.9ms/step | 15.7 samples/s +Step 43890/44000 | Loss: 2.9104 | LR: 7.98e-09 | 1274.6ms/step | 15.7 samples/s +Step 43900/44000 | Loss: 2.9200 | LR: 6.60e-09 | 1277.2ms/step | 15.7 samples/s +Step 43910/44000 | Loss: 2.9234 | LR: 5.34e-09 | 1278.3ms/step | 15.6 samples/s +Step 43920/44000 | Loss: 2.9083 | LR: 4.22e-09 | 1278.6ms/step | 15.6 samples/s +Step 43930/44000 | Loss: 2.9003 | LR: 3.23e-09 | 1282.7ms/step | 15.6 samples/s +Step 43940/44000 | Loss: 2.9094 | LR: 2.37e-09 | 1281.9ms/step | 15.6 samples/s +Step 43950/44000 | Loss: 2.9238 | LR: 1.65e-09 | 1282.0ms/step | 15.6 samples/s +Step 43960/44000 | Loss: 2.9014 | LR: 1.06e-09 | 1283.7ms/step | 15.6 samples/s +Step 43970/44000 | Loss: 2.9311 | LR: 5.94e-10 | 1284.7ms/step | 15.6 samples/s +Step 43980/44000 | Loss: 2.9328 | LR: 2.64e-10 | 1285.6ms/step | 15.6 samples/s +Step 43990/44000 | Loss: 2.9251 | LR: 6.60e-11 | 1286.2ms/step | 15.6 samples/s +Step 44000/44000 | Loss: 2.9672 | LR: 0.00e+00 | 1284.5ms/step | 15.6 samples/s +Validation at step 44000: Loss = 2.9343, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s1/run_20260707_034557/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss ███▇▇▆▆▅▅▅▄▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▇▆▆▅▅▄▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ██████████████▁███████████▁█████████████ +wandb: speed/steps_per_sec ██████████████████▁█████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.93426 +wandb: eval/perplexity 18.80758 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1284.54742 +wandb: speed/samples_per_sec 15.56969 +wandb: speed/steps_per_sec 0.77848 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m fixed s1 8k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/dbtl6d2o +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260707_170941-dbtl6d2o/logs +Training complete! diff --git a/epfl_final/logs/l8k365m_fixed_s2.log b/epfl_final/logs/l8k365m_fixed_s2.log new file mode 100644 index 0000000000000000000000000000000000000000..e82ae14e8b869765915a65138ed1eda3f13036dc --- /dev/null +++ b/epfl_final/logs/l8k365m_fixed_s2.log @@ -0,0 +1,1769 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m fixed s2 8k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "fixed", + "seed2" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_fixed_s2", + "sparse_mask_seed": 2, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_034618-lgjoqt2r +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m fixed s2 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/lgjoqt2r +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0409 | LR: 6.67e-06 | 6265.5ms/step | 3.2 samples/s +Step 20/44000 | Loss: 11.9259 | LR: 1.33e-05 | 1242.8ms/step | 16.1 samples/s +Step 30/44000 | Loss: 11.5559 | LR: 2.00e-05 | 1245.0ms/step | 16.1 samples/s +Step 40/44000 | Loss: 11.0136 | LR: 2.67e-05 | 1239.5ms/step | 16.1 samples/s +Step 50/44000 | Loss: 10.7337 | LR: 3.33e-05 | 1240.1ms/step | 16.1 samples/s +Step 60/44000 | Loss: 10.4900 | LR: 4.00e-05 | 1236.6ms/step | 16.2 samples/s +Step 70/44000 | Loss: 10.2545 | LR: 4.67e-05 | 1232.1ms/step | 16.2 samples/s +Step 80/44000 | Loss: 9.9531 | LR: 5.33e-05 | 1232.4ms/step | 16.2 samples/s +Step 90/44000 | Loss: 9.5934 | LR: 6.00e-05 | 1233.2ms/step | 16.2 samples/s +Step 100/44000 | Loss: 9.1493 | LR: 6.67e-05 | 1235.4ms/step | 16.2 samples/s +Step 110/44000 | Loss: 8.6193 | LR: 7.33e-05 | 1237.3ms/step | 16.2 samples/s +Step 120/44000 | Loss: 8.1614 | LR: 8.00e-05 | 1236.9ms/step | 16.2 samples/s +Step 130/44000 | Loss: 7.7753 | LR: 8.67e-05 | 1237.7ms/step | 16.2 samples/s +Step 140/44000 | Loss: 7.5379 | LR: 9.33e-05 | 1240.6ms/step | 16.1 samples/s +Step 150/44000 | Loss: 7.3517 | LR: 1.00e-04 | 1241.0ms/step | 16.1 samples/s +Step 160/44000 | Loss: 7.2426 | LR: 1.07e-04 | 1242.6ms/step | 16.1 samples/s +Step 170/44000 | Loss: 7.1297 | LR: 1.13e-04 | 1243.1ms/step | 16.1 samples/s +Step 180/44000 | Loss: 7.0381 | LR: 1.20e-04 | 1244.0ms/step | 16.1 samples/s +Step 190/44000 | Loss: 6.9451 | LR: 1.27e-04 | 1245.5ms/step | 16.1 samples/s +Step 200/44000 | Loss: 6.8813 | LR: 1.33e-04 | 1244.3ms/step | 16.1 samples/s +Step 210/44000 | Loss: 6.8274 | LR: 1.40e-04 | 1245.9ms/step | 16.1 samples/s +Step 220/44000 | Loss: 6.7155 | LR: 1.47e-04 | 1246.4ms/step | 16.0 samples/s +Step 230/44000 | Loss: 6.6087 | LR: 1.53e-04 | 1245.8ms/step | 16.1 samples/s +Step 240/44000 | Loss: 6.5402 | LR: 1.60e-04 | 1247.1ms/step | 16.0 samples/s +Step 250/44000 | Loss: 6.4655 | LR: 1.67e-04 | 1246.3ms/step | 16.0 samples/s +Validation at step 250: Loss = 6.4485, Perplexity = 631.78 +Step 260/44000 | Loss: 6.4265 | LR: 1.73e-04 | 5330.2ms/step | 3.8 samples/s +Step 270/44000 | Loss: 6.3490 | LR: 1.80e-04 | 1254.6ms/step | 15.9 samples/s +Step 280/44000 | Loss: 6.3112 | LR: 1.87e-04 | 1256.3ms/step | 15.9 samples/s +Step 290/44000 | Loss: 6.2811 | LR: 1.93e-04 | 1254.8ms/step | 15.9 samples/s +Step 300/44000 | Loss: 6.2338 | LR: 2.00e-04 | 1250.2ms/step | 16.0 samples/s +Step 310/44000 | Loss: 6.1903 | LR: 2.07e-04 | 1254.8ms/step | 15.9 samples/s +Step 320/44000 | Loss: 6.1380 | LR: 2.13e-04 | 1252.9ms/step | 16.0 samples/s +Step 330/44000 | Loss: 6.0864 | LR: 2.20e-04 | 1252.5ms/step | 16.0 samples/s +Step 340/44000 | Loss: 6.0643 | LR: 2.27e-04 | 1252.8ms/step | 16.0 samples/s +Step 350/44000 | Loss: 5.9890 | LR: 2.33e-04 | 1255.5ms/step | 15.9 samples/s +Step 360/44000 | Loss: 5.9563 | LR: 2.40e-04 | 1256.4ms/step | 15.9 samples/s +Step 370/44000 | Loss: 5.9348 | LR: 2.47e-04 | 1255.8ms/step | 15.9 samples/s +Step 380/44000 | Loss: 5.9258 | LR: 2.53e-04 | 1255.3ms/step | 15.9 samples/s +Step 390/44000 | Loss: 5.8386 | LR: 2.60e-04 | 1252.8ms/step | 16.0 samples/s +Step 400/44000 | Loss: 5.8425 | LR: 2.67e-04 | 1253.0ms/step | 16.0 samples/s +Step 410/44000 | Loss: 5.8030 | LR: 2.73e-04 | 1253.0ms/step | 16.0 samples/s +Step 420/44000 | Loss: 5.8033 | LR: 2.80e-04 | 1253.2ms/step | 16.0 samples/s +Step 430/44000 | Loss: 5.7300 | LR: 2.87e-04 | 1255.2ms/step | 15.9 samples/s +Step 440/44000 | Loss: 5.7085 | LR: 2.93e-04 | 1253.2ms/step | 16.0 samples/s +Step 450/44000 | Loss: 5.6799 | LR: 3.00e-04 | 1252.5ms/step | 16.0 samples/s +Step 460/44000 | Loss: 5.6017 | LR: 3.07e-04 | 1253.3ms/step | 16.0 samples/s +Step 470/44000 | Loss: 5.6134 | LR: 3.13e-04 | 1252.8ms/step | 16.0 samples/s +Step 480/44000 | Loss: 5.5606 | LR: 3.20e-04 | 1253.7ms/step | 16.0 samples/s +Step 490/44000 | Loss: 5.5130 | LR: 3.27e-04 | 1252.1ms/step | 16.0 samples/s +Step 500/44000 | Loss: 5.4994 | LR: 3.33e-04 | 1252.5ms/step | 16.0 samples/s +Validation at step 500: Loss = 5.4829, Perplexity = 240.55 +Step 510/44000 | Loss: 5.4660 | LR: 3.40e-04 | 3368.2ms/step | 5.9 samples/s +Step 520/44000 | Loss: 5.4230 | LR: 3.47e-04 | 1252.7ms/step | 16.0 samples/s +Step 530/44000 | Loss: 5.4440 | LR: 3.53e-04 | 1254.0ms/step | 15.9 samples/s +Step 540/44000 | Loss: 5.3651 | LR: 3.60e-04 | 1254.6ms/step | 15.9 samples/s +Step 550/44000 | Loss: 5.3353 | LR: 3.67e-04 | 1253.9ms/step | 16.0 samples/s +Step 560/44000 | Loss: 5.3123 | LR: 3.73e-04 | 1254.4ms/step | 15.9 samples/s +Step 570/44000 | Loss: 5.3157 | LR: 3.80e-04 | 1255.6ms/step | 15.9 samples/s +Step 580/44000 | Loss: 5.2635 | LR: 3.87e-04 | 1255.0ms/step | 15.9 samples/s +Step 590/44000 | Loss: 5.2155 | LR: 3.93e-04 | 1253.9ms/step | 15.9 samples/s +Step 600/44000 | Loss: 5.2116 | LR: 4.00e-04 | 1255.7ms/step | 15.9 samples/s +Step 610/44000 | Loss: 5.1683 | LR: 4.07e-04 | 1254.5ms/step | 15.9 samples/s +Step 620/44000 | Loss: 5.1555 | LR: 4.13e-04 | 1253.9ms/step | 16.0 samples/s +Step 630/44000 | Loss: 5.1051 | LR: 4.20e-04 | 1253.3ms/step | 16.0 samples/s +Step 640/44000 | Loss: 5.0687 | LR: 4.27e-04 | 1254.4ms/step | 15.9 samples/s +Step 650/44000 | Loss: 5.0528 | LR: 4.33e-04 | 1254.1ms/step | 15.9 samples/s +Step 660/44000 | Loss: 5.0386 | LR: 4.40e-04 | 1254.3ms/step | 15.9 samples/s +Step 670/44000 | Loss: 5.0128 | LR: 4.47e-04 | 1254.8ms/step | 15.9 samples/s +Step 680/44000 | Loss: 4.9599 | LR: 4.53e-04 | 1253.8ms/step | 16.0 samples/s +Step 690/44000 | Loss: 4.9519 | LR: 4.60e-04 | 1255.0ms/step | 15.9 samples/s +Step 700/44000 | Loss: 4.9385 | LR: 4.67e-04 | 1255.5ms/step | 15.9 samples/s +Step 710/44000 | Loss: 4.8909 | LR: 4.73e-04 | 1259.3ms/step | 15.9 samples/s +Step 720/44000 | Loss: 4.8310 | LR: 4.80e-04 | 1254.8ms/step | 15.9 samples/s +Step 730/44000 | Loss: 4.8285 | LR: 4.87e-04 | 1255.8ms/step | 15.9 samples/s +Step 740/44000 | Loss: 4.7766 | LR: 4.93e-04 | 1255.7ms/step | 15.9 samples/s +Step 750/44000 | Loss: 4.7916 | LR: 5.00e-04 | 1256.2ms/step | 15.9 samples/s +Validation at step 750: Loss = 4.7852, Perplexity = 119.72 +Step 760/44000 | Loss: 4.7616 | LR: 5.00e-04 | 3355.4ms/step | 6.0 samples/s +Step 770/44000 | Loss: 4.7184 | LR: 5.00e-04 | 1252.2ms/step | 16.0 samples/s +Step 780/44000 | Loss: 4.7242 | LR: 5.00e-04 | 1252.3ms/step | 16.0 samples/s +Step 790/44000 | Loss: 4.6954 | LR: 5.00e-04 | 1252.0ms/step | 16.0 samples/s +Step 800/44000 | Loss: 4.6529 | LR: 5.00e-04 | 1252.5ms/step | 16.0 samples/s +Step 810/44000 | Loss: 4.6387 | LR: 5.00e-04 | 1252.9ms/step | 16.0 samples/s +Step 820/44000 | Loss: 4.6194 | LR: 5.00e-04 | 1252.8ms/step | 16.0 samples/s +Step 830/44000 | Loss: 4.5906 | LR: 5.00e-04 | 1254.0ms/step | 15.9 samples/s +Step 840/44000 | Loss: 4.5641 | LR: 5.00e-04 | 1254.1ms/step | 15.9 samples/s +Step 850/44000 | Loss: 4.5619 | LR: 5.00e-04 | 1253.4ms/step | 16.0 samples/s +Step 860/44000 | Loss: 4.5770 | LR: 5.00e-04 | 1253.3ms/step | 16.0 samples/s +Step 870/44000 | Loss: 4.4972 | LR: 5.00e-04 | 1254.8ms/step | 15.9 samples/s +Step 880/44000 | Loss: 4.5100 | LR: 5.00e-04 | 1254.4ms/step | 15.9 samples/s +Step 890/44000 | Loss: 4.4851 | LR: 5.00e-04 | 1254.6ms/step | 15.9 samples/s +Step 900/44000 | Loss: 4.4595 | LR: 5.00e-04 | 1253.2ms/step | 16.0 samples/s +Step 910/44000 | Loss: 4.4334 | LR: 5.00e-04 | 1252.6ms/step | 16.0 samples/s +Step 920/44000 | Loss: 4.4303 | LR: 5.00e-04 | 1253.6ms/step | 16.0 samples/s +Step 930/44000 | Loss: 4.4030 | LR: 5.00e-04 | 1254.9ms/step | 15.9 samples/s +Step 940/44000 | Loss: 4.4001 | LR: 5.00e-04 | 1255.0ms/step | 15.9 samples/s +Step 950/44000 | Loss: 4.3887 | LR: 5.00e-04 | 1253.3ms/step | 16.0 samples/s +Step 960/44000 | Loss: 4.3453 | LR: 5.00e-04 | 1253.1ms/step | 16.0 samples/s +Step 970/44000 | Loss: 4.3746 | LR: 5.00e-04 | 1252.3ms/step | 16.0 samples/s +Step 980/44000 | Loss: 4.3555 | LR: 5.00e-04 | 1252.6ms/step | 16.0 samples/s +Step 990/44000 | Loss: 4.3344 | LR: 5.00e-04 | 1254.5ms/step | 15.9 samples/s +Step 1000/44000 | Loss: 4.3263 | LR: 5.00e-04 | 1252.9ms/step | 16.0 samples/s +Validation at step 1000: Loss = 4.3311, Perplexity = 76.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-1000 +Step 1010/44000 | Loss: 4.2880 | LR: 5.00e-04 | 3671.4ms/step | 5.4 samples/s +Step 1020/44000 | Loss: 4.2937 | LR: 5.00e-04 | 1259.8ms/step | 15.9 samples/s +Step 1030/44000 | Loss: 4.2464 | LR: 5.00e-04 | 1258.4ms/step | 15.9 samples/s +Step 1040/44000 | Loss: 4.2576 | LR: 5.00e-04 | 1259.8ms/step | 15.9 samples/s +Step 1050/44000 | Loss: 4.2562 | LR: 5.00e-04 | 1259.6ms/step | 15.9 samples/s +Step 1060/44000 | Loss: 4.2492 | LR: 5.00e-04 | 1254.0ms/step | 15.9 samples/s +Step 1070/44000 | Loss: 4.2300 | LR: 5.00e-04 | 1252.5ms/step | 16.0 samples/s +Step 1080/44000 | Loss: 4.2141 | LR: 5.00e-04 | 1254.7ms/step | 15.9 samples/s +Step 1090/44000 | Loss: 4.2336 | LR: 5.00e-04 | 1252.7ms/step | 16.0 samples/s +Step 1100/44000 | Loss: 4.2100 | LR: 5.00e-04 | 1252.7ms/step | 16.0 samples/s +Step 1110/44000 | Loss: 4.1927 | LR: 5.00e-04 | 1254.6ms/step | 15.9 samples/s +Step 1120/44000 | Loss: 4.1792 | LR: 5.00e-04 | 1253.3ms/step | 16.0 samples/s +Step 1130/44000 | Loss: 4.1495 | LR: 5.00e-04 | 1253.9ms/step | 16.0 samples/s +Step 1140/44000 | Loss: 4.1703 | LR: 5.00e-04 | 1254.1ms/step | 15.9 samples/s +Step 1150/44000 | Loss: 4.1198 | LR: 5.00e-04 | 1253.7ms/step | 16.0 samples/s +Step 1160/44000 | Loss: 4.1218 | LR: 5.00e-04 | 1255.7ms/step | 15.9 samples/s +Step 1170/44000 | Loss: 4.1285 | LR: 5.00e-04 | 1253.4ms/step | 16.0 samples/s +Step 1180/44000 | Loss: 4.1119 | LR: 5.00e-04 | 1253.9ms/step | 16.0 samples/s +Step 1190/44000 | Loss: 4.1343 | LR: 5.00e-04 | 1254.5ms/step | 15.9 samples/s +Step 1200/44000 | Loss: 4.1314 | LR: 5.00e-04 | 1254.7ms/step | 15.9 samples/s +Step 1210/44000 | Loss: 4.1339 | LR: 5.00e-04 | 1254.9ms/step | 15.9 samples/s +Step 1220/44000 | Loss: 4.0843 | LR: 5.00e-04 | 1254.6ms/step | 15.9 samples/s +Step 1230/44000 | Loss: 4.0747 | LR: 5.00e-04 | 1254.4ms/step | 15.9 samples/s +Step 1240/44000 | Loss: 4.0794 | LR: 5.00e-04 | 1253.7ms/step | 16.0 samples/s +Step 1250/44000 | Loss: 4.0852 | LR: 5.00e-04 | 1255.9ms/step | 15.9 samples/s +Validation at step 1250: Loss = 4.0796, Perplexity = 59.12 +Step 1260/44000 | Loss: 4.0536 | LR: 5.00e-04 | 3359.8ms/step | 6.0 samples/s +Step 1270/44000 | Loss: 4.0523 | LR: 5.00e-04 | 1252.9ms/step | 16.0 samples/s +Step 1280/44000 | Loss: 4.0625 | LR: 5.00e-04 | 1252.1ms/step | 16.0 samples/s +Step 1290/44000 | Loss: 4.0098 | LR: 5.00e-04 | 1253.0ms/step | 16.0 samples/s +Step 1300/44000 | Loss: 4.0578 | LR: 5.00e-04 | 1252.1ms/step | 16.0 samples/s +Step 1310/44000 | Loss: 3.9973 | LR: 5.00e-04 | 1253.4ms/step | 16.0 samples/s +Step 1320/44000 | Loss: 4.0260 | LR: 5.00e-04 | 1252.6ms/step | 16.0 samples/s +Step 1330/44000 | Loss: 4.0318 | LR: 5.00e-04 | 1251.4ms/step | 16.0 samples/s +Step 1340/44000 | Loss: 3.9929 | LR: 5.00e-04 | 1252.4ms/step | 16.0 samples/s +Step 1350/44000 | Loss: 3.9962 | LR: 5.00e-04 | 1252.6ms/step | 16.0 samples/s +Step 1360/44000 | Loss: 3.9911 | LR: 5.00e-04 | 1252.7ms/step | 16.0 samples/s +Step 1370/44000 | Loss: 3.9675 | LR: 5.00e-04 | 1252.5ms/step | 16.0 samples/s +Step 1380/44000 | Loss: 3.9802 | LR: 5.00e-04 | 1253.4ms/step | 16.0 samples/s +Step 1390/44000 | Loss: 3.9800 | LR: 5.00e-04 | 1254.1ms/step | 15.9 samples/s +Step 1400/44000 | Loss: 3.9765 | LR: 5.00e-04 | 1257.1ms/step | 15.9 samples/s +Step 1410/44000 | Loss: 3.9698 | LR: 5.00e-04 | 1254.6ms/step | 15.9 samples/s +Step 1420/44000 | Loss: 3.9594 | LR: 5.00e-04 | 1254.2ms/step | 15.9 samples/s +Step 1430/44000 | Loss: 3.9745 | LR: 5.00e-04 | 1254.1ms/step | 15.9 samples/s +Step 1440/44000 | Loss: 3.9701 | LR: 5.00e-04 | 1253.8ms/step | 16.0 samples/s +Step 1450/44000 | Loss: 3.9545 | LR: 5.00e-04 | 1253.2ms/step | 16.0 samples/s +Step 1460/44000 | Loss: 3.9522 | LR: 5.00e-04 | 1254.4ms/step | 15.9 samples/s +Step 1470/44000 | Loss: 3.9628 | LR: 5.00e-04 | 1255.8ms/step | 15.9 samples/s +Step 1480/44000 | Loss: 3.9561 | LR: 5.00e-04 | 1253.3ms/step | 16.0 samples/s +Step 1490/44000 | Loss: 3.9515 | LR: 5.00e-04 | 1254.5ms/step | 15.9 samples/s +Step 1500/44000 | Loss: 3.9300 | LR: 5.00e-04 | 1254.5ms/step | 15.9 samples/s +Validation at step 1500: Loss = 3.9269, Perplexity = 50.75 +Step 1510/44000 | Loss: 3.9367 | LR: 5.00e-04 | 3368.4ms/step | 5.9 samples/s +Step 1520/44000 | Loss: 3.9375 | LR: 5.00e-04 | 1255.1ms/step | 15.9 samples/s +Step 1530/44000 | Loss: 3.9082 | LR: 5.00e-04 | 1254.3ms/step | 15.9 samples/s +Step 1540/44000 | Loss: 3.9182 | LR: 5.00e-04 | 1254.6ms/step | 15.9 samples/s +Step 1550/44000 | Loss: 3.8827 | LR: 5.00e-04 | 1255.5ms/step | 15.9 samples/s +Step 1560/44000 | Loss: 3.8752 | LR: 5.00e-04 | 1254.1ms/step | 15.9 samples/s +Step 1570/44000 | Loss: 3.8689 | LR: 5.00e-04 | 1255.5ms/step | 15.9 samples/s +Step 1580/44000 | Loss: 3.8907 | LR: 5.00e-04 | 1255.0ms/step | 15.9 samples/s +Step 1590/44000 | Loss: 3.8622 | LR: 5.00e-04 | 1255.5ms/step | 15.9 samples/s +Step 1600/44000 | Loss: 3.8709 | LR: 5.00e-04 | 1255.3ms/step | 15.9 samples/s +Step 1610/44000 | Loss: 3.8507 | LR: 5.00e-04 | 1254.5ms/step | 15.9 samples/s +Step 1620/44000 | Loss: 3.8346 | LR: 5.00e-04 | 1255.0ms/step | 15.9 samples/s +Step 1630/44000 | Loss: 3.8508 | LR: 4.99e-04 | 1255.4ms/step | 15.9 samples/s +Step 1640/44000 | Loss: 3.8532 | LR: 4.99e-04 | 1254.6ms/step | 15.9 samples/s +Step 1650/44000 | Loss: 3.8286 | LR: 4.99e-04 | 1253.5ms/step | 16.0 samples/s +Step 1660/44000 | Loss: 3.8313 | LR: 4.99e-04 | 1253.3ms/step | 16.0 samples/s +Step 1670/44000 | Loss: 3.8248 | LR: 4.99e-04 | 1253.9ms/step | 16.0 samples/s +Step 1680/44000 | Loss: 3.8296 | LR: 4.99e-04 | 1252.9ms/step | 16.0 samples/s +Step 1690/44000 | Loss: 3.8175 | LR: 4.99e-04 | 1253.2ms/step | 16.0 samples/s +Step 1700/44000 | Loss: 3.8557 | LR: 4.99e-04 | 1252.9ms/step | 16.0 samples/s +Step 1710/44000 | Loss: 3.8037 | LR: 4.99e-04 | 1253.2ms/step | 16.0 samples/s +Step 1720/44000 | Loss: 3.8128 | LR: 4.99e-04 | 1258.3ms/step | 15.9 samples/s +Step 1730/44000 | Loss: 3.8145 | LR: 4.99e-04 | 1254.5ms/step | 15.9 samples/s +Step 1740/44000 | Loss: 3.7800 | LR: 4.99e-04 | 1253.8ms/step | 16.0 samples/s +Step 1750/44000 | Loss: 3.8003 | LR: 4.99e-04 | 1254.7ms/step | 15.9 samples/s +Validation at step 1750: Loss = 3.8160, Perplexity = 45.42 +Step 1760/44000 | Loss: 3.7850 | LR: 4.99e-04 | 3360.6ms/step | 6.0 samples/s +Step 1770/44000 | Loss: 3.7838 | LR: 4.99e-04 | 1253.2ms/step | 16.0 samples/s +Step 1780/44000 | Loss: 3.8197 | LR: 4.99e-04 | 1254.8ms/step | 15.9 samples/s +Step 1790/44000 | Loss: 3.7825 | LR: 4.99e-04 | 1254.8ms/step | 15.9 samples/s +Step 1800/44000 | Loss: 3.7805 | LR: 4.99e-04 | 1254.7ms/step | 15.9 samples/s +Step 1810/44000 | Loss: 3.7979 | LR: 4.99e-04 | 1254.3ms/step | 15.9 samples/s +Step 1820/44000 | Loss: 3.7875 | LR: 4.99e-04 | 1254.7ms/step | 15.9 samples/s +Step 1830/44000 | Loss: 3.7732 | LR: 4.99e-04 | 1254.4ms/step | 15.9 samples/s +Step 1840/44000 | Loss: 3.7605 | LR: 4.99e-04 | 1254.2ms/step | 15.9 samples/s +Step 1850/44000 | Loss: 3.7869 | LR: 4.99e-04 | 1253.2ms/step | 16.0 samples/s +Step 1860/44000 | Loss: 3.7727 | LR: 4.99e-04 | 1251.5ms/step | 16.0 samples/s +Step 1870/44000 | Loss: 3.7750 | LR: 4.99e-04 | 1252.4ms/step | 16.0 samples/s +Step 1880/44000 | Loss: 3.7758 | LR: 4.99e-04 | 1252.1ms/step | 16.0 samples/s +Step 1890/44000 | Loss: 3.7684 | LR: 4.99e-04 | 1254.6ms/step | 15.9 samples/s +Step 1900/44000 | Loss: 3.7772 | LR: 4.99e-04 | 1254.7ms/step | 15.9 samples/s +Step 1910/44000 | Loss: 3.7474 | LR: 4.99e-04 | 1254.1ms/step | 15.9 samples/s +Step 1920/44000 | Loss: 3.7669 | LR: 4.99e-04 | 1253.4ms/step | 16.0 samples/s +Step 1930/44000 | Loss: 3.7590 | LR: 4.99e-04 | 1253.9ms/step | 15.9 samples/s +Step 1940/44000 | Loss: 3.7545 | LR: 4.99e-04 | 1254.7ms/step | 15.9 samples/s +Step 1950/44000 | Loss: 3.7518 | LR: 4.99e-04 | 1254.8ms/step | 15.9 samples/s +Step 1960/44000 | Loss: 3.7385 | LR: 4.99e-04 | 1254.0ms/step | 15.9 samples/s +Step 1970/44000 | Loss: 3.7440 | LR: 4.99e-04 | 1254.4ms/step | 15.9 samples/s +Step 1980/44000 | Loss: 3.7214 | LR: 4.99e-04 | 1253.9ms/step | 16.0 samples/s +Step 1990/44000 | Loss: 3.7052 | LR: 4.99e-04 | 1254.9ms/step | 15.9 samples/s +Step 2000/44000 | Loss: 3.7150 | LR: 4.99e-04 | 1254.5ms/step | 15.9 samples/s +Validation at step 2000: Loss = 3.7340, Perplexity = 41.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-2000 +Step 2010/44000 | Loss: 3.6936 | LR: 4.99e-04 | 3669.1ms/step | 5.5 samples/s +Step 2020/44000 | Loss: 3.7522 | LR: 4.99e-04 | 1258.9ms/step | 15.9 samples/s +Step 2030/44000 | Loss: 3.7273 | LR: 4.99e-04 | 1260.2ms/step | 15.9 samples/s +Step 2040/44000 | Loss: 3.7123 | LR: 4.99e-04 | 1259.9ms/step | 15.9 samples/s +Step 2050/44000 | Loss: 3.7017 | LR: 4.99e-04 | 1258.4ms/step | 15.9 samples/s +Step 2060/44000 | Loss: 3.6740 | LR: 4.99e-04 | 1259.1ms/step | 15.9 samples/s +Step 2070/44000 | Loss: 3.6952 | LR: 4.99e-04 | 1257.3ms/step | 15.9 samples/s +Step 2080/44000 | Loss: 3.6658 | LR: 4.99e-04 | 1258.8ms/step | 15.9 samples/s +Step 2090/44000 | Loss: 3.7043 | LR: 4.99e-04 | 1258.4ms/step | 15.9 samples/s +Step 2100/44000 | Loss: 3.6952 | LR: 4.99e-04 | 1253.3ms/step | 16.0 samples/s +Step 2110/44000 | Loss: 3.7121 | LR: 4.99e-04 | 1251.5ms/step | 16.0 samples/s +Step 2120/44000 | Loss: 3.6988 | LR: 4.99e-04 | 1251.8ms/step | 16.0 samples/s +Step 2130/44000 | Loss: 3.7054 | LR: 4.99e-04 | 1252.6ms/step | 16.0 samples/s +Step 2140/44000 | Loss: 3.6588 | LR: 4.99e-04 | 1252.3ms/step | 16.0 samples/s +Step 2150/44000 | Loss: 3.6729 | LR: 4.99e-04 | 1252.0ms/step | 16.0 samples/s +Step 2160/44000 | Loss: 3.6794 | LR: 4.99e-04 | 1252.3ms/step | 16.0 samples/s +Step 2170/44000 | Loss: 3.6820 | LR: 4.99e-04 | 1251.0ms/step | 16.0 samples/s +Step 2180/44000 | Loss: 3.6659 | LR: 4.99e-04 | 1252.5ms/step | 16.0 samples/s +Step 2190/44000 | Loss: 3.6762 | LR: 4.99e-04 | 1252.1ms/step | 16.0 samples/s +Step 2200/44000 | Loss: 3.6500 | LR: 4.99e-04 | 1252.0ms/step | 16.0 samples/s +Step 2210/44000 | Loss: 3.6521 | LR: 4.99e-04 | 1252.3ms/step | 16.0 samples/s +Step 2220/44000 | Loss: 3.6331 | LR: 4.99e-04 | 1254.0ms/step | 15.9 samples/s +Step 2230/44000 | Loss: 3.6753 | LR: 4.99e-04 | 1254.0ms/step | 15.9 samples/s +Step 2240/44000 | Loss: 3.6549 | LR: 4.99e-04 | 1253.7ms/step | 16.0 samples/s +Step 2250/44000 | Loss: 3.6579 | LR: 4.99e-04 | 1254.2ms/step | 15.9 samples/s +Validation at step 2250: Loss = 3.6674, Perplexity = 39.15 +Step 2260/44000 | Loss: 3.6524 | LR: 4.98e-04 | 3368.7ms/step | 5.9 samples/s +Step 2270/44000 | Loss: 3.6399 | LR: 4.98e-04 | 1254.2ms/step | 15.9 samples/s +Step 2280/44000 | Loss: 3.6671 | LR: 4.98e-04 | 1254.5ms/step | 15.9 samples/s +Step 2290/44000 | Loss: 3.6570 | LR: 4.98e-04 | 1253.5ms/step | 16.0 samples/s +Step 2300/44000 | Loss: 3.6347 | LR: 4.98e-04 | 1253.2ms/step | 16.0 samples/s +Step 2310/44000 | Loss: 3.6494 | LR: 4.98e-04 | 1253.5ms/step | 16.0 samples/s +Step 2320/44000 | Loss: 3.6370 | LR: 4.98e-04 | 1253.0ms/step | 16.0 samples/s +Step 2330/44000 | Loss: 3.6400 | LR: 4.98e-04 | 1253.3ms/step | 16.0 samples/s +Step 2340/44000 | Loss: 3.6168 | LR: 4.98e-04 | 1252.9ms/step | 16.0 samples/s +Step 2350/44000 | Loss: 3.6274 | LR: 4.98e-04 | 1253.3ms/step | 16.0 samples/s +Step 2360/44000 | Loss: 3.6113 | LR: 4.98e-04 | 1253.3ms/step | 16.0 samples/s +Step 2370/44000 | Loss: 3.6513 | LR: 4.98e-04 | 1252.7ms/step | 16.0 samples/s +Step 2380/44000 | Loss: 3.6281 | LR: 4.98e-04 | 1252.2ms/step | 16.0 samples/s +Step 2390/44000 | Loss: 3.5932 | LR: 4.98e-04 | 1252.4ms/step | 16.0 samples/s +Step 2400/44000 | Loss: 3.6031 | LR: 4.98e-04 | 1254.7ms/step | 15.9 samples/s +Step 2410/44000 | Loss: 3.6470 | LR: 4.98e-04 | 1253.5ms/step | 16.0 samples/s +Step 2420/44000 | Loss: 3.6524 | LR: 4.98e-04 | 1252.6ms/step | 16.0 samples/s +Step 2430/44000 | Loss: 3.6205 | LR: 4.98e-04 | 1254.1ms/step | 15.9 samples/s +Step 2440/44000 | Loss: 3.5985 | LR: 4.98e-04 | 1254.6ms/step | 15.9 samples/s +Step 2450/44000 | Loss: 3.6272 | LR: 4.98e-04 | 1254.8ms/step | 15.9 samples/s +Step 2460/44000 | Loss: 3.6200 | LR: 4.98e-04 | 1254.6ms/step | 15.9 samples/s +Step 2470/44000 | Loss: 3.5978 | LR: 4.98e-04 | 1255.9ms/step | 15.9 samples/s +Step 2480/44000 | Loss: 3.6287 | LR: 4.98e-04 | 1255.4ms/step | 15.9 samples/s +Step 2490/44000 | Loss: 3.5845 | LR: 4.98e-04 | 1257.2ms/step | 15.9 samples/s +Step 2500/44000 | Loss: 3.6016 | LR: 4.98e-04 | 1257.6ms/step | 15.9 samples/s +Validation at step 2500: Loss = 3.6167, Perplexity = 37.21 +Step 2510/44000 | Loss: 3.5857 | LR: 4.98e-04 | 3372.1ms/step | 5.9 samples/s +Step 2520/44000 | Loss: 3.6035 | LR: 4.98e-04 | 1254.5ms/step | 15.9 samples/s +Step 2530/44000 | Loss: 3.6031 | LR: 4.98e-04 | 1255.5ms/step | 15.9 samples/s +Step 2540/44000 | Loss: 3.6152 | LR: 4.98e-04 | 1255.1ms/step | 15.9 samples/s +Step 2550/44000 | Loss: 3.6035 | LR: 4.98e-04 | 1255.3ms/step | 15.9 samples/s +Step 2560/44000 | Loss: 3.6150 | LR: 4.98e-04 | 1254.5ms/step | 15.9 samples/s +Step 2570/44000 | Loss: 3.5840 | LR: 4.98e-04 | 1254.7ms/step | 15.9 samples/s +Step 2580/44000 | Loss: 3.6228 | LR: 4.98e-04 | 1254.1ms/step | 15.9 samples/s +Step 2590/44000 | Loss: 3.5793 | LR: 4.98e-04 | 1254.6ms/step | 15.9 samples/s +Step 2600/44000 | Loss: 3.5864 | LR: 4.98e-04 | 1254.7ms/step | 15.9 samples/s +Step 2610/44000 | Loss: 3.5818 | LR: 4.98e-04 | 1255.2ms/step | 15.9 samples/s +Step 2620/44000 | Loss: 3.5849 | LR: 4.98e-04 | 1260.1ms/step | 15.9 samples/s +Step 2630/44000 | Loss: 3.5761 | LR: 4.98e-04 | 1254.8ms/step | 15.9 samples/s +Step 2640/44000 | Loss: 3.5891 | LR: 4.98e-04 | 1253.8ms/step | 16.0 samples/s +Step 2650/44000 | Loss: 3.6052 | LR: 4.98e-04 | 1255.0ms/step | 15.9 samples/s +Step 2660/44000 | Loss: 3.5748 | LR: 4.98e-04 | 1254.2ms/step | 15.9 samples/s +Step 2670/44000 | Loss: 3.5660 | LR: 4.98e-04 | 1254.1ms/step | 15.9 samples/s +Step 2680/44000 | Loss: 3.5869 | LR: 4.98e-04 | 1254.8ms/step | 15.9 samples/s +Step 2690/44000 | Loss: 3.5765 | LR: 4.98e-04 | 1254.5ms/step | 15.9 samples/s +Step 2700/44000 | Loss: 3.6088 | LR: 4.97e-04 | 1253.5ms/step | 16.0 samples/s +Step 2710/44000 | Loss: 3.5741 | LR: 4.97e-04 | 1254.1ms/step | 15.9 samples/s +Step 2720/44000 | Loss: 3.5761 | LR: 4.97e-04 | 1253.8ms/step | 16.0 samples/s +Step 2730/44000 | Loss: 3.5690 | LR: 4.97e-04 | 1252.8ms/step | 16.0 samples/s +Step 2740/44000 | Loss: 3.5699 | LR: 4.97e-04 | 1252.9ms/step | 16.0 samples/s +Step 2750/44000 | Loss: 3.5590 | LR: 4.97e-04 | 1253.1ms/step | 16.0 samples/s +Validation at step 2750: Loss = 3.5739, Perplexity = 35.66 +Step 2760/44000 | Loss: 3.5299 | LR: 4.97e-04 | 3361.3ms/step | 6.0 samples/s +Step 2770/44000 | Loss: 3.5561 | LR: 4.97e-04 | 1254.1ms/step | 15.9 samples/s +Step 2780/44000 | Loss: 3.5337 | LR: 4.97e-04 | 1253.5ms/step | 16.0 samples/s +Step 2790/44000 | Loss: 3.5584 | LR: 4.97e-04 | 1254.6ms/step | 15.9 samples/s +Step 2800/44000 | Loss: 3.5524 | LR: 4.97e-04 | 1253.8ms/step | 16.0 samples/s +Step 2810/44000 | Loss: 3.5361 | LR: 4.97e-04 | 1253.6ms/step | 16.0 samples/s +Step 2820/44000 | Loss: 3.5476 | LR: 4.97e-04 | 1253.2ms/step | 16.0 samples/s +Step 2830/44000 | Loss: 3.5231 | LR: 4.97e-04 | 1253.9ms/step | 16.0 samples/s +Step 2840/44000 | Loss: 3.5322 | LR: 4.97e-04 | 1253.8ms/step | 16.0 samples/s +Step 2850/44000 | Loss: 3.5410 | LR: 4.97e-04 | 1254.1ms/step | 15.9 samples/s +Step 2860/44000 | Loss: 3.5358 | LR: 4.97e-04 | 1254.0ms/step | 15.9 samples/s +Step 2870/44000 | Loss: 3.5458 | LR: 4.97e-04 | 1254.7ms/step | 15.9 samples/s +Step 2880/44000 | Loss: 3.5074 | LR: 4.97e-04 | 1254.5ms/step | 15.9 samples/s +Step 2890/44000 | Loss: 3.5507 | LR: 4.97e-04 | 1253.9ms/step | 15.9 samples/s +Step 2900/44000 | Loss: 3.5499 | LR: 4.97e-04 | 1258.3ms/step | 15.9 samples/s +Step 2910/44000 | Loss: 3.5219 | LR: 4.97e-04 | 1258.6ms/step | 15.9 samples/s +Step 2920/44000 | Loss: 3.5074 | LR: 4.97e-04 | 1256.0ms/step | 15.9 samples/s +Step 2930/44000 | Loss: 3.5044 | LR: 4.97e-04 | 1254.3ms/step | 15.9 samples/s +Step 2940/44000 | Loss: 3.5271 | LR: 4.97e-04 | 1254.5ms/step | 15.9 samples/s +Step 2950/44000 | Loss: 3.5282 | LR: 4.97e-04 | 1254.2ms/step | 15.9 samples/s +Step 2960/44000 | Loss: 3.5418 | LR: 4.97e-04 | 1255.3ms/step | 15.9 samples/s +Step 2970/44000 | Loss: 3.5255 | LR: 4.97e-04 | 1255.4ms/step | 15.9 samples/s +Step 2980/44000 | Loss: 3.5300 | LR: 4.97e-04 | 1255.4ms/step | 15.9 samples/s +Step 2990/44000 | Loss: 3.5159 | LR: 4.97e-04 | 1254.7ms/step | 15.9 samples/s +Step 3000/44000 | Loss: 3.5472 | LR: 4.97e-04 | 1255.3ms/step | 15.9 samples/s +Validation at step 3000: Loss = 3.5342, Perplexity = 34.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-3000 +Step 3010/44000 | Loss: 3.5246 | LR: 4.97e-04 | 3663.5ms/step | 5.5 samples/s +Step 3020/44000 | Loss: 3.5048 | LR: 4.97e-04 | 1257.4ms/step | 15.9 samples/s +Step 3030/44000 | Loss: 3.5246 | LR: 4.97e-04 | 1253.6ms/step | 16.0 samples/s +Step 3040/44000 | Loss: 3.4875 | LR: 4.97e-04 | 1253.4ms/step | 16.0 samples/s +Step 3050/44000 | Loss: 3.5133 | LR: 4.97e-04 | 1253.0ms/step | 16.0 samples/s +Step 3060/44000 | Loss: 3.4835 | LR: 4.96e-04 | 1252.9ms/step | 16.0 samples/s +Step 3070/44000 | Loss: 3.5314 | LR: 4.96e-04 | 1254.7ms/step | 15.9 samples/s +Step 3080/44000 | Loss: 3.5511 | LR: 4.96e-04 | 1254.9ms/step | 15.9 samples/s +Step 3090/44000 | Loss: 3.5414 | LR: 4.96e-04 | 1256.5ms/step | 15.9 samples/s +Step 3100/44000 | Loss: 3.4983 | LR: 4.96e-04 | 1258.0ms/step | 15.9 samples/s +Step 3110/44000 | Loss: 3.5007 | LR: 4.96e-04 | 1253.6ms/step | 16.0 samples/s +Step 3120/44000 | Loss: 3.5203 | LR: 4.96e-04 | 1252.5ms/step | 16.0 samples/s +Step 3130/44000 | Loss: 3.4889 | LR: 4.96e-04 | 1254.0ms/step | 15.9 samples/s +Step 3140/44000 | Loss: 3.5069 | LR: 4.96e-04 | 1254.5ms/step | 15.9 samples/s +Step 3150/44000 | Loss: 3.5088 | LR: 4.96e-04 | 1254.8ms/step | 15.9 samples/s +Step 3160/44000 | Loss: 3.5253 | LR: 4.96e-04 | 1253.8ms/step | 16.0 samples/s +Step 3170/44000 | Loss: 3.4655 | LR: 4.96e-04 | 1255.8ms/step | 15.9 samples/s +Step 3180/44000 | Loss: 3.5073 | LR: 4.96e-04 | 1255.9ms/step | 15.9 samples/s +Step 3190/44000 | Loss: 3.4954 | LR: 4.96e-04 | 1254.2ms/step | 15.9 samples/s +Step 3200/44000 | Loss: 3.5199 | LR: 4.96e-04 | 1254.6ms/step | 15.9 samples/s +Step 3210/44000 | Loss: 3.4932 | LR: 4.96e-04 | 1253.8ms/step | 16.0 samples/s +Step 3220/44000 | Loss: 3.4951 | LR: 4.96e-04 | 1258.0ms/step | 15.9 samples/s +Step 3230/44000 | Loss: 3.4940 | LR: 4.96e-04 | 1258.3ms/step | 15.9 samples/s +Step 3240/44000 | Loss: 3.4906 | LR: 4.96e-04 | 1259.2ms/step | 15.9 samples/s +Step 3250/44000 | Loss: 3.4945 | LR: 4.96e-04 | 1254.9ms/step | 15.9 samples/s +Validation at step 3250: Loss = 3.5003, Perplexity = 33.12 +Step 3260/44000 | Loss: 3.5124 | LR: 4.96e-04 | 3362.3ms/step | 5.9 samples/s +Step 3270/44000 | Loss: 3.4777 | LR: 4.96e-04 | 1253.2ms/step | 16.0 samples/s +Step 3280/44000 | Loss: 3.4660 | LR: 4.96e-04 | 1252.9ms/step | 16.0 samples/s +Step 3290/44000 | Loss: 3.4862 | LR: 4.96e-04 | 1252.7ms/step | 16.0 samples/s +Step 3300/44000 | Loss: 3.5151 | LR: 4.96e-04 | 1253.2ms/step | 16.0 samples/s +Step 3310/44000 | Loss: 3.4870 | LR: 4.96e-04 | 1252.7ms/step | 16.0 samples/s +Step 3320/44000 | Loss: 3.5104 | LR: 4.96e-04 | 1254.0ms/step | 15.9 samples/s +Step 3330/44000 | Loss: 3.5125 | LR: 4.96e-04 | 1252.8ms/step | 16.0 samples/s +Step 3340/44000 | Loss: 3.4707 | LR: 4.96e-04 | 1253.2ms/step | 16.0 samples/s +Step 3350/44000 | Loss: 3.4837 | LR: 4.96e-04 | 1253.7ms/step | 16.0 samples/s +Step 3360/44000 | Loss: 3.4664 | LR: 4.96e-04 | 1253.5ms/step | 16.0 samples/s +Step 3370/44000 | Loss: 3.4404 | LR: 4.95e-04 | 1253.3ms/step | 16.0 samples/s +Step 3380/44000 | Loss: 3.4868 | LR: 4.95e-04 | 1253.6ms/step | 16.0 samples/s +Step 3390/44000 | Loss: 3.4649 | LR: 4.95e-04 | 1254.0ms/step | 15.9 samples/s +Step 3400/44000 | Loss: 3.4982 | LR: 4.95e-04 | 1253.1ms/step | 16.0 samples/s +Step 3410/44000 | Loss: 3.4718 | LR: 4.95e-04 | 1255.2ms/step | 15.9 samples/s +Step 3420/44000 | Loss: 3.4825 | LR: 4.95e-04 | 1255.8ms/step | 15.9 samples/s +Step 3430/44000 | Loss: 3.4741 | LR: 4.95e-04 | 1256.8ms/step | 15.9 samples/s +Step 3440/44000 | Loss: 3.4657 | LR: 4.95e-04 | 1257.4ms/step | 15.9 samples/s +Step 3450/44000 | Loss: 3.4500 | LR: 4.95e-04 | 1252.3ms/step | 16.0 samples/s +Step 3460/44000 | Loss: 3.4878 | LR: 4.95e-04 | 1251.0ms/step | 16.0 samples/s +Step 3470/44000 | Loss: 3.4584 | LR: 4.95e-04 | 1252.5ms/step | 16.0 samples/s +Step 3480/44000 | Loss: 3.5026 | LR: 4.95e-04 | 1252.8ms/step | 16.0 samples/s +Step 3490/44000 | Loss: 3.4382 | LR: 4.95e-04 | 1253.6ms/step | 16.0 samples/s +Step 3500/44000 | Loss: 3.4454 | LR: 4.95e-04 | 1253.6ms/step | 16.0 samples/s +Validation at step 3500: Loss = 3.4723, Perplexity = 32.21 +Step 3510/44000 | Loss: 3.4773 | LR: 4.95e-04 | 3369.3ms/step | 5.9 samples/s +Step 3520/44000 | Loss: 3.4655 | LR: 4.95e-04 | 1255.3ms/step | 15.9 samples/s +Step 3530/44000 | Loss: 3.4618 | LR: 4.95e-04 | 1255.9ms/step | 15.9 samples/s +Step 3540/44000 | Loss: 3.4713 | LR: 4.95e-04 | 1259.2ms/step | 15.9 samples/s +Step 3550/44000 | Loss: 3.4451 | LR: 4.95e-04 | 1253.9ms/step | 15.9 samples/s +Step 3560/44000 | Loss: 3.4320 | LR: 4.95e-04 | 1255.0ms/step | 15.9 samples/s +Step 3570/44000 | Loss: 3.4578 | LR: 4.95e-04 | 1255.7ms/step | 15.9 samples/s +Step 3580/44000 | Loss: 3.4349 | LR: 4.95e-04 | 1254.5ms/step | 15.9 samples/s +Step 3590/44000 | Loss: 3.4476 | LR: 4.95e-04 | 1254.7ms/step | 15.9 samples/s +Step 3600/44000 | Loss: 3.4373 | LR: 4.95e-04 | 1254.2ms/step | 15.9 samples/s +Step 3610/44000 | Loss: 3.4501 | LR: 4.95e-04 | 1254.8ms/step | 15.9 samples/s +Step 3620/44000 | Loss: 3.4488 | LR: 4.95e-04 | 1253.5ms/step | 16.0 samples/s +Step 3630/44000 | Loss: 3.4399 | LR: 4.95e-04 | 1254.3ms/step | 15.9 samples/s +Step 3640/44000 | Loss: 3.4506 | LR: 4.95e-04 | 1254.3ms/step | 15.9 samples/s +Step 3650/44000 | Loss: 3.4375 | LR: 4.94e-04 | 1253.0ms/step | 16.0 samples/s +Step 3660/44000 | Loss: 3.4374 | LR: 4.94e-04 | 1254.0ms/step | 15.9 samples/s +Step 3670/44000 | Loss: 3.4310 | LR: 4.94e-04 | 1253.7ms/step | 16.0 samples/s +Step 3680/44000 | Loss: 3.4440 | LR: 4.94e-04 | 1254.1ms/step | 15.9 samples/s +Step 3690/44000 | Loss: 3.4387 | LR: 4.94e-04 | 1254.2ms/step | 15.9 samples/s +Step 3700/44000 | Loss: 3.4526 | LR: 4.94e-04 | 1254.1ms/step | 15.9 samples/s +Step 3710/44000 | Loss: 3.4475 | LR: 4.94e-04 | 1254.0ms/step | 15.9 samples/s +Step 3720/44000 | Loss: 3.4005 | LR: 4.94e-04 | 1254.8ms/step | 15.9 samples/s +Step 3730/44000 | Loss: 3.4512 | LR: 4.94e-04 | 1255.5ms/step | 15.9 samples/s +Step 3740/44000 | Loss: 3.4452 | LR: 4.94e-04 | 1255.8ms/step | 15.9 samples/s +Step 3750/44000 | Loss: 3.4509 | LR: 4.94e-04 | 1255.0ms/step | 15.9 samples/s +Validation at step 3750: Loss = 3.4434, Perplexity = 31.29 +Step 3760/44000 | Loss: 3.4311 | LR: 4.94e-04 | 3364.6ms/step | 5.9 samples/s +Step 3770/44000 | Loss: 3.4259 | LR: 4.94e-04 | 1255.7ms/step | 15.9 samples/s +Step 3780/44000 | Loss: 3.4202 | LR: 4.94e-04 | 1255.8ms/step | 15.9 samples/s +Step 3790/44000 | Loss: 3.4418 | LR: 4.94e-04 | 1253.0ms/step | 16.0 samples/s +Step 3800/44000 | Loss: 3.4228 | LR: 4.94e-04 | 1254.2ms/step | 15.9 samples/s +Step 3810/44000 | Loss: 3.4289 | LR: 4.94e-04 | 1255.5ms/step | 15.9 samples/s +Step 3820/44000 | Loss: 3.4360 | LR: 4.94e-04 | 1254.9ms/step | 15.9 samples/s +Step 3830/44000 | Loss: 3.4284 | LR: 4.94e-04 | 1259.0ms/step | 15.9 samples/s +Step 3840/44000 | Loss: 3.4143 | LR: 4.94e-04 | 1257.9ms/step | 15.9 samples/s +Step 3850/44000 | Loss: 3.4235 | LR: 4.94e-04 | 1255.0ms/step | 15.9 samples/s +Step 3860/44000 | Loss: 3.4225 | LR: 4.94e-04 | 1254.4ms/step | 15.9 samples/s +Step 3870/44000 | Loss: 3.4274 | LR: 4.94e-04 | 1255.4ms/step | 15.9 samples/s +Step 3880/44000 | Loss: 3.3963 | LR: 4.94e-04 | 1255.5ms/step | 15.9 samples/s +Step 3890/44000 | Loss: 3.4514 | LR: 4.94e-04 | 1255.0ms/step | 15.9 samples/s +Step 3900/44000 | Loss: 3.4114 | LR: 4.93e-04 | 1254.7ms/step | 15.9 samples/s +Step 3910/44000 | Loss: 3.4020 | LR: 4.93e-04 | 1254.3ms/step | 15.9 samples/s +Step 3920/44000 | Loss: 3.4056 | LR: 4.93e-04 | 1254.9ms/step | 15.9 samples/s +Step 3930/44000 | Loss: 3.4158 | LR: 4.93e-04 | 1254.6ms/step | 15.9 samples/s +Step 3940/44000 | Loss: 3.4158 | LR: 4.93e-04 | 1254.2ms/step | 15.9 samples/s +Step 3950/44000 | Loss: 3.4418 | LR: 4.93e-04 | 1253.5ms/step | 16.0 samples/s +Step 3960/44000 | Loss: 3.4328 | LR: 4.93e-04 | 1254.1ms/step | 15.9 samples/s +Step 3970/44000 | Loss: 3.3867 | LR: 4.93e-04 | 1256.8ms/step | 15.9 samples/s +Step 3980/44000 | Loss: 3.3972 | LR: 4.93e-04 | 1258.0ms/step | 15.9 samples/s +Step 3990/44000 | Loss: 3.4266 | LR: 4.93e-04 | 1254.5ms/step | 15.9 samples/s +Step 4000/44000 | Loss: 3.4089 | LR: 4.93e-04 | 1254.8ms/step | 15.9 samples/s +Validation at step 4000: Loss = 3.4226, Perplexity = 30.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-4000 +Step 4010/44000 | Loss: 3.4591 | LR: 4.93e-04 | 3673.4ms/step | 5.4 samples/s +Step 4020/44000 | Loss: 3.4204 | LR: 4.93e-04 | 1256.7ms/step | 15.9 samples/s +Step 4030/44000 | Loss: 3.3958 | LR: 4.93e-04 | 1258.5ms/step | 15.9 samples/s +Step 4040/44000 | Loss: 3.4120 | LR: 4.93e-04 | 1259.9ms/step | 15.9 samples/s +Step 4050/44000 | Loss: 3.4459 | LR: 4.93e-04 | 1260.1ms/step | 15.9 samples/s +Step 4060/44000 | Loss: 3.3924 | LR: 4.93e-04 | 1259.6ms/step | 15.9 samples/s +Step 4070/44000 | Loss: 3.4128 | LR: 4.93e-04 | 1259.4ms/step | 15.9 samples/s +Step 4080/44000 | Loss: 3.4310 | LR: 4.93e-04 | 1259.2ms/step | 15.9 samples/s +Step 4090/44000 | Loss: 3.3823 | LR: 4.93e-04 | 1260.7ms/step | 15.9 samples/s +Step 4100/44000 | Loss: 3.4136 | LR: 4.93e-04 | 1259.8ms/step | 15.9 samples/s +Step 4110/44000 | Loss: 3.3845 | LR: 4.93e-04 | 1260.0ms/step | 15.9 samples/s +Step 4120/44000 | Loss: 3.4122 | LR: 4.93e-04 | 1260.1ms/step | 15.9 samples/s +Step 4130/44000 | Loss: 3.4126 | LR: 4.93e-04 | 1259.6ms/step | 15.9 samples/s +Step 4140/44000 | Loss: 3.3939 | LR: 4.92e-04 | 1260.8ms/step | 15.9 samples/s +Step 4150/44000 | Loss: 3.4397 | LR: 4.92e-04 | 1260.3ms/step | 15.9 samples/s +Step 4160/44000 | Loss: 3.4196 | LR: 4.92e-04 | 1260.9ms/step | 15.9 samples/s +Step 4170/44000 | Loss: 3.4035 | LR: 4.92e-04 | 1260.2ms/step | 15.9 samples/s +Step 4180/44000 | Loss: 3.3823 | LR: 4.92e-04 | 1261.2ms/step | 15.9 samples/s +Step 4190/44000 | Loss: 3.3938 | LR: 4.92e-04 | 1262.1ms/step | 15.8 samples/s +Step 4200/44000 | Loss: 3.3848 | LR: 4.92e-04 | 1260.3ms/step | 15.9 samples/s +Step 4210/44000 | Loss: 3.4183 | LR: 4.92e-04 | 1260.2ms/step | 15.9 samples/s +Step 4220/44000 | Loss: 3.3834 | LR: 4.92e-04 | 1261.2ms/step | 15.9 samples/s +Step 4230/44000 | Loss: 3.4043 | LR: 4.92e-04 | 1264.2ms/step | 15.8 samples/s +Step 4240/44000 | Loss: 3.4637 | LR: 4.92e-04 | 1262.5ms/step | 15.8 samples/s +Step 4250/44000 | Loss: 3.4365 | LR: 4.92e-04 | 1259.6ms/step | 15.9 samples/s +Validation at step 4250: Loss = 3.4054, Perplexity = 30.13 +Step 4260/44000 | Loss: 3.4098 | LR: 4.92e-04 | 3388.7ms/step | 5.9 samples/s +Step 4270/44000 | Loss: 3.3455 | LR: 4.92e-04 | 1261.6ms/step | 15.9 samples/s +Step 4280/44000 | Loss: 3.3693 | LR: 4.92e-04 | 1260.3ms/step | 15.9 samples/s +Step 4290/44000 | Loss: 3.4016 | LR: 4.92e-04 | 1259.8ms/step | 15.9 samples/s +Step 4300/44000 | Loss: 3.3903 | LR: 4.92e-04 | 1260.1ms/step | 15.9 samples/s +Step 4310/44000 | Loss: 3.3925 | LR: 4.92e-04 | 1260.2ms/step | 15.9 samples/s +Step 4320/44000 | Loss: 3.3976 | LR: 4.92e-04 | 1259.2ms/step | 15.9 samples/s +Step 4330/44000 | Loss: 3.3846 | LR: 4.92e-04 | 1259.4ms/step | 15.9 samples/s +Step 4340/44000 | Loss: 3.3995 | LR: 4.92e-04 | 1258.8ms/step | 15.9 samples/s +Step 4350/44000 | Loss: 3.4065 | LR: 4.92e-04 | 1258.8ms/step | 15.9 samples/s +Step 4360/44000 | Loss: 3.3550 | LR: 4.91e-04 | 1259.2ms/step | 15.9 samples/s +Step 4370/44000 | Loss: 3.3745 | LR: 4.91e-04 | 1260.0ms/step | 15.9 samples/s +Step 4380/44000 | Loss: 3.4165 | LR: 4.91e-04 | 1260.7ms/step | 15.9 samples/s +Step 4390/44000 | Loss: 3.4052 | LR: 4.91e-04 | 1261.9ms/step | 15.8 samples/s +Step 4400/44000 | Loss: 3.4048 | LR: 4.91e-04 | 1261.2ms/step | 15.9 samples/s +Step 4410/44000 | Loss: 3.3773 | LR: 4.91e-04 | 1260.5ms/step | 15.9 samples/s +Step 4420/44000 | Loss: 3.3847 | LR: 4.91e-04 | 1261.2ms/step | 15.9 samples/s +Step 4430/44000 | Loss: 3.3872 | LR: 4.91e-04 | 1261.1ms/step | 15.9 samples/s +Step 4440/44000 | Loss: 3.3960 | LR: 4.91e-04 | 1260.0ms/step | 15.9 samples/s +Step 4450/44000 | Loss: 3.3875 | LR: 4.91e-04 | 1262.7ms/step | 15.8 samples/s +Step 4460/44000 | Loss: 3.3864 | LR: 4.91e-04 | 1261.0ms/step | 15.9 samples/s +Step 4470/44000 | Loss: 3.3500 | LR: 4.91e-04 | 1259.8ms/step | 15.9 samples/s +Step 4480/44000 | Loss: 3.3846 | LR: 4.91e-04 | 1260.3ms/step | 15.9 samples/s +Step 4490/44000 | Loss: 3.3900 | LR: 4.91e-04 | 1259.6ms/step | 15.9 samples/s +Step 4500/44000 | Loss: 3.3802 | LR: 4.91e-04 | 1259.7ms/step | 15.9 samples/s +Validation at step 4500: Loss = 3.3855, Perplexity = 29.53 +Step 4510/44000 | Loss: 3.3739 | LR: 4.91e-04 | 3383.4ms/step | 5.9 samples/s +Step 4520/44000 | Loss: 3.3909 | LR: 4.91e-04 | 1261.0ms/step | 15.9 samples/s +Step 4530/44000 | Loss: 3.3936 | LR: 4.91e-04 | 1260.2ms/step | 15.9 samples/s +Step 4540/44000 | Loss: 3.3543 | LR: 4.91e-04 | 1260.5ms/step | 15.9 samples/s +Step 4550/44000 | Loss: 3.3481 | LR: 4.91e-04 | 1261.4ms/step | 15.9 samples/s +Step 4560/44000 | Loss: 3.3554 | LR: 4.90e-04 | 1260.2ms/step | 15.9 samples/s +Step 4570/44000 | Loss: 3.3746 | LR: 4.90e-04 | 1257.5ms/step | 15.9 samples/s +Step 4580/44000 | Loss: 3.3334 | LR: 4.90e-04 | 1255.4ms/step | 15.9 samples/s +Step 4590/44000 | Loss: 3.3343 | LR: 4.90e-04 | 1255.0ms/step | 15.9 samples/s +Step 4600/44000 | Loss: 3.3546 | LR: 4.90e-04 | 1254.3ms/step | 15.9 samples/s +Step 4610/44000 | Loss: 3.3621 | LR: 4.90e-04 | 1254.6ms/step | 15.9 samples/s +Step 4620/44000 | Loss: 3.3530 | LR: 4.90e-04 | 1253.9ms/step | 16.0 samples/s +Step 4630/44000 | Loss: 3.3646 | LR: 4.90e-04 | 1256.0ms/step | 15.9 samples/s +Step 4640/44000 | Loss: 3.3753 | LR: 4.90e-04 | 1254.8ms/step | 15.9 samples/s +Step 4650/44000 | Loss: 3.3630 | LR: 4.90e-04 | 1254.7ms/step | 15.9 samples/s +Step 4660/44000 | Loss: 3.3417 | LR: 4.90e-04 | 1254.6ms/step | 15.9 samples/s +Step 4670/44000 | Loss: 3.3351 | LR: 4.90e-04 | 1254.6ms/step | 15.9 samples/s +Step 4680/44000 | Loss: 3.3577 | LR: 4.90e-04 | 1252.9ms/step | 16.0 samples/s +Step 4690/44000 | Loss: 3.3512 | LR: 4.90e-04 | 1255.4ms/step | 15.9 samples/s +Step 4700/44000 | Loss: 3.3859 | LR: 4.90e-04 | 1253.8ms/step | 16.0 samples/s +Step 4710/44000 | Loss: 3.3610 | LR: 4.90e-04 | 1252.6ms/step | 16.0 samples/s +Step 4720/44000 | Loss: 3.3408 | LR: 4.90e-04 | 1252.6ms/step | 16.0 samples/s +Step 4730/44000 | Loss: 3.3610 | LR: 4.90e-04 | 1253.7ms/step | 16.0 samples/s +Step 4740/44000 | Loss: 3.3508 | LR: 4.90e-04 | 1253.8ms/step | 16.0 samples/s +Step 4750/44000 | Loss: 3.3604 | LR: 4.90e-04 | 1254.3ms/step | 15.9 samples/s +Validation at step 4750: Loss = 3.3646, Perplexity = 28.92 +Step 4760/44000 | Loss: 3.3384 | LR: 4.89e-04 | 3361.3ms/step | 6.0 samples/s +Step 4770/44000 | Loss: 3.3633 | LR: 4.89e-04 | 1252.4ms/step | 16.0 samples/s +Step 4780/44000 | Loss: 3.3606 | LR: 4.89e-04 | 1253.2ms/step | 16.0 samples/s +Step 4790/44000 | Loss: 3.3330 | LR: 4.89e-04 | 1253.1ms/step | 16.0 samples/s +Step 4800/44000 | Loss: 3.3557 | LR: 4.89e-04 | 1253.3ms/step | 16.0 samples/s +Step 4810/44000 | Loss: 3.3550 | LR: 4.89e-04 | 1252.8ms/step | 16.0 samples/s +Step 4820/44000 | Loss: 3.3574 | LR: 4.89e-04 | 1254.0ms/step | 15.9 samples/s +Step 4830/44000 | Loss: 3.3455 | LR: 4.89e-04 | 1253.9ms/step | 16.0 samples/s +Step 4840/44000 | Loss: 3.3574 | LR: 4.89e-04 | 1254.5ms/step | 15.9 samples/s +Step 4850/44000 | Loss: 3.3596 | LR: 4.89e-04 | 1254.1ms/step | 15.9 samples/s +Step 4860/44000 | Loss: 3.3676 | LR: 4.89e-04 | 1254.0ms/step | 15.9 samples/s +Step 4870/44000 | Loss: 3.3317 | LR: 4.89e-04 | 1255.2ms/step | 15.9 samples/s +Step 4880/44000 | Loss: 3.3320 | LR: 4.89e-04 | 1254.3ms/step | 15.9 samples/s +Step 4890/44000 | Loss: 3.3222 | LR: 4.89e-04 | 1254.4ms/step | 15.9 samples/s +Step 4900/44000 | Loss: 3.3053 | LR: 4.89e-04 | 1255.1ms/step | 15.9 samples/s +Step 4910/44000 | Loss: 3.3416 | LR: 4.89e-04 | 1255.3ms/step | 15.9 samples/s +Step 4920/44000 | Loss: 3.3424 | LR: 4.89e-04 | 1254.5ms/step | 15.9 samples/s +Step 4930/44000 | Loss: 3.3258 | LR: 4.89e-04 | 1254.5ms/step | 15.9 samples/s +Step 4940/44000 | Loss: 3.3353 | LR: 4.89e-04 | 1254.2ms/step | 15.9 samples/s +Step 4950/44000 | Loss: 3.3388 | LR: 4.88e-04 | 1254.8ms/step | 15.9 samples/s +Step 4960/44000 | Loss: 3.3277 | LR: 4.88e-04 | 1253.8ms/step | 16.0 samples/s +Step 4970/44000 | Loss: 3.3195 | LR: 4.88e-04 | 1254.2ms/step | 15.9 samples/s +Step 4980/44000 | Loss: 3.3165 | LR: 4.88e-04 | 1254.5ms/step | 15.9 samples/s +Step 4990/44000 | Loss: 3.3420 | LR: 4.88e-04 | 1253.8ms/step | 16.0 samples/s +Step 5000/44000 | Loss: 3.3407 | LR: 4.88e-04 | 1253.9ms/step | 15.9 samples/s +Validation at step 5000: Loss = 3.3502, Perplexity = 28.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-5000 +Step 5010/44000 | Loss: 3.3482 | LR: 4.88e-04 | 3659.4ms/step | 5.5 samples/s +Step 5020/44000 | Loss: 3.3284 | LR: 4.88e-04 | 1259.0ms/step | 15.9 samples/s +Step 5030/44000 | Loss: 3.3363 | LR: 4.88e-04 | 1259.8ms/step | 15.9 samples/s +Step 5040/44000 | Loss: 3.3455 | LR: 4.88e-04 | 1260.1ms/step | 15.9 samples/s +Step 5050/44000 | Loss: 3.3199 | LR: 4.88e-04 | 1259.8ms/step | 15.9 samples/s +Step 5060/44000 | Loss: 3.3495 | LR: 4.88e-04 | 1259.9ms/step | 15.9 samples/s +Step 5070/44000 | Loss: 3.3541 | LR: 4.88e-04 | 1259.9ms/step | 15.9 samples/s +Step 5080/44000 | Loss: 3.3447 | LR: 4.88e-04 | 1260.2ms/step | 15.9 samples/s +Step 5090/44000 | Loss: 3.3172 | LR: 4.88e-04 | 1260.3ms/step | 15.9 samples/s +Step 5100/44000 | Loss: 3.3446 | LR: 4.88e-04 | 1259.5ms/step | 15.9 samples/s +Step 5110/44000 | Loss: 3.3629 | LR: 4.88e-04 | 1261.0ms/step | 15.9 samples/s +Step 5120/44000 | Loss: 3.3601 | LR: 4.88e-04 | 1261.3ms/step | 15.9 samples/s +Step 5130/44000 | Loss: 3.3416 | LR: 4.87e-04 | 1259.3ms/step | 15.9 samples/s +Step 5140/44000 | Loss: 3.3171 | LR: 4.87e-04 | 1258.7ms/step | 15.9 samples/s +Step 5150/44000 | Loss: 3.3606 | LR: 4.87e-04 | 1260.0ms/step | 15.9 samples/s +Step 5160/44000 | Loss: 3.3589 | LR: 4.87e-04 | 1259.8ms/step | 15.9 samples/s +Step 5170/44000 | Loss: 3.3483 | LR: 4.87e-04 | 1259.8ms/step | 15.9 samples/s +Step 5180/44000 | Loss: 3.3070 | LR: 4.87e-04 | 1260.5ms/step | 15.9 samples/s +Step 5190/44000 | Loss: 3.3334 | LR: 4.87e-04 | 1259.7ms/step | 15.9 samples/s +Step 5200/44000 | Loss: 3.3021 | LR: 4.87e-04 | 1261.7ms/step | 15.9 samples/s +Step 5210/44000 | Loss: 3.3374 | LR: 4.87e-04 | 1260.5ms/step | 15.9 samples/s +Step 5220/44000 | Loss: 3.3314 | LR: 4.87e-04 | 1260.2ms/step | 15.9 samples/s +Step 5230/44000 | Loss: 3.2845 | LR: 4.87e-04 | 1261.1ms/step | 15.9 samples/s +Step 5240/44000 | Loss: 3.3145 | LR: 4.87e-04 | 1260.1ms/step | 15.9 samples/s +Step 5250/44000 | Loss: 3.3148 | LR: 4.87e-04 | 1259.9ms/step | 15.9 samples/s +Validation at step 5250: Loss = 3.3351, Perplexity = 28.08 +Step 5260/44000 | Loss: 3.3374 | LR: 4.87e-04 | 3388.2ms/step | 5.9 samples/s +Step 5270/44000 | Loss: 3.3060 | LR: 4.87e-04 | 1259.4ms/step | 15.9 samples/s +Step 5280/44000 | Loss: 3.3341 | LR: 4.87e-04 | 1260.1ms/step | 15.9 samples/s +Step 5290/44000 | Loss: 3.3190 | LR: 4.87e-04 | 1259.3ms/step | 15.9 samples/s +Step 5300/44000 | Loss: 3.2712 | LR: 4.86e-04 | 1258.9ms/step | 15.9 samples/s +Step 5310/44000 | Loss: 3.3123 | LR: 4.86e-04 | 1260.5ms/step | 15.9 samples/s +Step 5320/44000 | Loss: 3.3246 | LR: 4.86e-04 | 1259.9ms/step | 15.9 samples/s +Step 5330/44000 | Loss: 3.3207 | LR: 4.86e-04 | 1259.5ms/step | 15.9 samples/s +Step 5340/44000 | Loss: 3.3328 | LR: 4.86e-04 | 1258.7ms/step | 15.9 samples/s +Step 5350/44000 | Loss: 3.2998 | LR: 4.86e-04 | 1259.1ms/step | 15.9 samples/s +Step 5360/44000 | Loss: 3.3231 | LR: 4.86e-04 | 1258.4ms/step | 15.9 samples/s +Step 5370/44000 | Loss: 3.3227 | LR: 4.86e-04 | 1259.4ms/step | 15.9 samples/s +Step 5380/44000 | Loss: 3.3095 | LR: 4.86e-04 | 1259.7ms/step | 15.9 samples/s +Step 5390/44000 | Loss: 3.3328 | LR: 4.86e-04 | 1258.7ms/step | 15.9 samples/s +Step 5400/44000 | Loss: 3.3377 | LR: 4.86e-04 | 1262.2ms/step | 15.8 samples/s +Step 5410/44000 | Loss: 3.3245 | LR: 4.86e-04 | 1259.8ms/step | 15.9 samples/s +Step 5420/44000 | Loss: 3.3202 | LR: 4.86e-04 | 1260.0ms/step | 15.9 samples/s +Step 5430/44000 | Loss: 3.3018 | LR: 4.86e-04 | 1260.7ms/step | 15.9 samples/s +Step 5440/44000 | Loss: 3.3147 | LR: 4.86e-04 | 1263.6ms/step | 15.8 samples/s +Step 5450/44000 | Loss: 3.3267 | LR: 4.86e-04 | 1262.1ms/step | 15.8 samples/s +Step 5460/44000 | Loss: 3.3428 | LR: 4.86e-04 | 1260.5ms/step | 15.9 samples/s +Step 5470/44000 | Loss: 3.3222 | LR: 4.85e-04 | 1259.3ms/step | 15.9 samples/s +Step 5480/44000 | Loss: 3.3037 | LR: 4.85e-04 | 1260.0ms/step | 15.9 samples/s +Step 5490/44000 | Loss: 3.2907 | LR: 4.85e-04 | 1260.3ms/step | 15.9 samples/s +Step 5500/44000 | Loss: 3.3156 | LR: 4.85e-04 | 1260.4ms/step | 15.9 samples/s +Validation at step 5500: Loss = 3.3195, Perplexity = 27.65 +Step 5510/44000 | Loss: 3.3106 | LR: 4.85e-04 | 3381.5ms/step | 5.9 samples/s +Step 5520/44000 | Loss: 3.3130 | LR: 4.85e-04 | 1259.7ms/step | 15.9 samples/s +Step 5530/44000 | Loss: 3.2823 | LR: 4.85e-04 | 1260.4ms/step | 15.9 samples/s +Step 5540/44000 | Loss: 3.3165 | LR: 4.85e-04 | 1260.2ms/step | 15.9 samples/s +Step 5550/44000 | Loss: 3.2987 | LR: 4.85e-04 | 1259.9ms/step | 15.9 samples/s +Step 5560/44000 | Loss: 3.3085 | LR: 4.85e-04 | 1261.4ms/step | 15.9 samples/s +Step 5570/44000 | Loss: 3.3011 | LR: 4.85e-04 | 1260.3ms/step | 15.9 samples/s +Step 5580/44000 | Loss: 3.2913 | LR: 4.85e-04 | 1260.5ms/step | 15.9 samples/s +Step 5590/44000 | Loss: 3.2840 | LR: 4.85e-04 | 1261.0ms/step | 15.9 samples/s +Step 5600/44000 | Loss: 3.2853 | LR: 4.85e-04 | 1260.4ms/step | 15.9 samples/s +Step 5610/44000 | Loss: 3.3169 | LR: 4.85e-04 | 1260.1ms/step | 15.9 samples/s +Step 5620/44000 | Loss: 3.2986 | LR: 4.85e-04 | 1260.4ms/step | 15.9 samples/s +Step 5630/44000 | Loss: 3.3180 | LR: 4.84e-04 | 1260.9ms/step | 15.9 samples/s +Step 5640/44000 | Loss: 3.2919 | LR: 4.84e-04 | 1261.8ms/step | 15.8 samples/s +Step 5650/44000 | Loss: 3.2877 | LR: 4.84e-04 | 1261.5ms/step | 15.9 samples/s +Step 5660/44000 | Loss: 3.2751 | LR: 4.84e-04 | 1261.5ms/step | 15.9 samples/s +Step 5670/44000 | Loss: 3.2933 | LR: 4.84e-04 | 1260.9ms/step | 15.9 samples/s +Step 5680/44000 | Loss: 3.2760 | LR: 4.84e-04 | 1261.1ms/step | 15.9 samples/s +Step 5690/44000 | Loss: 3.3236 | LR: 4.84e-04 | 1261.1ms/step | 15.9 samples/s +Step 5700/44000 | Loss: 3.3138 | LR: 4.84e-04 | 1263.1ms/step | 15.8 samples/s +Step 5710/44000 | Loss: 3.2625 | LR: 4.84e-04 | 1263.3ms/step | 15.8 samples/s +Step 5720/44000 | Loss: 3.2614 | LR: 4.84e-04 | 1257.9ms/step | 15.9 samples/s +Step 5730/44000 | Loss: 3.2809 | LR: 4.84e-04 | 1262.0ms/step | 15.8 samples/s +Step 5740/44000 | Loss: 3.2839 | LR: 4.84e-04 | 1261.8ms/step | 15.8 samples/s +Step 5750/44000 | Loss: 3.2873 | LR: 4.84e-04 | 1260.7ms/step | 15.9 samples/s +Validation at step 5750: Loss = 3.3068, Perplexity = 27.30 +Step 5760/44000 | Loss: 3.3114 | LR: 4.84e-04 | 3381.4ms/step | 5.9 samples/s +Step 5770/44000 | Loss: 3.3166 | LR: 4.84e-04 | 1260.6ms/step | 15.9 samples/s +Step 5780/44000 | Loss: 3.2939 | LR: 4.83e-04 | 1261.8ms/step | 15.9 samples/s +Step 5790/44000 | Loss: 3.2784 | LR: 4.83e-04 | 1260.5ms/step | 15.9 samples/s +Step 5800/44000 | Loss: 3.3035 | LR: 4.83e-04 | 1259.2ms/step | 15.9 samples/s +Step 5810/44000 | Loss: 3.3166 | LR: 4.83e-04 | 1259.7ms/step | 15.9 samples/s +Step 5820/44000 | Loss: 3.2989 | LR: 4.83e-04 | 1260.0ms/step | 15.9 samples/s +Step 5830/44000 | Loss: 3.2948 | LR: 4.83e-04 | 1258.5ms/step | 15.9 samples/s +Step 5840/44000 | Loss: 3.2885 | LR: 4.83e-04 | 1259.0ms/step | 15.9 samples/s +Step 5850/44000 | Loss: 3.2699 | LR: 4.83e-04 | 1258.6ms/step | 15.9 samples/s +Step 5860/44000 | Loss: 3.2987 | LR: 4.83e-04 | 1257.8ms/step | 15.9 samples/s +Step 5870/44000 | Loss: 3.3053 | LR: 4.83e-04 | 1258.4ms/step | 15.9 samples/s +Step 5880/44000 | Loss: 3.2841 | LR: 4.83e-04 | 1260.3ms/step | 15.9 samples/s +Step 5890/44000 | Loss: 3.2997 | LR: 4.83e-04 | 1259.3ms/step | 15.9 samples/s +Step 5900/44000 | Loss: 3.2876 | LR: 4.83e-04 | 1259.0ms/step | 15.9 samples/s +Step 5910/44000 | Loss: 3.2961 | LR: 4.83e-04 | 1259.9ms/step | 15.9 samples/s +Step 5920/44000 | Loss: 3.3051 | LR: 4.83e-04 | 1260.0ms/step | 15.9 samples/s +Step 5930/44000 | Loss: 3.2560 | LR: 4.83e-04 | 1259.8ms/step | 15.9 samples/s +Step 5940/44000 | Loss: 3.3061 | LR: 4.82e-04 | 1259.8ms/step | 15.9 samples/s +Step 5950/44000 | Loss: 3.2663 | LR: 4.82e-04 | 1259.9ms/step | 15.9 samples/s +Step 5960/44000 | Loss: 3.2589 | LR: 4.82e-04 | 1260.3ms/step | 15.9 samples/s +Step 5970/44000 | Loss: 3.2644 | LR: 4.82e-04 | 1259.9ms/step | 15.9 samples/s +Step 5980/44000 | Loss: 3.2602 | LR: 4.82e-04 | 1260.2ms/step | 15.9 samples/s +Step 5990/44000 | Loss: 3.2593 | LR: 4.82e-04 | 1259.9ms/step | 15.9 samples/s +Step 6000/44000 | Loss: 3.2860 | LR: 4.82e-04 | 1260.3ms/step | 15.9 samples/s +Validation at step 6000: Loss = 3.2945, Perplexity = 26.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-6000 +Step 6010/44000 | Loss: 3.3081 | LR: 4.82e-04 | 3673.8ms/step | 5.4 samples/s +Step 6020/44000 | Loss: 3.2697 | LR: 4.82e-04 | 1261.7ms/step | 15.9 samples/s +Step 6030/44000 | Loss: 3.2759 | LR: 4.82e-04 | 1263.0ms/step | 15.8 samples/s +Step 6040/44000 | Loss: 3.2776 | LR: 4.82e-04 | 1264.4ms/step | 15.8 samples/s +Step 6050/44000 | Loss: 3.2815 | LR: 4.82e-04 | 1260.8ms/step | 15.9 samples/s +Step 6060/44000 | Loss: 3.3051 | LR: 4.82e-04 | 1260.1ms/step | 15.9 samples/s +Step 6070/44000 | Loss: 3.2831 | LR: 4.82e-04 | 1259.6ms/step | 15.9 samples/s +Step 6080/44000 | Loss: 3.2784 | LR: 4.81e-04 | 1259.5ms/step | 15.9 samples/s +Step 6090/44000 | Loss: 3.3069 | LR: 4.81e-04 | 1260.1ms/step | 15.9 samples/s +Step 6100/44000 | Loss: 3.2593 | LR: 4.81e-04 | 1260.1ms/step | 15.9 samples/s +Step 6110/44000 | Loss: 3.2749 | LR: 4.81e-04 | 1260.2ms/step | 15.9 samples/s +Step 6120/44000 | Loss: 3.2552 | LR: 4.81e-04 | 1261.5ms/step | 15.9 samples/s +Step 6130/44000 | Loss: 3.2726 | LR: 4.81e-04 | 1264.4ms/step | 15.8 samples/s +Step 6140/44000 | Loss: 3.2587 | LR: 4.81e-04 | 1265.0ms/step | 15.8 samples/s +Step 6150/44000 | Loss: 3.2686 | LR: 4.81e-04 | 1264.9ms/step | 15.8 samples/s +Step 6160/44000 | Loss: 3.2656 | LR: 4.81e-04 | 1262.3ms/step | 15.8 samples/s +Step 6170/44000 | Loss: 3.2833 | LR: 4.81e-04 | 1263.1ms/step | 15.8 samples/s +Step 6180/44000 | Loss: 3.2788 | LR: 4.81e-04 | 1262.7ms/step | 15.8 samples/s +Step 6190/44000 | Loss: 3.2640 | LR: 4.81e-04 | 1260.3ms/step | 15.9 samples/s +Step 6200/44000 | Loss: 3.2781 | LR: 4.81e-04 | 1262.9ms/step | 15.8 samples/s +Step 6210/44000 | Loss: 3.2740 | LR: 4.81e-04 | 1261.1ms/step | 15.9 samples/s +Step 6220/44000 | Loss: 3.3041 | LR: 4.81e-04 | 1260.1ms/step | 15.9 samples/s +Step 6230/44000 | Loss: 3.2707 | LR: 4.80e-04 | 1260.9ms/step | 15.9 samples/s +Step 6240/44000 | Loss: 3.2903 | LR: 4.80e-04 | 1260.2ms/step | 15.9 samples/s +Step 6250/44000 | Loss: 3.2854 | LR: 4.80e-04 | 1258.5ms/step | 15.9 samples/s +Validation at step 6250: Loss = 3.2934, Perplexity = 26.93 +Step 6260/44000 | Loss: 3.2835 | LR: 4.80e-04 | 3379.7ms/step | 5.9 samples/s +Step 6270/44000 | Loss: 3.3059 | LR: 4.80e-04 | 1260.4ms/step | 15.9 samples/s +Step 6280/44000 | Loss: 3.2555 | LR: 4.80e-04 | 1264.7ms/step | 15.8 samples/s +Step 6290/44000 | Loss: 3.2856 | LR: 4.80e-04 | 1263.6ms/step | 15.8 samples/s +Step 6300/44000 | Loss: 3.2626 | LR: 4.80e-04 | 1260.7ms/step | 15.9 samples/s +Step 6310/44000 | Loss: 3.2684 | LR: 4.80e-04 | 1262.4ms/step | 15.8 samples/s +Step 6320/44000 | Loss: 3.2615 | LR: 4.80e-04 | 1260.5ms/step | 15.9 samples/s +Step 6330/44000 | Loss: 3.2681 | LR: 4.80e-04 | 1259.4ms/step | 15.9 samples/s +Step 6340/44000 | Loss: 3.2794 | LR: 4.80e-04 | 1260.5ms/step | 15.9 samples/s +Step 6350/44000 | Loss: 3.2380 | LR: 4.80e-04 | 1260.1ms/step | 15.9 samples/s +Step 6360/44000 | Loss: 3.2935 | LR: 4.80e-04 | 1260.1ms/step | 15.9 samples/s +Step 6370/44000 | Loss: 3.2632 | LR: 4.79e-04 | 1260.5ms/step | 15.9 samples/s +Step 6380/44000 | Loss: 3.2517 | LR: 4.79e-04 | 1259.3ms/step | 15.9 samples/s +Step 6390/44000 | Loss: 3.2452 | LR: 4.79e-04 | 1260.3ms/step | 15.9 samples/s +Step 6400/44000 | Loss: 3.2764 | LR: 4.79e-04 | 1262.3ms/step | 15.8 samples/s +Step 6410/44000 | Loss: 3.2455 | LR: 4.79e-04 | 1261.0ms/step | 15.9 samples/s +Step 6420/44000 | Loss: 3.2769 | LR: 4.79e-04 | 1260.9ms/step | 15.9 samples/s +Step 6430/44000 | Loss: 3.2400 | LR: 4.79e-04 | 1261.1ms/step | 15.9 samples/s +Step 6440/44000 | Loss: 3.2567 | LR: 4.79e-04 | 1260.4ms/step | 15.9 samples/s +Step 6450/44000 | Loss: 3.2526 | LR: 4.79e-04 | 1259.6ms/step | 15.9 samples/s +Step 6460/44000 | Loss: 3.2625 | LR: 4.79e-04 | 1259.9ms/step | 15.9 samples/s +Step 6470/44000 | Loss: 3.2421 | LR: 4.79e-04 | 1259.5ms/step | 15.9 samples/s +Step 6480/44000 | Loss: 3.2556 | LR: 4.79e-04 | 1259.7ms/step | 15.9 samples/s +Step 6490/44000 | Loss: 3.2478 | LR: 4.79e-04 | 1258.8ms/step | 15.9 samples/s +Step 6500/44000 | Loss: 3.2462 | LR: 4.79e-04 | 1258.9ms/step | 15.9 samples/s +Validation at step 6500: Loss = 3.2735, Perplexity = 26.40 +Step 6510/44000 | Loss: 3.2702 | LR: 4.78e-04 | 3379.4ms/step | 5.9 samples/s +Step 6520/44000 | Loss: 3.2785 | LR: 4.78e-04 | 1260.6ms/step | 15.9 samples/s +Step 6530/44000 | Loss: 3.2764 | LR: 4.78e-04 | 1261.0ms/step | 15.9 samples/s +Step 6540/44000 | Loss: 3.2720 | LR: 4.78e-04 | 1258.7ms/step | 15.9 samples/s +Step 6550/44000 | Loss: 3.2725 | LR: 4.78e-04 | 1260.2ms/step | 15.9 samples/s +Step 6560/44000 | Loss: 3.2527 | LR: 4.78e-04 | 1260.0ms/step | 15.9 samples/s +Step 6570/44000 | Loss: 3.2532 | LR: 4.78e-04 | 1260.2ms/step | 15.9 samples/s +Step 6580/44000 | Loss: 3.2930 | LR: 4.78e-04 | 1259.7ms/step | 15.9 samples/s +Step 6590/44000 | Loss: 3.2611 | LR: 4.78e-04 | 1260.0ms/step | 15.9 samples/s +Step 6600/44000 | Loss: 3.2492 | LR: 4.78e-04 | 1260.1ms/step | 15.9 samples/s +Step 6610/44000 | Loss: 3.2143 | LR: 4.78e-04 | 1260.0ms/step | 15.9 samples/s +Step 6620/44000 | Loss: 3.2471 | LR: 4.78e-04 | 1261.8ms/step | 15.9 samples/s +Step 6630/44000 | Loss: 3.2503 | LR: 4.78e-04 | 1262.3ms/step | 15.8 samples/s +Step 6640/44000 | Loss: 3.2400 | LR: 4.77e-04 | 1260.1ms/step | 15.9 samples/s +Step 6650/44000 | Loss: 3.2661 | LR: 4.77e-04 | 1259.8ms/step | 15.9 samples/s +Step 6660/44000 | Loss: 3.2913 | LR: 4.77e-04 | 1260.3ms/step | 15.9 samples/s +Step 6670/44000 | Loss: 3.2385 | LR: 4.77e-04 | 1260.1ms/step | 15.9 samples/s +Step 6680/44000 | Loss: 3.2620 | LR: 4.77e-04 | 1260.6ms/step | 15.9 samples/s +Step 6690/44000 | Loss: 3.2637 | LR: 4.77e-04 | 1261.2ms/step | 15.9 samples/s +Step 6700/44000 | Loss: 3.2639 | LR: 4.77e-04 | 1261.8ms/step | 15.9 samples/s +Step 6710/44000 | Loss: 3.2616 | LR: 4.77e-04 | 1260.5ms/step | 15.9 samples/s +Step 6720/44000 | Loss: 3.2282 | LR: 4.77e-04 | 1263.4ms/step | 15.8 samples/s +Step 6730/44000 | Loss: 3.2625 | LR: 4.77e-04 | 1262.1ms/step | 15.8 samples/s +Step 6740/44000 | Loss: 3.2389 | LR: 4.77e-04 | 1260.1ms/step | 15.9 samples/s +Step 6750/44000 | Loss: 3.2436 | LR: 4.77e-04 | 1261.2ms/step | 15.9 samples/s +Validation at step 6750: Loss = 3.2645, Perplexity = 26.17 +Step 6760/44000 | Loss: 3.2699 | LR: 4.77e-04 | 3384.7ms/step | 5.9 samples/s +Step 6770/44000 | Loss: 3.2661 | LR: 4.76e-04 | 1259.9ms/step | 15.9 samples/s +Step 6780/44000 | Loss: 3.2633 | LR: 4.76e-04 | 1260.5ms/step | 15.9 samples/s +Step 6790/44000 | Loss: 3.2553 | LR: 4.76e-04 | 1259.8ms/step | 15.9 samples/s +Step 6800/44000 | Loss: 3.2098 | LR: 4.76e-04 | 1259.7ms/step | 15.9 samples/s +Step 6810/44000 | Loss: 3.2699 | LR: 4.76e-04 | 1260.4ms/step | 15.9 samples/s +Step 6820/44000 | Loss: 3.2373 | LR: 4.76e-04 | 1261.5ms/step | 15.9 samples/s +Step 6830/44000 | Loss: 3.2523 | LR: 4.76e-04 | 1260.1ms/step | 15.9 samples/s +Step 6840/44000 | Loss: 3.2548 | LR: 4.76e-04 | 1260.4ms/step | 15.9 samples/s +Step 6850/44000 | Loss: 3.2572 | LR: 4.76e-04 | 1261.8ms/step | 15.9 samples/s +Step 6860/44000 | Loss: 3.2560 | LR: 4.76e-04 | 1259.9ms/step | 15.9 samples/s +Step 6870/44000 | Loss: 3.2316 | LR: 4.76e-04 | 1260.5ms/step | 15.9 samples/s +Step 6880/44000 | Loss: 3.2037 | LR: 4.76e-04 | 1259.9ms/step | 15.9 samples/s +Step 6890/44000 | Loss: 3.2606 | LR: 4.76e-04 | 1263.5ms/step | 15.8 samples/s +Step 6900/44000 | Loss: 3.2725 | LR: 4.75e-04 | 1263.3ms/step | 15.8 samples/s +Step 6910/44000 | Loss: 3.2359 | LR: 4.75e-04 | 1259.9ms/step | 15.9 samples/s +Step 6920/44000 | Loss: 3.2552 | LR: 4.75e-04 | 1260.2ms/step | 15.9 samples/s +Step 6930/44000 | Loss: 3.2368 | LR: 4.75e-04 | 1260.4ms/step | 15.9 samples/s +Step 6940/44000 | Loss: 3.2625 | LR: 4.75e-04 | 1260.2ms/step | 15.9 samples/s +Step 6950/44000 | Loss: 3.2613 | LR: 4.75e-04 | 1259.9ms/step | 15.9 samples/s +Step 6960/44000 | Loss: 3.2350 | LR: 4.75e-04 | 1259.9ms/step | 15.9 samples/s +Step 6970/44000 | Loss: 3.2573 | LR: 4.75e-04 | 1260.0ms/step | 15.9 samples/s +Step 6980/44000 | Loss: 3.2245 | LR: 4.75e-04 | 1260.2ms/step | 15.9 samples/s +Step 6990/44000 | Loss: 3.2309 | LR: 4.75e-04 | 1259.4ms/step | 15.9 samples/s +Step 7000/44000 | Loss: 3.2382 | LR: 4.75e-04 | 1260.2ms/step | 15.9 samples/s +Validation at step 7000: Loss = 3.2543, Perplexity = 25.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-7000 +Step 7010/44000 | Loss: 3.2301 | LR: 4.75e-04 | 3675.6ms/step | 5.4 samples/s +Step 7020/44000 | Loss: 3.2314 | LR: 4.75e-04 | 1252.3ms/step | 16.0 samples/s +Step 7030/44000 | Loss: 3.2534 | LR: 4.74e-04 | 1253.3ms/step | 16.0 samples/s +Step 7040/44000 | Loss: 3.2451 | LR: 4.74e-04 | 1253.7ms/step | 16.0 samples/s +Step 7050/44000 | Loss: 3.2710 | LR: 4.74e-04 | 1254.2ms/step | 15.9 samples/s +Step 7060/44000 | Loss: 3.2528 | LR: 4.74e-04 | 1254.5ms/step | 15.9 samples/s +Step 7070/44000 | Loss: 3.2394 | LR: 4.74e-04 | 1253.4ms/step | 16.0 samples/s +Step 7080/44000 | Loss: 3.2300 | LR: 4.74e-04 | 1254.5ms/step | 15.9 samples/s +Step 7090/44000 | Loss: 3.2092 | LR: 4.74e-04 | 1255.0ms/step | 15.9 samples/s +Step 7100/44000 | Loss: 3.2359 | LR: 4.74e-04 | 1253.5ms/step | 16.0 samples/s +Step 7110/44000 | Loss: 3.2106 | LR: 4.74e-04 | 1254.4ms/step | 15.9 samples/s +Step 7120/44000 | Loss: 3.2303 | LR: 4.74e-04 | 1253.2ms/step | 16.0 samples/s +Step 7130/44000 | Loss: 3.2438 | LR: 4.74e-04 | 1257.9ms/step | 15.9 samples/s +Step 7140/44000 | Loss: 3.2637 | LR: 4.74e-04 | 1253.9ms/step | 16.0 samples/s +Step 7150/44000 | Loss: 3.2439 | LR: 4.73e-04 | 1254.6ms/step | 15.9 samples/s +Step 7160/44000 | Loss: 3.2540 | LR: 4.73e-04 | 1254.0ms/step | 15.9 samples/s +Step 7170/44000 | Loss: 3.2552 | LR: 4.73e-04 | 1253.6ms/step | 16.0 samples/s +Step 7180/44000 | Loss: 3.2380 | LR: 4.73e-04 | 1254.9ms/step | 15.9 samples/s +Step 7190/44000 | Loss: 3.1999 | LR: 4.73e-04 | 1255.2ms/step | 15.9 samples/s +Step 7200/44000 | Loss: 3.2517 | LR: 4.73e-04 | 1253.9ms/step | 16.0 samples/s +Step 7210/44000 | Loss: 3.2722 | LR: 4.73e-04 | 1253.4ms/step | 16.0 samples/s +Step 7220/44000 | Loss: 3.2553 | LR: 4.73e-04 | 1253.1ms/step | 16.0 samples/s +Step 7230/44000 | Loss: 3.2523 | LR: 4.73e-04 | 1252.6ms/step | 16.0 samples/s +Step 7240/44000 | Loss: 3.2423 | LR: 4.73e-04 | 1253.8ms/step | 16.0 samples/s +Step 7250/44000 | Loss: 3.2470 | LR: 4.73e-04 | 1254.3ms/step | 15.9 samples/s +Validation at step 7250: Loss = 3.2440, Perplexity = 25.64 +Step 7260/44000 | Loss: 3.2264 | LR: 4.73e-04 | 3365.3ms/step | 5.9 samples/s +Step 7270/44000 | Loss: 3.2367 | LR: 4.72e-04 | 1256.9ms/step | 15.9 samples/s +Step 7280/44000 | Loss: 3.2171 | LR: 4.72e-04 | 1254.5ms/step | 15.9 samples/s +Step 7290/44000 | Loss: 3.2580 | LR: 4.72e-04 | 1253.1ms/step | 16.0 samples/s +Step 7300/44000 | Loss: 3.2410 | LR: 4.72e-04 | 1254.4ms/step | 15.9 samples/s +Step 7310/44000 | Loss: 3.2571 | LR: 4.72e-04 | 1253.5ms/step | 16.0 samples/s +Step 7320/44000 | Loss: 3.2581 | LR: 4.72e-04 | 1254.6ms/step | 15.9 samples/s +Step 7330/44000 | Loss: 3.2514 | LR: 4.72e-04 | 1253.4ms/step | 16.0 samples/s +Step 7340/44000 | Loss: 3.2600 | LR: 4.72e-04 | 1251.3ms/step | 16.0 samples/s +Step 7350/44000 | Loss: 3.2082 | LR: 4.72e-04 | 1255.7ms/step | 15.9 samples/s +Step 7360/44000 | Loss: 3.2156 | LR: 4.72e-04 | 1254.8ms/step | 15.9 samples/s +Step 7370/44000 | Loss: 3.2076 | LR: 4.72e-04 | 1252.2ms/step | 16.0 samples/s +Step 7380/44000 | Loss: 3.2356 | LR: 4.72e-04 | 1251.5ms/step | 16.0 samples/s +Step 7390/44000 | Loss: 3.2417 | LR: 4.71e-04 | 1255.4ms/step | 15.9 samples/s +Step 7400/44000 | Loss: 3.2334 | LR: 4.71e-04 | 1254.0ms/step | 15.9 samples/s +Step 7410/44000 | Loss: 3.2572 | LR: 4.71e-04 | 1253.5ms/step | 16.0 samples/s +Step 7420/44000 | Loss: 3.2595 | LR: 4.71e-04 | 1253.8ms/step | 16.0 samples/s +Step 7430/44000 | Loss: 3.2373 | LR: 4.71e-04 | 1253.6ms/step | 16.0 samples/s +Step 7440/44000 | Loss: 3.2407 | LR: 4.71e-04 | 1255.0ms/step | 15.9 samples/s +Step 7450/44000 | Loss: 3.2544 | LR: 4.71e-04 | 1253.9ms/step | 15.9 samples/s +Step 7460/44000 | Loss: 3.1984 | LR: 4.71e-04 | 1254.6ms/step | 15.9 samples/s +Step 7470/44000 | Loss: 3.2415 | LR: 4.71e-04 | 1254.1ms/step | 15.9 samples/s +Step 7480/44000 | Loss: 3.1953 | LR: 4.71e-04 | 1254.1ms/step | 15.9 samples/s +Step 7490/44000 | Loss: 3.2361 | LR: 4.71e-04 | 1253.4ms/step | 16.0 samples/s +Step 7500/44000 | Loss: 3.1997 | LR: 4.71e-04 | 1252.8ms/step | 16.0 samples/s +Validation at step 7500: Loss = 3.2364, Perplexity = 25.44 +Step 7510/44000 | Loss: 3.2117 | LR: 4.70e-04 | 3368.2ms/step | 5.9 samples/s +Step 7520/44000 | Loss: 3.2273 | LR: 4.70e-04 | 1253.8ms/step | 16.0 samples/s +Step 7530/44000 | Loss: 3.2566 | LR: 4.70e-04 | 1254.6ms/step | 15.9 samples/s +Step 7540/44000 | Loss: 3.2126 | LR: 4.70e-04 | 1255.6ms/step | 15.9 samples/s +Step 7550/44000 | Loss: 3.2171 | LR: 4.70e-04 | 1254.5ms/step | 15.9 samples/s +Step 7560/44000 | Loss: 3.2201 | LR: 4.70e-04 | 1255.4ms/step | 15.9 samples/s +Step 7570/44000 | Loss: 3.2506 | LR: 4.70e-04 | 1254.7ms/step | 15.9 samples/s +Step 7580/44000 | Loss: 3.2072 | LR: 4.70e-04 | 1254.2ms/step | 15.9 samples/s +Step 7590/44000 | Loss: 3.2507 | LR: 4.70e-04 | 1254.0ms/step | 15.9 samples/s +Step 7600/44000 | Loss: 3.2297 | LR: 4.70e-04 | 1253.9ms/step | 16.0 samples/s +Step 7610/44000 | Loss: 3.2492 | LR: 4.70e-04 | 1254.7ms/step | 15.9 samples/s +Step 7620/44000 | Loss: 3.2021 | LR: 4.70e-04 | 1254.3ms/step | 15.9 samples/s +Step 7630/44000 | Loss: 3.2563 | LR: 4.69e-04 | 1254.6ms/step | 15.9 samples/s +Step 7640/44000 | Loss: 3.2142 | LR: 4.69e-04 | 1255.0ms/step | 15.9 samples/s +Step 7650/44000 | Loss: 3.2261 | LR: 4.69e-04 | 1255.4ms/step | 15.9 samples/s +Step 7660/44000 | Loss: 3.2361 | LR: 4.69e-04 | 1253.9ms/step | 16.0 samples/s +Step 7670/44000 | Loss: 3.2173 | LR: 4.69e-04 | 1254.5ms/step | 15.9 samples/s +Step 7680/44000 | Loss: 3.2375 | LR: 4.69e-04 | 1254.1ms/step | 15.9 samples/s +Step 7690/44000 | Loss: 3.2006 | LR: 4.69e-04 | 1255.2ms/step | 15.9 samples/s +Step 7700/44000 | Loss: 3.2247 | LR: 4.69e-04 | 1254.4ms/step | 15.9 samples/s +Step 7710/44000 | Loss: 3.2472 | LR: 4.69e-04 | 1255.6ms/step | 15.9 samples/s +Step 7720/44000 | Loss: 3.2472 | LR: 4.69e-04 | 1253.9ms/step | 15.9 samples/s +Step 7730/44000 | Loss: 3.2416 | LR: 4.69e-04 | 1255.0ms/step | 15.9 samples/s +Step 7740/44000 | Loss: 3.2110 | LR: 4.68e-04 | 1255.3ms/step | 15.9 samples/s +Step 7750/44000 | Loss: 3.1894 | LR: 4.68e-04 | 1254.2ms/step | 15.9 samples/s +Validation at step 7750: Loss = 3.2276, Perplexity = 25.22 +Step 7760/44000 | Loss: 3.1997 | LR: 4.68e-04 | 3365.9ms/step | 5.9 samples/s +Step 7770/44000 | Loss: 3.2026 | LR: 4.68e-04 | 1254.5ms/step | 15.9 samples/s +Step 7780/44000 | Loss: 3.1999 | LR: 4.68e-04 | 1255.4ms/step | 15.9 samples/s +Step 7790/44000 | Loss: 3.2228 | LR: 4.68e-04 | 1255.3ms/step | 15.9 samples/s +Step 7800/44000 | Loss: 3.2125 | LR: 4.68e-04 | 1255.2ms/step | 15.9 samples/s +Step 7810/44000 | Loss: 3.2012 | LR: 4.68e-04 | 1255.7ms/step | 15.9 samples/s +Step 7820/44000 | Loss: 3.2167 | LR: 4.68e-04 | 1254.1ms/step | 15.9 samples/s +Step 7830/44000 | Loss: 3.2320 | LR: 4.68e-04 | 1254.0ms/step | 15.9 samples/s +Step 7840/44000 | Loss: 3.2113 | LR: 4.68e-04 | 1253.8ms/step | 16.0 samples/s +Step 7850/44000 | Loss: 3.1915 | LR: 4.67e-04 | 1254.0ms/step | 15.9 samples/s +Step 7860/44000 | Loss: 3.2095 | LR: 4.67e-04 | 1255.9ms/step | 15.9 samples/s +Step 7870/44000 | Loss: 3.2125 | LR: 4.67e-04 | 1254.0ms/step | 15.9 samples/s +Step 7880/44000 | Loss: 3.1724 | LR: 4.67e-04 | 1255.1ms/step | 15.9 samples/s +Step 7890/44000 | Loss: 3.1925 | LR: 4.67e-04 | 1255.3ms/step | 15.9 samples/s +Step 7900/44000 | Loss: 3.2401 | LR: 4.67e-04 | 1254.7ms/step | 15.9 samples/s +Step 7910/44000 | Loss: 3.1884 | LR: 4.67e-04 | 1255.1ms/step | 15.9 samples/s +Step 7920/44000 | Loss: 3.2241 | LR: 4.67e-04 | 1255.6ms/step | 15.9 samples/s +Step 7930/44000 | Loss: 3.2019 | LR: 4.67e-04 | 1254.3ms/step | 15.9 samples/s +Step 7940/44000 | Loss: 3.2185 | LR: 4.67e-04 | 1255.0ms/step | 15.9 samples/s +Step 7950/44000 | Loss: 3.2135 | LR: 4.67e-04 | 1255.0ms/step | 15.9 samples/s +Step 7960/44000 | Loss: 3.2080 | LR: 4.66e-04 | 1253.6ms/step | 16.0 samples/s +Step 7970/44000 | Loss: 3.2019 | LR: 4.66e-04 | 1253.9ms/step | 16.0 samples/s +Step 7980/44000 | Loss: 3.2273 | LR: 4.66e-04 | 1255.7ms/step | 15.9 samples/s +Step 7990/44000 | Loss: 3.2234 | LR: 4.66e-04 | 1255.6ms/step | 15.9 samples/s +Step 8000/44000 | Loss: 3.1956 | LR: 4.66e-04 | 1254.1ms/step | 15.9 samples/s +Validation at step 8000: Loss = 3.2196, Perplexity = 25.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-8000 +Step 8010/44000 | Loss: 3.1662 | LR: 4.66e-04 | 3657.3ms/step | 5.5 samples/s +Step 8020/44000 | Loss: 3.1888 | LR: 4.66e-04 | 1255.9ms/step | 15.9 samples/s +Step 8030/44000 | Loss: 3.1891 | LR: 4.66e-04 | 1256.7ms/step | 15.9 samples/s +Step 8040/44000 | Loss: 3.2127 | LR: 4.66e-04 | 1259.2ms/step | 15.9 samples/s +Step 8050/44000 | Loss: 3.1963 | LR: 4.66e-04 | 1258.6ms/step | 15.9 samples/s +Step 8060/44000 | Loss: 3.2294 | LR: 4.66e-04 | 1260.0ms/step | 15.9 samples/s +Step 8070/44000 | Loss: 3.1945 | LR: 4.65e-04 | 1260.1ms/step | 15.9 samples/s +Step 8080/44000 | Loss: 3.1856 | LR: 4.65e-04 | 1259.7ms/step | 15.9 samples/s +Step 8090/44000 | Loss: 3.1874 | LR: 4.65e-04 | 1260.0ms/step | 15.9 samples/s +Step 8100/44000 | Loss: 3.2123 | LR: 4.65e-04 | 1258.9ms/step | 15.9 samples/s +Step 8110/44000 | Loss: 3.2055 | LR: 4.65e-04 | 1259.1ms/step | 15.9 samples/s +Step 8120/44000 | Loss: 3.1971 | LR: 4.65e-04 | 1260.2ms/step | 15.9 samples/s +Step 8130/44000 | Loss: 3.1976 | LR: 4.65e-04 | 1259.6ms/step | 15.9 samples/s +Step 8140/44000 | Loss: 3.2315 | LR: 4.65e-04 | 1260.1ms/step | 15.9 samples/s +Step 8150/44000 | Loss: 3.1948 | LR: 4.65e-04 | 1259.7ms/step | 15.9 samples/s +Step 8160/44000 | Loss: 3.1874 | LR: 4.65e-04 | 1262.5ms/step | 15.8 samples/s +Step 8170/44000 | Loss: 3.2124 | LR: 4.65e-04 | 1259.4ms/step | 15.9 samples/s +Step 8180/44000 | Loss: 3.2007 | LR: 4.64e-04 | 1254.9ms/step | 15.9 samples/s +Step 8190/44000 | Loss: 3.1933 | LR: 4.64e-04 | 1255.5ms/step | 15.9 samples/s +Step 8200/44000 | Loss: 3.2189 | LR: 4.64e-04 | 1255.2ms/step | 15.9 samples/s +Step 8210/44000 | Loss: 3.2187 | LR: 4.64e-04 | 1254.9ms/step | 15.9 samples/s +Step 8220/44000 | Loss: 3.2245 | LR: 4.64e-04 | 1256.0ms/step | 15.9 samples/s +Step 8230/44000 | Loss: 3.1864 | LR: 4.64e-04 | 1255.4ms/step | 15.9 samples/s +Step 8240/44000 | Loss: 3.1992 | LR: 4.64e-04 | 1256.0ms/step | 15.9 samples/s +Step 8250/44000 | Loss: 3.1948 | LR: 4.64e-04 | 1255.0ms/step | 15.9 samples/s +Validation at step 8250: Loss = 3.2116, Perplexity = 24.82 +Step 8260/44000 | Loss: 3.1802 | LR: 4.64e-04 | 3365.2ms/step | 5.9 samples/s +Step 8270/44000 | Loss: 3.2237 | LR: 4.64e-04 | 1255.1ms/step | 15.9 samples/s +Step 8280/44000 | Loss: 3.2023 | LR: 4.64e-04 | 1255.4ms/step | 15.9 samples/s +Step 8290/44000 | Loss: 3.2059 | LR: 4.63e-04 | 1255.0ms/step | 15.9 samples/s +Step 8300/44000 | Loss: 3.1604 | LR: 4.63e-04 | 1254.5ms/step | 15.9 samples/s +Step 8310/44000 | Loss: 3.2127 | LR: 4.63e-04 | 1254.8ms/step | 15.9 samples/s +Step 8320/44000 | Loss: 3.1881 | LR: 4.63e-04 | 1255.1ms/step | 15.9 samples/s +Step 8330/44000 | Loss: 3.1937 | LR: 4.63e-04 | 1255.0ms/step | 15.9 samples/s +Step 8340/44000 | Loss: 3.2414 | LR: 4.63e-04 | 1255.0ms/step | 15.9 samples/s +Step 8350/44000 | Loss: 3.1890 | LR: 4.63e-04 | 1254.7ms/step | 15.9 samples/s +Step 8360/44000 | Loss: 3.2143 | LR: 4.63e-04 | 1254.8ms/step | 15.9 samples/s +Step 8370/44000 | Loss: 3.2007 | LR: 4.63e-04 | 1254.4ms/step | 15.9 samples/s +Step 8380/44000 | Loss: 3.2353 | LR: 4.63e-04 | 1256.3ms/step | 15.9 samples/s +Step 8390/44000 | Loss: 3.1715 | LR: 4.62e-04 | 1259.2ms/step | 15.9 samples/s +Step 8400/44000 | Loss: 3.1864 | LR: 4.62e-04 | 1259.2ms/step | 15.9 samples/s +Step 8410/44000 | Loss: 3.1851 | LR: 4.62e-04 | 1256.5ms/step | 15.9 samples/s +Step 8420/44000 | Loss: 3.2252 | LR: 4.62e-04 | 1254.5ms/step | 15.9 samples/s +Step 8430/44000 | Loss: 3.1882 | LR: 4.62e-04 | 1252.6ms/step | 16.0 samples/s +Step 8440/44000 | Loss: 3.2148 | LR: 4.62e-04 | 1253.8ms/step | 16.0 samples/s +Step 8450/44000 | Loss: 3.1662 | LR: 4.62e-04 | 1253.3ms/step | 16.0 samples/s +Step 8460/44000 | Loss: 3.2160 | LR: 4.62e-04 | 1252.9ms/step | 16.0 samples/s +Step 8470/44000 | Loss: 3.1620 | LR: 4.62e-04 | 1253.0ms/step | 16.0 samples/s +Step 8480/44000 | Loss: 3.1877 | LR: 4.62e-04 | 1253.9ms/step | 16.0 samples/s +Step 8490/44000 | Loss: 3.2103 | LR: 4.62e-04 | 1255.9ms/step | 15.9 samples/s +Step 8500/44000 | Loss: 3.2175 | LR: 4.61e-04 | 1256.5ms/step | 15.9 samples/s +Validation at step 8500: Loss = 3.2040, Perplexity = 24.63 +Step 8510/44000 | Loss: 3.2151 | LR: 4.61e-04 | 3370.9ms/step | 5.9 samples/s +Step 8520/44000 | Loss: 3.1873 | LR: 4.61e-04 | 1253.7ms/step | 16.0 samples/s +Step 8530/44000 | Loss: 3.2130 | LR: 4.61e-04 | 1258.9ms/step | 15.9 samples/s +Step 8540/44000 | Loss: 3.1712 | LR: 4.61e-04 | 1256.1ms/step | 15.9 samples/s +Step 8550/44000 | Loss: 3.1915 | LR: 4.61e-04 | 1255.0ms/step | 15.9 samples/s +Step 8560/44000 | Loss: 3.1988 | LR: 4.61e-04 | 1255.8ms/step | 15.9 samples/s +Step 8570/44000 | Loss: 3.1668 | LR: 4.61e-04 | 1255.4ms/step | 15.9 samples/s +Step 8580/44000 | Loss: 3.1967 | LR: 4.61e-04 | 1254.8ms/step | 15.9 samples/s +Step 8590/44000 | Loss: 3.1957 | LR: 4.61e-04 | 1255.4ms/step | 15.9 samples/s +Step 8600/44000 | Loss: 3.1850 | LR: 4.60e-04 | 1254.7ms/step | 15.9 samples/s +Step 8610/44000 | Loss: 3.1630 | LR: 4.60e-04 | 1254.7ms/step | 15.9 samples/s +Step 8620/44000 | Loss: 3.1942 | LR: 4.60e-04 | 1253.8ms/step | 16.0 samples/s +Step 8630/44000 | Loss: 3.1834 | LR: 4.60e-04 | 1254.7ms/step | 15.9 samples/s +Step 8640/44000 | Loss: 3.1909 | LR: 4.60e-04 | 1254.8ms/step | 15.9 samples/s +Step 8650/44000 | Loss: 3.1951 | LR: 4.60e-04 | 1254.5ms/step | 15.9 samples/s +Step 8660/44000 | Loss: 3.1572 | LR: 4.60e-04 | 1254.2ms/step | 15.9 samples/s +Step 8670/44000 | Loss: 3.1979 | LR: 4.60e-04 | 1255.2ms/step | 15.9 samples/s +Step 8680/44000 | Loss: 3.1969 | LR: 4.60e-04 | 1255.0ms/step | 15.9 samples/s +Step 8690/44000 | Loss: 3.1763 | LR: 4.60e-04 | 1253.4ms/step | 16.0 samples/s +Step 8700/44000 | Loss: 3.1541 | LR: 4.59e-04 | 1253.9ms/step | 15.9 samples/s +Step 8710/44000 | Loss: 3.1862 | LR: 4.59e-04 | 1253.3ms/step | 16.0 samples/s +Step 8720/44000 | Loss: 3.1943 | LR: 4.59e-04 | 1252.6ms/step | 16.0 samples/s +Step 8730/44000 | Loss: 3.1965 | LR: 4.59e-04 | 1252.3ms/step | 16.0 samples/s +Step 8740/44000 | Loss: 3.1855 | LR: 4.59e-04 | 1253.5ms/step | 16.0 samples/s +Step 8750/44000 | Loss: 3.1590 | LR: 4.59e-04 | 1253.1ms/step | 16.0 samples/s +Validation at step 8750: Loss = 3.1984, Perplexity = 24.49 +Step 8760/44000 | Loss: 3.1711 | LR: 4.59e-04 | 3361.4ms/step | 5.9 samples/s +Step 8770/44000 | Loss: 3.2125 | LR: 4.59e-04 | 1253.4ms/step | 16.0 samples/s +Step 8780/44000 | Loss: 3.1766 | LR: 4.59e-04 | 1253.0ms/step | 16.0 samples/s +Step 8790/44000 | Loss: 3.1647 | LR: 4.59e-04 | 1254.2ms/step | 15.9 samples/s +Step 8800/44000 | Loss: 3.1578 | LR: 4.58e-04 | 1255.0ms/step | 15.9 samples/s +Step 8810/44000 | Loss: 3.1799 | LR: 4.58e-04 | 1255.7ms/step | 15.9 samples/s +Step 8820/44000 | Loss: 3.1639 | LR: 4.58e-04 | 1253.3ms/step | 16.0 samples/s +Step 8830/44000 | Loss: 3.1696 | LR: 4.58e-04 | 1255.2ms/step | 15.9 samples/s +Step 8840/44000 | Loss: 3.1802 | LR: 4.58e-04 | 1255.6ms/step | 15.9 samples/s +Step 8850/44000 | Loss: 3.1818 | LR: 4.58e-04 | 1255.0ms/step | 15.9 samples/s +Step 8860/44000 | Loss: 3.1995 | LR: 4.58e-04 | 1255.1ms/step | 15.9 samples/s +Step 8870/44000 | Loss: 3.1821 | LR: 4.58e-04 | 1255.0ms/step | 15.9 samples/s +Step 8880/44000 | Loss: 3.1611 | LR: 4.58e-04 | 1254.9ms/step | 15.9 samples/s +Step 8890/44000 | Loss: 3.2070 | LR: 4.58e-04 | 1254.3ms/step | 15.9 samples/s +Step 8900/44000 | Loss: 3.1684 | LR: 4.57e-04 | 1254.0ms/step | 15.9 samples/s +Step 8910/44000 | Loss: 3.2039 | LR: 4.57e-04 | 1260.1ms/step | 15.9 samples/s +Step 8920/44000 | Loss: 3.1841 | LR: 4.57e-04 | 1257.5ms/step | 15.9 samples/s +Step 8930/44000 | Loss: 3.1811 | LR: 4.57e-04 | 1255.9ms/step | 15.9 samples/s +Step 8940/44000 | Loss: 3.1729 | LR: 4.57e-04 | 1255.2ms/step | 15.9 samples/s +Step 8950/44000 | Loss: 3.1806 | LR: 4.57e-04 | 1253.4ms/step | 16.0 samples/s +Step 8960/44000 | Loss: 3.1636 | LR: 4.57e-04 | 1254.2ms/step | 15.9 samples/s +Step 8970/44000 | Loss: 3.1852 | LR: 4.57e-04 | 1254.2ms/step | 15.9 samples/s +Step 8980/44000 | Loss: 3.1844 | LR: 4.57e-04 | 1254.6ms/step | 15.9 samples/s +Step 8990/44000 | Loss: 3.1699 | LR: 4.57e-04 | 1255.9ms/step | 15.9 samples/s +Step 9000/44000 | Loss: 3.1718 | LR: 4.56e-04 | 1254.0ms/step | 15.9 samples/s +Validation at step 9000: Loss = 3.1912, Perplexity = 24.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-9000 +Step 9010/44000 | Loss: 3.1707 | LR: 4.56e-04 | 3682.5ms/step | 5.4 samples/s +Step 9020/44000 | Loss: 3.1882 | LR: 4.56e-04 | 1256.8ms/step | 15.9 samples/s +Step 9030/44000 | Loss: 3.1626 | LR: 4.56e-04 | 1258.5ms/step | 15.9 samples/s +Step 9040/44000 | Loss: 3.1666 | LR: 4.56e-04 | 1259.6ms/step | 15.9 samples/s +Step 9050/44000 | Loss: 3.1678 | LR: 4.56e-04 | 1260.2ms/step | 15.9 samples/s +Step 9060/44000 | Loss: 3.2047 | LR: 4.56e-04 | 1259.6ms/step | 15.9 samples/s +Step 9070/44000 | Loss: 3.1997 | LR: 4.56e-04 | 1259.9ms/step | 15.9 samples/s +Step 9080/44000 | Loss: 3.2215 | LR: 4.56e-04 | 1260.6ms/step | 15.9 samples/s +Step 9090/44000 | Loss: 3.1504 | LR: 4.56e-04 | 1261.4ms/step | 15.9 samples/s +Step 9100/44000 | Loss: 3.1560 | LR: 4.55e-04 | 1260.7ms/step | 15.9 samples/s +Step 9110/44000 | Loss: 3.1523 | LR: 4.55e-04 | 1259.7ms/step | 15.9 samples/s +Step 9120/44000 | Loss: 3.1551 | LR: 4.55e-04 | 1261.2ms/step | 15.9 samples/s +Step 9130/44000 | Loss: 3.2011 | LR: 4.55e-04 | 1262.5ms/step | 15.8 samples/s +Step 9140/44000 | Loss: 3.1490 | LR: 4.55e-04 | 1263.2ms/step | 15.8 samples/s +Step 9150/44000 | Loss: 3.1769 | LR: 4.55e-04 | 1261.6ms/step | 15.9 samples/s +Step 9160/44000 | Loss: 3.1526 | LR: 4.55e-04 | 1261.9ms/step | 15.8 samples/s +Step 9170/44000 | Loss: 3.1332 | LR: 4.55e-04 | 1261.2ms/step | 15.9 samples/s +Step 9180/44000 | Loss: 3.1818 | LR: 4.55e-04 | 1260.2ms/step | 15.9 samples/s +Step 9190/44000 | Loss: 3.1577 | LR: 4.54e-04 | 1260.6ms/step | 15.9 samples/s +Step 9200/44000 | Loss: 3.1856 | LR: 4.54e-04 | 1265.0ms/step | 15.8 samples/s +Step 9210/44000 | Loss: 3.1798 | LR: 4.54e-04 | 1263.6ms/step | 15.8 samples/s +Step 9220/44000 | Loss: 3.1701 | LR: 4.54e-04 | 1261.8ms/step | 15.9 samples/s +Step 9230/44000 | Loss: 3.1697 | LR: 4.54e-04 | 1259.2ms/step | 15.9 samples/s +Step 9240/44000 | Loss: 3.2202 | LR: 4.54e-04 | 1254.3ms/step | 15.9 samples/s +Step 9250/44000 | Loss: 3.1742 | LR: 4.54e-04 | 1254.4ms/step | 15.9 samples/s +Validation at step 9250: Loss = 3.1844, Perplexity = 24.15 +Step 9260/44000 | Loss: 3.1804 | LR: 4.54e-04 | 3362.9ms/step | 5.9 samples/s +Step 9270/44000 | Loss: 3.1782 | LR: 4.54e-04 | 1254.3ms/step | 15.9 samples/s +Step 9280/44000 | Loss: 3.1769 | LR: 4.54e-04 | 1252.9ms/step | 16.0 samples/s +Step 9290/44000 | Loss: 3.1883 | LR: 4.53e-04 | 1251.7ms/step | 16.0 samples/s +Step 9300/44000 | Loss: 3.1489 | LR: 4.53e-04 | 1251.7ms/step | 16.0 samples/s +Step 9310/44000 | Loss: 3.1686 | LR: 4.53e-04 | 1251.3ms/step | 16.0 samples/s +Step 9320/44000 | Loss: 3.1519 | LR: 4.53e-04 | 1252.4ms/step | 16.0 samples/s +Step 9330/44000 | Loss: 3.1727 | LR: 4.53e-04 | 1254.6ms/step | 15.9 samples/s +Step 9340/44000 | Loss: 3.1559 | LR: 4.53e-04 | 1254.7ms/step | 15.9 samples/s +Step 9350/44000 | Loss: 3.1461 | LR: 4.53e-04 | 1254.7ms/step | 15.9 samples/s +Step 9360/44000 | Loss: 3.1731 | LR: 4.53e-04 | 1254.6ms/step | 15.9 samples/s +Step 9370/44000 | Loss: 3.1756 | LR: 4.53e-04 | 1254.3ms/step | 15.9 samples/s +Step 9380/44000 | Loss: 3.1719 | LR: 4.52e-04 | 1254.1ms/step | 15.9 samples/s +Step 9390/44000 | Loss: 3.1838 | LR: 4.52e-04 | 1254.6ms/step | 15.9 samples/s +Step 9400/44000 | Loss: 3.1607 | LR: 4.52e-04 | 1256.3ms/step | 15.9 samples/s +Step 9410/44000 | Loss: 3.1662 | LR: 4.52e-04 | 1255.4ms/step | 15.9 samples/s +Step 9420/44000 | Loss: 3.1762 | LR: 4.52e-04 | 1256.5ms/step | 15.9 samples/s +Step 9430/44000 | Loss: 3.1619 | LR: 4.52e-04 | 1253.5ms/step | 16.0 samples/s +Step 9440/44000 | Loss: 3.1409 | LR: 4.52e-04 | 1253.0ms/step | 16.0 samples/s +Step 9450/44000 | Loss: 3.1641 | LR: 4.52e-04 | 1254.6ms/step | 15.9 samples/s +Step 9460/44000 | Loss: 3.1770 | LR: 4.52e-04 | 1255.2ms/step | 15.9 samples/s +Step 9470/44000 | Loss: 3.1555 | LR: 4.52e-04 | 1253.0ms/step | 16.0 samples/s +Step 9480/44000 | Loss: 3.1534 | LR: 4.51e-04 | 1253.0ms/step | 16.0 samples/s +Step 9490/44000 | Loss: 3.1676 | LR: 4.51e-04 | 1252.7ms/step | 16.0 samples/s +Step 9500/44000 | Loss: 3.1698 | LR: 4.51e-04 | 1253.0ms/step | 16.0 samples/s +Validation at step 9500: Loss = 3.1781, Perplexity = 24.00 +Step 9510/44000 | Loss: 3.2008 | LR: 4.51e-04 | 3367.0ms/step | 5.9 samples/s +Step 9520/44000 | Loss: 3.1545 | LR: 4.51e-04 | 1252.9ms/step | 16.0 samples/s +Step 9530/44000 | Loss: 3.1925 | LR: 4.51e-04 | 1253.0ms/step | 16.0 samples/s +Step 9540/44000 | Loss: 3.1907 | LR: 4.51e-04 | 1253.4ms/step | 16.0 samples/s +Step 9550/44000 | Loss: 3.1777 | LR: 4.51e-04 | 1253.3ms/step | 16.0 samples/s +Step 9560/44000 | Loss: 3.1582 | LR: 4.51e-04 | 1253.5ms/step | 16.0 samples/s +Step 9570/44000 | Loss: 3.1544 | LR: 4.50e-04 | 1253.3ms/step | 16.0 samples/s +Step 9580/44000 | Loss: 3.1834 | LR: 4.50e-04 | 1253.5ms/step | 16.0 samples/s +Step 9590/44000 | Loss: 3.1486 | LR: 4.50e-04 | 1253.4ms/step | 16.0 samples/s +Step 9600/44000 | Loss: 3.1517 | LR: 4.50e-04 | 1253.8ms/step | 16.0 samples/s +Step 9610/44000 | Loss: 3.1521 | LR: 4.50e-04 | 1253.2ms/step | 16.0 samples/s +Step 9620/44000 | Loss: 3.1916 | LR: 4.50e-04 | 1252.5ms/step | 16.0 samples/s +Step 9630/44000 | Loss: 3.1686 | LR: 4.50e-04 | 1252.6ms/step | 16.0 samples/s +Step 9640/44000 | Loss: 3.1652 | LR: 4.50e-04 | 1253.0ms/step | 16.0 samples/s +Step 9650/44000 | Loss: 3.1691 | LR: 4.50e-04 | 1253.3ms/step | 16.0 samples/s +Step 9660/44000 | Loss: 3.1431 | LR: 4.49e-04 | 1252.4ms/step | 16.0 samples/s +Step 9670/44000 | Loss: 3.1650 | LR: 4.49e-04 | 1252.2ms/step | 16.0 samples/s +Step 9680/44000 | Loss: 3.1697 | LR: 4.49e-04 | 1252.6ms/step | 16.0 samples/s +Step 9690/44000 | Loss: 3.1688 | LR: 4.49e-04 | 1253.4ms/step | 16.0 samples/s +Step 9700/44000 | Loss: 3.1719 | LR: 4.49e-04 | 1253.1ms/step | 16.0 samples/s +Step 9710/44000 | Loss: 3.1697 | LR: 4.49e-04 | 1253.6ms/step | 16.0 samples/s +Step 9720/44000 | Loss: 3.1568 | LR: 4.49e-04 | 1253.2ms/step | 16.0 samples/s +Step 9730/44000 | Loss: 3.1723 | LR: 4.49e-04 | 1253.3ms/step | 16.0 samples/s +Step 9740/44000 | Loss: 3.1596 | LR: 4.49e-04 | 1253.5ms/step | 16.0 samples/s +Step 9750/44000 | Loss: 3.1548 | LR: 4.48e-04 | 1252.0ms/step | 16.0 samples/s +Validation at step 9750: Loss = 3.1728, Perplexity = 23.87 +Step 9760/44000 | Loss: 3.1316 | LR: 4.48e-04 | 3361.1ms/step | 6.0 samples/s +Step 9770/44000 | Loss: 3.1344 | LR: 4.48e-04 | 1252.1ms/step | 16.0 samples/s +Step 9780/44000 | Loss: 3.1599 | LR: 4.48e-04 | 1253.8ms/step | 16.0 samples/s +Step 9790/44000 | Loss: 3.1763 | LR: 4.48e-04 | 1253.1ms/step | 16.0 samples/s +Step 9800/44000 | Loss: 3.1591 | LR: 4.48e-04 | 1253.9ms/step | 16.0 samples/s +Step 9810/44000 | Loss: 3.1785 | LR: 4.48e-04 | 1253.1ms/step | 16.0 samples/s +Step 9820/44000 | Loss: 3.1777 | LR: 4.48e-04 | 1253.3ms/step | 16.0 samples/s +Step 9830/44000 | Loss: 3.2051 | LR: 4.48e-04 | 1253.6ms/step | 16.0 samples/s +Step 9840/44000 | Loss: 3.1509 | LR: 4.47e-04 | 1252.6ms/step | 16.0 samples/s +Step 9850/44000 | Loss: 3.1701 | LR: 4.47e-04 | 1254.1ms/step | 15.9 samples/s +Step 9860/44000 | Loss: 3.1324 | LR: 4.47e-04 | 1258.4ms/step | 15.9 samples/s +Step 9870/44000 | Loss: 3.1551 | LR: 4.47e-04 | 1257.5ms/step | 15.9 samples/s +Step 9880/44000 | Loss: 3.1577 | LR: 4.47e-04 | 1253.6ms/step | 16.0 samples/s +Step 9890/44000 | Loss: 3.1398 | LR: 4.47e-04 | 1254.7ms/step | 15.9 samples/s +Step 9900/44000 | Loss: 3.1643 | LR: 4.47e-04 | 1254.5ms/step | 15.9 samples/s +Step 9910/44000 | Loss: 3.1781 | LR: 4.47e-04 | 1255.3ms/step | 15.9 samples/s +Step 9920/44000 | Loss: 3.1436 | LR: 4.47e-04 | 1255.4ms/step | 15.9 samples/s +Step 9930/44000 | Loss: 3.1367 | LR: 4.46e-04 | 1254.0ms/step | 15.9 samples/s +Step 9940/44000 | Loss: 3.1604 | LR: 4.46e-04 | 1255.8ms/step | 15.9 samples/s +Step 9950/44000 | Loss: 3.1426 | LR: 4.46e-04 | 1255.0ms/step | 15.9 samples/s +Step 9960/44000 | Loss: 3.1568 | LR: 4.46e-04 | 1254.9ms/step | 15.9 samples/s +Step 9970/44000 | Loss: 3.2003 | LR: 4.46e-04 | 1254.6ms/step | 15.9 samples/s +Step 9980/44000 | Loss: 3.1501 | LR: 4.46e-04 | 1254.0ms/step | 15.9 samples/s +Step 9990/44000 | Loss: 3.1602 | LR: 4.46e-04 | 1255.0ms/step | 15.9 samples/s +Step 10000/44000 | Loss: 3.1556 | LR: 4.46e-04 | 1254.8ms/step | 15.9 samples/s +Validation at step 10000: Loss = 3.1667, Perplexity = 23.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-10000 +Step 10010/44000 | Loss: 3.1498 | LR: 4.46e-04 | 3663.7ms/step | 5.5 samples/s +Step 10020/44000 | Loss: 3.1786 | LR: 4.45e-04 | 1258.7ms/step | 15.9 samples/s +Step 10030/44000 | Loss: 3.1601 | LR: 4.45e-04 | 1257.8ms/step | 15.9 samples/s +Step 10040/44000 | Loss: 3.1275 | LR: 4.45e-04 | 1260.0ms/step | 15.9 samples/s +Step 10050/44000 | Loss: 3.1371 | LR: 4.45e-04 | 1259.8ms/step | 15.9 samples/s +Step 10060/44000 | Loss: 3.1367 | LR: 4.45e-04 | 1260.4ms/step | 15.9 samples/s +Step 10070/44000 | Loss: 3.1396 | LR: 4.45e-04 | 1260.0ms/step | 15.9 samples/s +Step 10080/44000 | Loss: 3.1494 | LR: 4.45e-04 | 1259.6ms/step | 15.9 samples/s +Step 10090/44000 | Loss: 3.1477 | LR: 4.45e-04 | 1255.6ms/step | 15.9 samples/s +Step 10100/44000 | Loss: 3.1651 | LR: 4.45e-04 | 1254.1ms/step | 15.9 samples/s +Step 10110/44000 | Loss: 3.1727 | LR: 4.44e-04 | 1254.9ms/step | 15.9 samples/s +Step 10120/44000 | Loss: 3.1954 | LR: 4.44e-04 | 1254.9ms/step | 15.9 samples/s +Step 10130/44000 | Loss: 3.1426 | LR: 4.44e-04 | 1259.6ms/step | 15.9 samples/s +Step 10140/44000 | Loss: 3.2011 | LR: 4.44e-04 | 1258.4ms/step | 15.9 samples/s +Step 10150/44000 | Loss: 3.1574 | LR: 4.44e-04 | 1257.8ms/step | 15.9 samples/s +Step 10160/44000 | Loss: 3.1593 | LR: 4.44e-04 | 1253.3ms/step | 16.0 samples/s +Step 10170/44000 | Loss: 3.1307 | LR: 4.44e-04 | 1253.0ms/step | 16.0 samples/s +Step 10180/44000 | Loss: 3.1658 | LR: 4.44e-04 | 1256.1ms/step | 15.9 samples/s +Step 10190/44000 | Loss: 3.1356 | LR: 4.43e-04 | 1259.7ms/step | 15.9 samples/s +Step 10200/44000 | Loss: 3.1386 | LR: 4.43e-04 | 1259.2ms/step | 15.9 samples/s +Step 10210/44000 | Loss: 3.1541 | LR: 4.43e-04 | 1257.1ms/step | 15.9 samples/s +Step 10220/44000 | Loss: 3.1397 | LR: 4.43e-04 | 1253.5ms/step | 16.0 samples/s +Step 10230/44000 | Loss: 3.1630 | LR: 4.43e-04 | 1254.4ms/step | 15.9 samples/s +Step 10240/44000 | Loss: 3.1403 | LR: 4.43e-04 | 1254.3ms/step | 15.9 samples/s +Step 10250/44000 | Loss: 3.1608 | LR: 4.43e-04 | 1254.6ms/step | 15.9 samples/s +Validation at step 10250: Loss = 3.1602, Perplexity = 23.57 +Step 10260/44000 | Loss: 3.1206 | LR: 4.43e-04 | 3363.4ms/step | 5.9 samples/s +Step 10270/44000 | Loss: 3.1091 | LR: 4.43e-04 | 1254.6ms/step | 15.9 samples/s +Step 10280/44000 | Loss: 3.1104 | LR: 4.42e-04 | 1255.0ms/step | 15.9 samples/s +Step 10290/44000 | Loss: 3.1042 | LR: 4.42e-04 | 1254.4ms/step | 15.9 samples/s +Step 10300/44000 | Loss: 3.1371 | LR: 4.42e-04 | 1254.5ms/step | 15.9 samples/s +Step 10310/44000 | Loss: 3.1623 | LR: 4.42e-04 | 1255.3ms/step | 15.9 samples/s +Step 10320/44000 | Loss: 3.1280 | LR: 4.42e-04 | 1255.7ms/step | 15.9 samples/s +Step 10330/44000 | Loss: 3.1694 | LR: 4.42e-04 | 1254.4ms/step | 15.9 samples/s +Step 10340/44000 | Loss: 3.1434 | LR: 4.42e-04 | 1255.2ms/step | 15.9 samples/s +Step 10350/44000 | Loss: 3.1568 | LR: 4.42e-04 | 1255.2ms/step | 15.9 samples/s +Step 10360/44000 | Loss: 3.1291 | LR: 4.42e-04 | 1254.0ms/step | 15.9 samples/s +Step 10370/44000 | Loss: 3.1567 | LR: 4.41e-04 | 1253.8ms/step | 16.0 samples/s +Step 10380/44000 | Loss: 3.1681 | LR: 4.41e-04 | 1254.2ms/step | 15.9 samples/s +Step 10390/44000 | Loss: 3.1379 | LR: 4.41e-04 | 1253.2ms/step | 16.0 samples/s +Step 10400/44000 | Loss: 3.1694 | LR: 4.41e-04 | 1254.2ms/step | 15.9 samples/s +Step 10410/44000 | Loss: 3.1751 | LR: 4.41e-04 | 1254.3ms/step | 15.9 samples/s +Step 10420/44000 | Loss: 3.1695 | LR: 4.41e-04 | 1255.5ms/step | 15.9 samples/s +Step 10430/44000 | Loss: 3.1598 | LR: 4.41e-04 | 1255.3ms/step | 15.9 samples/s +Step 10440/44000 | Loss: 3.1296 | LR: 4.41e-04 | 1254.6ms/step | 15.9 samples/s +Step 10450/44000 | Loss: 3.1421 | LR: 4.40e-04 | 1255.5ms/step | 15.9 samples/s +Step 10460/44000 | Loss: 3.1368 | LR: 4.40e-04 | 1255.9ms/step | 15.9 samples/s +Step 10470/44000 | Loss: 3.1661 | LR: 4.40e-04 | 1255.8ms/step | 15.9 samples/s +Step 10480/44000 | Loss: 3.1343 | LR: 4.40e-04 | 1254.9ms/step | 15.9 samples/s +Step 10490/44000 | Loss: 3.1159 | LR: 4.40e-04 | 1255.5ms/step | 15.9 samples/s +Step 10500/44000 | Loss: 3.1702 | LR: 4.40e-04 | 1254.7ms/step | 15.9 samples/s +Validation at step 10500: Loss = 3.1578, Perplexity = 23.52 +Step 10510/44000 | Loss: 3.1795 | LR: 4.40e-04 | 3377.1ms/step | 5.9 samples/s +Step 10520/44000 | Loss: 3.1188 | LR: 4.40e-04 | 1256.2ms/step | 15.9 samples/s +Step 10530/44000 | Loss: 3.1375 | LR: 4.40e-04 | 1255.1ms/step | 15.9 samples/s +Step 10540/44000 | Loss: 3.1471 | LR: 4.39e-04 | 1255.4ms/step | 15.9 samples/s +Step 10550/44000 | Loss: 3.1387 | LR: 4.39e-04 | 1255.0ms/step | 15.9 samples/s +Step 10560/44000 | Loss: 3.1254 | LR: 4.39e-04 | 1255.4ms/step | 15.9 samples/s +Step 10570/44000 | Loss: 3.1393 | LR: 4.39e-04 | 1255.2ms/step | 15.9 samples/s +Step 10580/44000 | Loss: 3.1570 | LR: 4.39e-04 | 1255.9ms/step | 15.9 samples/s +Step 10590/44000 | Loss: 3.1065 | LR: 4.39e-04 | 1255.0ms/step | 15.9 samples/s +Step 10600/44000 | Loss: 3.1285 | LR: 4.39e-04 | 1254.9ms/step | 15.9 samples/s +Step 10610/44000 | Loss: 3.1184 | LR: 4.39e-04 | 1254.7ms/step | 15.9 samples/s +Step 10620/44000 | Loss: 3.1646 | LR: 4.38e-04 | 1255.6ms/step | 15.9 samples/s +Step 10630/44000 | Loss: 3.1533 | LR: 4.38e-04 | 1254.4ms/step | 15.9 samples/s +Step 10640/44000 | Loss: 3.1315 | LR: 4.38e-04 | 1254.9ms/step | 15.9 samples/s +Step 10650/44000 | Loss: 3.1465 | LR: 4.38e-04 | 1255.3ms/step | 15.9 samples/s +Step 10660/44000 | Loss: 3.1344 | LR: 4.38e-04 | 1255.1ms/step | 15.9 samples/s +Step 10670/44000 | Loss: 3.1370 | LR: 4.38e-04 | 1259.4ms/step | 15.9 samples/s +Step 10680/44000 | Loss: 3.1542 | LR: 4.38e-04 | 1257.6ms/step | 15.9 samples/s +Step 10690/44000 | Loss: 3.1061 | LR: 4.38e-04 | 1258.8ms/step | 15.9 samples/s +Step 10700/44000 | Loss: 3.0789 | LR: 4.37e-04 | 1253.7ms/step | 16.0 samples/s +Step 10710/44000 | Loss: 3.1122 | LR: 4.37e-04 | 1253.5ms/step | 16.0 samples/s +Step 10720/44000 | Loss: 3.1008 | LR: 4.37e-04 | 1253.6ms/step | 16.0 samples/s +Step 10730/44000 | Loss: 3.1126 | LR: 4.37e-04 | 1253.5ms/step | 16.0 samples/s +Step 10740/44000 | Loss: 3.1249 | LR: 4.37e-04 | 1253.7ms/step | 16.0 samples/s +Step 10750/44000 | Loss: 3.1176 | LR: 4.37e-04 | 1253.6ms/step | 16.0 samples/s +Validation at step 10750: Loss = 3.1512, Perplexity = 23.36 +Step 10760/44000 | Loss: 3.1531 | LR: 4.37e-04 | 3367.1ms/step | 5.9 samples/s +Step 10770/44000 | Loss: 3.1544 | LR: 4.37e-04 | 1253.3ms/step | 16.0 samples/s +Step 10780/44000 | Loss: 3.1279 | LR: 4.37e-04 | 1253.4ms/step | 16.0 samples/s +Step 10790/44000 | Loss: 3.1369 | LR: 4.36e-04 | 1253.2ms/step | 16.0 samples/s +Step 10800/44000 | Loss: 3.1216 | LR: 4.36e-04 | 1253.0ms/step | 16.0 samples/s +Step 10810/44000 | Loss: 3.1399 | LR: 4.36e-04 | 1253.5ms/step | 16.0 samples/s +Step 10820/44000 | Loss: 3.1480 | LR: 4.36e-04 | 1252.6ms/step | 16.0 samples/s +Step 10830/44000 | Loss: 3.1301 | LR: 4.36e-04 | 1254.4ms/step | 15.9 samples/s +Step 10840/44000 | Loss: 3.1559 | LR: 4.36e-04 | 1253.8ms/step | 16.0 samples/s +Step 10850/44000 | Loss: 3.1491 | LR: 4.36e-04 | 1254.9ms/step | 15.9 samples/s +Step 10860/44000 | Loss: 3.1700 | LR: 4.36e-04 | 1256.0ms/step | 15.9 samples/s +Step 10870/44000 | Loss: 3.1326 | LR: 4.35e-04 | 1253.7ms/step | 16.0 samples/s +Step 10880/44000 | Loss: 3.1282 | LR: 4.35e-04 | 1254.1ms/step | 15.9 samples/s +Step 10890/44000 | Loss: 3.1355 | LR: 4.35e-04 | 1255.5ms/step | 15.9 samples/s +Step 10900/44000 | Loss: 3.1382 | LR: 4.35e-04 | 1254.9ms/step | 15.9 samples/s +Step 10910/44000 | Loss: 3.1153 | LR: 4.35e-04 | 1255.4ms/step | 15.9 samples/s +Step 10920/44000 | Loss: 3.1009 | LR: 4.35e-04 | 1259.3ms/step | 15.9 samples/s +Step 10930/44000 | Loss: 3.1683 | LR: 4.35e-04 | 1258.7ms/step | 15.9 samples/s +Step 10940/44000 | Loss: 3.1374 | LR: 4.35e-04 | 1257.4ms/step | 15.9 samples/s +Step 10950/44000 | Loss: 3.1172 | LR: 4.34e-04 | 1256.1ms/step | 15.9 samples/s +Step 10960/44000 | Loss: 3.1468 | LR: 4.34e-04 | 1252.7ms/step | 16.0 samples/s +Step 10970/44000 | Loss: 3.1315 | LR: 4.34e-04 | 1252.5ms/step | 16.0 samples/s +Step 10980/44000 | Loss: 3.1223 | LR: 4.34e-04 | 1255.7ms/step | 15.9 samples/s +Step 10990/44000 | Loss: 3.1364 | LR: 4.34e-04 | 1255.2ms/step | 15.9 samples/s +Step 11000/44000 | Loss: 3.1277 | LR: 4.34e-04 | 1255.3ms/step | 15.9 samples/s +Validation at step 11000: Loss = 3.1455, Perplexity = 23.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-11000 +Step 11010/44000 | Loss: 3.1448 | LR: 4.34e-04 | 3687.1ms/step | 5.4 samples/s +Step 11020/44000 | Loss: 3.1522 | LR: 4.34e-04 | 1256.1ms/step | 15.9 samples/s +Step 11030/44000 | Loss: 3.0981 | LR: 4.33e-04 | 1258.0ms/step | 15.9 samples/s +Step 11040/44000 | Loss: 3.1415 | LR: 4.33e-04 | 1254.7ms/step | 15.9 samples/s +Step 11050/44000 | Loss: 3.1219 | LR: 4.33e-04 | 1254.0ms/step | 15.9 samples/s +Step 11060/44000 | Loss: 3.1368 | LR: 4.33e-04 | 1253.6ms/step | 16.0 samples/s +Step 11070/44000 | Loss: 3.1294 | LR: 4.33e-04 | 1254.1ms/step | 15.9 samples/s +Step 11080/44000 | Loss: 3.0954 | LR: 4.33e-04 | 1252.9ms/step | 16.0 samples/s +Step 11090/44000 | Loss: 3.1542 | LR: 4.33e-04 | 1253.4ms/step | 16.0 samples/s +Step 11100/44000 | Loss: 3.1288 | LR: 4.33e-04 | 1253.1ms/step | 16.0 samples/s +Step 11110/44000 | Loss: 3.0937 | LR: 4.32e-04 | 1253.5ms/step | 16.0 samples/s +Step 11120/44000 | Loss: 3.1213 | LR: 4.32e-04 | 1254.1ms/step | 15.9 samples/s +Step 11130/44000 | Loss: 3.1185 | LR: 4.32e-04 | 1254.4ms/step | 15.9 samples/s +Step 11140/44000 | Loss: 3.1214 | LR: 4.32e-04 | 1254.1ms/step | 15.9 samples/s +Step 11150/44000 | Loss: 3.1414 | LR: 4.32e-04 | 1254.5ms/step | 15.9 samples/s +Step 11160/44000 | Loss: 3.1251 | LR: 4.32e-04 | 1254.8ms/step | 15.9 samples/s +Step 11170/44000 | Loss: 3.1206 | LR: 4.32e-04 | 1254.5ms/step | 15.9 samples/s +Step 11180/44000 | Loss: 3.1302 | LR: 4.32e-04 | 1253.1ms/step | 16.0 samples/s +Step 11190/44000 | Loss: 3.1285 | LR: 4.31e-04 | 1253.5ms/step | 16.0 samples/s +Step 11200/44000 | Loss: 3.1282 | LR: 4.31e-04 | 1254.0ms/step | 15.9 samples/s +Step 11210/44000 | Loss: 3.1314 | LR: 4.31e-04 | 1253.9ms/step | 16.0 samples/s +Step 11220/44000 | Loss: 3.1056 | LR: 4.31e-04 | 1254.7ms/step | 15.9 samples/s +Step 11230/44000 | Loss: 3.1296 | LR: 4.31e-04 | 1254.1ms/step | 15.9 samples/s +Step 11240/44000 | Loss: 3.1189 | LR: 4.31e-04 | 1254.7ms/step | 15.9 samples/s +Step 11250/44000 | Loss: 3.1256 | LR: 4.31e-04 | 1254.8ms/step | 15.9 samples/s +Validation at step 11250: Loss = 3.1402, Perplexity = 23.11 +Step 11260/44000 | Loss: 3.0998 | LR: 4.31e-04 | 3362.1ms/step | 5.9 samples/s +Step 11270/44000 | Loss: 3.1421 | LR: 4.30e-04 | 1254.4ms/step | 15.9 samples/s +Step 11280/44000 | Loss: 3.1173 | LR: 4.30e-04 | 1253.9ms/step | 15.9 samples/s +Step 11290/44000 | Loss: 3.1279 | LR: 4.30e-04 | 1254.6ms/step | 15.9 samples/s +Step 11300/44000 | Loss: 3.1456 | LR: 4.30e-04 | 1254.4ms/step | 15.9 samples/s +Step 11310/44000 | Loss: 3.1130 | LR: 4.30e-04 | 1253.0ms/step | 16.0 samples/s +Step 11320/44000 | Loss: 3.1254 | LR: 4.30e-04 | 1255.5ms/step | 15.9 samples/s +Step 11330/44000 | Loss: 3.1267 | LR: 4.30e-04 | 1255.9ms/step | 15.9 samples/s +Step 11340/44000 | Loss: 3.0973 | LR: 4.30e-04 | 1256.2ms/step | 15.9 samples/s +Step 11350/44000 | Loss: 3.1243 | LR: 4.29e-04 | 1254.5ms/step | 15.9 samples/s +Step 11360/44000 | Loss: 3.1073 | LR: 4.29e-04 | 1253.9ms/step | 16.0 samples/s +Step 11370/44000 | Loss: 3.1382 | LR: 4.29e-04 | 1254.4ms/step | 15.9 samples/s +Step 11380/44000 | Loss: 3.1373 | LR: 4.29e-04 | 1254.8ms/step | 15.9 samples/s +Step 11390/44000 | Loss: 3.1547 | LR: 4.29e-04 | 1254.0ms/step | 15.9 samples/s +Step 11400/44000 | Loss: 3.1184 | LR: 4.29e-04 | 1256.4ms/step | 15.9 samples/s +Step 11410/44000 | Loss: 3.0873 | LR: 4.29e-04 | 1255.5ms/step | 15.9 samples/s +Step 11420/44000 | Loss: 3.1359 | LR: 4.29e-04 | 1254.4ms/step | 15.9 samples/s +Step 11430/44000 | Loss: 3.1179 | LR: 4.28e-04 | 1254.6ms/step | 15.9 samples/s +Step 11440/44000 | Loss: 3.1425 | LR: 4.28e-04 | 1254.7ms/step | 15.9 samples/s +Step 11450/44000 | Loss: 3.1182 | LR: 4.28e-04 | 1255.2ms/step | 15.9 samples/s +Step 11460/44000 | Loss: 3.1528 | LR: 4.28e-04 | 1255.5ms/step | 15.9 samples/s +Step 11470/44000 | Loss: 3.1268 | LR: 4.28e-04 | 1254.0ms/step | 15.9 samples/s +Step 11480/44000 | Loss: 3.1168 | LR: 4.28e-04 | 1254.3ms/step | 15.9 samples/s +Step 11490/44000 | Loss: 3.1181 | LR: 4.28e-04 | 1253.3ms/step | 16.0 samples/s +Step 11500/44000 | Loss: 3.1200 | LR: 4.28e-04 | 1254.0ms/step | 15.9 samples/s +Validation at step 11500: Loss = 3.1350, Perplexity = 22.99 +Step 11510/44000 | Loss: 3.1343 | LR: 4.27e-04 | 3370.7ms/step | 5.9 samples/s +Step 11520/44000 | Loss: 3.0941 | LR: 4.27e-04 | 1253.6ms/step | 16.0 samples/s +Step 11530/44000 | Loss: 3.1236 | LR: 4.27e-04 | 1253.1ms/step | 16.0 samples/s +Step 11540/44000 | Loss: 3.1222 | LR: 4.27e-04 | 1253.1ms/step | 16.0 samples/s +Step 11550/44000 | Loss: 3.1049 | LR: 4.27e-04 | 1254.0ms/step | 15.9 samples/s +Step 11560/44000 | Loss: 3.1226 | LR: 4.27e-04 | 1255.0ms/step | 15.9 samples/s +Step 11570/44000 | Loss: 3.1236 | LR: 4.27e-04 | 1255.6ms/step | 15.9 samples/s +Step 11580/44000 | Loss: 3.1226 | LR: 4.27e-04 | 1256.4ms/step | 15.9 samples/s +Step 11590/44000 | Loss: 3.1317 | LR: 4.26e-04 | 1255.6ms/step | 15.9 samples/s +Step 11600/44000 | Loss: 3.1171 | LR: 4.26e-04 | 1254.6ms/step | 15.9 samples/s +Step 11610/44000 | Loss: 3.1248 | LR: 4.26e-04 | 1254.3ms/step | 15.9 samples/s +Step 11620/44000 | Loss: 3.1185 | LR: 4.26e-04 | 1254.5ms/step | 15.9 samples/s +Step 11630/44000 | Loss: 3.0983 | LR: 4.26e-04 | 1254.2ms/step | 15.9 samples/s +Step 11640/44000 | Loss: 3.1230 | LR: 4.26e-04 | 1254.1ms/step | 15.9 samples/s +Step 11650/44000 | Loss: 3.1240 | LR: 4.26e-04 | 1255.4ms/step | 15.9 samples/s +Step 11660/44000 | Loss: 3.0929 | LR: 4.26e-04 | 1255.3ms/step | 15.9 samples/s +Step 11670/44000 | Loss: 3.1166 | LR: 4.25e-04 | 1254.3ms/step | 15.9 samples/s +Step 11680/44000 | Loss: 3.0944 | LR: 4.25e-04 | 1254.1ms/step | 15.9 samples/s +Step 11690/44000 | Loss: 3.1494 | LR: 4.25e-04 | 1255.9ms/step | 15.9 samples/s +Step 11700/44000 | Loss: 3.0869 | LR: 4.25e-04 | 1255.5ms/step | 15.9 samples/s +Step 11710/44000 | Loss: 3.1377 | LR: 4.25e-04 | 1254.9ms/step | 15.9 samples/s +Step 11720/44000 | Loss: 3.1297 | LR: 4.25e-04 | 1255.4ms/step | 15.9 samples/s +Step 11730/44000 | Loss: 3.1279 | LR: 4.25e-04 | 1253.6ms/step | 16.0 samples/s +Step 11740/44000 | Loss: 3.1528 | LR: 4.24e-04 | 1254.8ms/step | 15.9 samples/s +Step 11750/44000 | Loss: 3.1150 | LR: 4.24e-04 | 1256.5ms/step | 15.9 samples/s +Validation at step 11750: Loss = 3.1310, Perplexity = 22.90 +Step 11760/44000 | Loss: 3.1496 | LR: 4.24e-04 | 3374.2ms/step | 5.9 samples/s +Step 11770/44000 | Loss: 3.1425 | LR: 4.24e-04 | 1254.3ms/step | 15.9 samples/s +Step 11780/44000 | Loss: 3.1483 | LR: 4.24e-04 | 1254.8ms/step | 15.9 samples/s +Step 11790/44000 | Loss: 3.1117 | LR: 4.24e-04 | 1253.4ms/step | 16.0 samples/s +Step 11800/44000 | Loss: 3.1051 | LR: 4.24e-04 | 1254.6ms/step | 15.9 samples/s +Step 11810/44000 | Loss: 3.0977 | LR: 4.24e-04 | 1252.9ms/step | 16.0 samples/s +Step 11820/44000 | Loss: 3.1251 | LR: 4.23e-04 | 1254.0ms/step | 15.9 samples/s +Step 11830/44000 | Loss: 3.1498 | LR: 4.23e-04 | 1253.3ms/step | 16.0 samples/s +Step 11840/44000 | Loss: 3.0897 | LR: 4.23e-04 | 1255.0ms/step | 15.9 samples/s +Step 11850/44000 | Loss: 3.1304 | LR: 4.23e-04 | 1255.2ms/step | 15.9 samples/s +Step 11860/44000 | Loss: 3.1051 | LR: 4.23e-04 | 1254.8ms/step | 15.9 samples/s +Step 11870/44000 | Loss: 3.1319 | LR: 4.23e-04 | 1253.7ms/step | 16.0 samples/s +Step 11880/44000 | Loss: 3.1154 | LR: 4.23e-04 | 1254.6ms/step | 15.9 samples/s +Step 11890/44000 | Loss: 3.0890 | LR: 4.23e-04 | 1254.2ms/step | 15.9 samples/s +Step 11900/44000 | Loss: 3.1475 | LR: 4.22e-04 | 1254.9ms/step | 15.9 samples/s +Step 11910/44000 | Loss: 3.1185 | LR: 4.22e-04 | 1254.6ms/step | 15.9 samples/s +Step 11920/44000 | Loss: 3.1164 | LR: 4.22e-04 | 1254.6ms/step | 15.9 samples/s +Step 11930/44000 | Loss: 3.1200 | LR: 4.22e-04 | 1255.0ms/step | 15.9 samples/s +Step 11940/44000 | Loss: 3.1053 | LR: 4.22e-04 | 1253.6ms/step | 16.0 samples/s +Step 11950/44000 | Loss: 3.0844 | LR: 4.22e-04 | 1254.7ms/step | 15.9 samples/s +Step 11960/44000 | Loss: 3.1052 | LR: 4.22e-04 | 1253.7ms/step | 16.0 samples/s +Step 11970/44000 | Loss: 3.1335 | LR: 4.21e-04 | 1252.7ms/step | 16.0 samples/s +Step 11980/44000 | Loss: 3.1078 | LR: 4.21e-04 | 1252.8ms/step | 16.0 samples/s +Step 11990/44000 | Loss: 3.1042 | LR: 4.21e-04 | 1254.0ms/step | 15.9 samples/s +Step 12000/44000 | Loss: 3.1214 | LR: 4.21e-04 | 1254.3ms/step | 15.9 samples/s +Validation at step 12000: Loss = 3.1257, Perplexity = 22.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-12000 +Step 12010/44000 | Loss: 3.1050 | LR: 4.21e-04 | 3664.4ms/step | 5.5 samples/s +Step 12020/44000 | Loss: 3.0868 | LR: 4.21e-04 | 1249.5ms/step | 16.0 samples/s +Step 12030/44000 | Loss: 3.1240 | LR: 4.21e-04 | 1252.1ms/step | 16.0 samples/s +Step 12040/44000 | Loss: 3.1263 | LR: 4.21e-04 | 1253.2ms/step | 16.0 samples/s +Step 12050/44000 | Loss: 3.1113 | LR: 4.20e-04 | 1254.4ms/step | 15.9 samples/s +Step 12060/44000 | Loss: 3.1046 | LR: 4.20e-04 | 1253.9ms/step | 16.0 samples/s +Step 12070/44000 | Loss: 3.1447 | LR: 4.20e-04 | 1252.6ms/step | 16.0 samples/s +Step 12080/44000 | Loss: 3.1170 | LR: 4.20e-04 | 1253.8ms/step | 16.0 samples/s +Step 12090/44000 | Loss: 3.1319 | LR: 4.20e-04 | 1255.7ms/step | 15.9 samples/s +Step 12100/44000 | Loss: 3.1237 | LR: 4.20e-04 | 1254.0ms/step | 15.9 samples/s +Step 12110/44000 | Loss: 3.1216 | LR: 4.20e-04 | 1254.9ms/step | 15.9 samples/s +Step 12120/44000 | Loss: 3.1201 | LR: 4.19e-04 | 1254.8ms/step | 15.9 samples/s +Step 12130/44000 | Loss: 3.1365 | LR: 4.19e-04 | 1254.2ms/step | 15.9 samples/s +Step 12140/44000 | Loss: 3.0870 | LR: 4.19e-04 | 1255.5ms/step | 15.9 samples/s +Step 12150/44000 | Loss: 3.1004 | LR: 4.19e-04 | 1255.1ms/step | 15.9 samples/s +Step 12160/44000 | Loss: 3.1340 | LR: 4.19e-04 | 1255.2ms/step | 15.9 samples/s +Step 12170/44000 | Loss: 3.1181 | LR: 4.19e-04 | 1254.0ms/step | 15.9 samples/s +Step 12180/44000 | Loss: 3.0963 | LR: 4.19e-04 | 1254.8ms/step | 15.9 samples/s +Step 12190/44000 | Loss: 3.1105 | LR: 4.19e-04 | 1255.8ms/step | 15.9 samples/s +Step 12200/44000 | Loss: 3.0783 | LR: 4.18e-04 | 1253.5ms/step | 16.0 samples/s +Step 12210/44000 | Loss: 3.1076 | LR: 4.18e-04 | 1255.1ms/step | 15.9 samples/s +Step 12220/44000 | Loss: 3.0838 | LR: 4.18e-04 | 1254.3ms/step | 15.9 samples/s +Step 12230/44000 | Loss: 3.0835 | LR: 4.18e-04 | 1253.3ms/step | 16.0 samples/s +Step 12240/44000 | Loss: 3.1377 | LR: 4.18e-04 | 1254.5ms/step | 15.9 samples/s +Step 12250/44000 | Loss: 3.1186 | LR: 4.18e-04 | 1254.7ms/step | 15.9 samples/s +Validation at step 12250: Loss = 3.1231, Perplexity = 22.72 +Step 12260/44000 | Loss: 3.0970 | LR: 4.18e-04 | 3374.5ms/step | 5.9 samples/s +Step 12270/44000 | Loss: 3.1226 | LR: 4.17e-04 | 1257.6ms/step | 15.9 samples/s +Step 12280/44000 | Loss: 3.1006 | LR: 4.17e-04 | 1258.1ms/step | 15.9 samples/s +Step 12290/44000 | Loss: 3.1177 | LR: 4.17e-04 | 1254.6ms/step | 15.9 samples/s +Step 12300/44000 | Loss: 3.1193 | LR: 4.17e-04 | 1255.0ms/step | 15.9 samples/s +Step 12310/44000 | Loss: 3.1237 | LR: 4.17e-04 | 1254.9ms/step | 15.9 samples/s +Step 12320/44000 | Loss: 3.1038 | LR: 4.17e-04 | 1255.9ms/step | 15.9 samples/s +Step 12330/44000 | Loss: 3.1330 | LR: 4.17e-04 | 1255.6ms/step | 15.9 samples/s +Step 12340/44000 | Loss: 3.1134 | LR: 4.17e-04 | 1254.6ms/step | 15.9 samples/s +Step 12350/44000 | Loss: 3.1290 | LR: 4.16e-04 | 1254.4ms/step | 15.9 samples/s +Step 12360/44000 | Loss: 3.1120 | LR: 4.16e-04 | 1255.2ms/step | 15.9 samples/s +Step 12370/44000 | Loss: 3.1069 | LR: 4.16e-04 | 1254.8ms/step | 15.9 samples/s +Step 12380/44000 | Loss: 3.0782 | LR: 4.16e-04 | 1256.1ms/step | 15.9 samples/s +Step 12390/44000 | Loss: 3.1129 | LR: 4.16e-04 | 1255.2ms/step | 15.9 samples/s +Step 12400/44000 | Loss: 3.0791 | LR: 4.16e-04 | 1254.2ms/step | 15.9 samples/s +Step 12410/44000 | Loss: 3.1018 | LR: 4.16e-04 | 1255.3ms/step | 15.9 samples/s +Step 12420/44000 | Loss: 3.1147 | LR: 4.15e-04 | 1254.2ms/step | 15.9 samples/s +Step 12430/44000 | Loss: 3.0956 | LR: 4.15e-04 | 1254.0ms/step | 15.9 samples/s +Step 12440/44000 | Loss: 3.1103 | LR: 4.15e-04 | 1254.4ms/step | 15.9 samples/s +Step 12450/44000 | Loss: 3.1057 | LR: 4.15e-04 | 1254.6ms/step | 15.9 samples/s +Step 12460/44000 | Loss: 3.1127 | LR: 4.15e-04 | 1253.5ms/step | 16.0 samples/s +Step 12470/44000 | Loss: 3.1048 | LR: 4.15e-04 | 1253.8ms/step | 16.0 samples/s +Step 12480/44000 | Loss: 3.0982 | LR: 4.15e-04 | 1254.2ms/step | 15.9 samples/s +Step 12490/44000 | Loss: 3.0794 | LR: 4.14e-04 | 1255.4ms/step | 15.9 samples/s +Step 12500/44000 | Loss: 3.0734 | LR: 4.14e-04 | 1254.5ms/step | 15.9 samples/s +Validation at step 12500: Loss = 3.1179, Perplexity = 22.60 +Step 12510/44000 | Loss: 3.1290 | LR: 4.14e-04 | 3370.0ms/step | 5.9 samples/s +Step 12520/44000 | Loss: 3.1118 | LR: 4.14e-04 | 1256.9ms/step | 15.9 samples/s +Step 12530/44000 | Loss: 3.1041 | LR: 4.14e-04 | 1253.6ms/step | 16.0 samples/s +Step 12540/44000 | Loss: 3.1052 | LR: 4.14e-04 | 1255.0ms/step | 15.9 samples/s +Step 12550/44000 | Loss: 3.1212 | LR: 4.14e-04 | 1257.0ms/step | 15.9 samples/s +Step 12560/44000 | Loss: 3.0844 | LR: 4.14e-04 | 1258.0ms/step | 15.9 samples/s +Step 12570/44000 | Loss: 3.0956 | LR: 4.13e-04 | 1257.4ms/step | 15.9 samples/s +Step 12580/44000 | Loss: 3.1405 | LR: 4.13e-04 | 1257.1ms/step | 15.9 samples/s +Step 12590/44000 | Loss: 3.0974 | LR: 4.13e-04 | 1255.2ms/step | 15.9 samples/s +Step 12600/44000 | Loss: 3.1035 | LR: 4.13e-04 | 1254.0ms/step | 15.9 samples/s +Step 12610/44000 | Loss: 3.1181 | LR: 4.13e-04 | 1253.5ms/step | 16.0 samples/s +Step 12620/44000 | Loss: 3.1498 | LR: 4.13e-04 | 1253.6ms/step | 16.0 samples/s +Step 12630/44000 | Loss: 3.1115 | LR: 4.13e-04 | 1252.5ms/step | 16.0 samples/s +Step 12640/44000 | Loss: 3.1023 | LR: 4.12e-04 | 1254.4ms/step | 15.9 samples/s +Step 12650/44000 | Loss: 3.1116 | LR: 4.12e-04 | 1254.8ms/step | 15.9 samples/s +Step 12660/44000 | Loss: 3.1001 | LR: 4.12e-04 | 1255.0ms/step | 15.9 samples/s +Step 12670/44000 | Loss: 3.1142 | LR: 4.12e-04 | 1255.7ms/step | 15.9 samples/s +Step 12680/44000 | Loss: 3.1167 | LR: 4.12e-04 | 1255.7ms/step | 15.9 samples/s +Step 12690/44000 | Loss: 3.1069 | LR: 4.12e-04 | 1256.9ms/step | 15.9 samples/s +Step 12700/44000 | Loss: 3.1008 | LR: 4.12e-04 | 1254.6ms/step | 15.9 samples/s +Step 12710/44000 | Loss: 3.0687 | LR: 4.11e-04 | 1255.3ms/step | 15.9 samples/s +Step 12720/44000 | Loss: 3.0651 | LR: 4.11e-04 | 1255.3ms/step | 15.9 samples/s +Step 12730/44000 | Loss: 3.0933 | LR: 4.11e-04 | 1254.5ms/step | 15.9 samples/s +Step 12740/44000 | Loss: 3.1102 | LR: 4.11e-04 | 1254.8ms/step | 15.9 samples/s +Step 12750/44000 | Loss: 3.0734 | LR: 4.11e-04 | 1256.5ms/step | 15.9 samples/s +Validation at step 12750: Loss = 3.1131, Perplexity = 22.49 +Step 12760/44000 | Loss: 3.0999 | LR: 4.11e-04 | 3368.9ms/step | 5.9 samples/s +Step 12770/44000 | Loss: 3.1084 | LR: 4.11e-04 | 1258.3ms/step | 15.9 samples/s +Step 12780/44000 | Loss: 3.0894 | LR: 4.10e-04 | 1259.3ms/step | 15.9 samples/s +Step 12790/44000 | Loss: 3.0944 | LR: 4.10e-04 | 1259.8ms/step | 15.9 samples/s +Step 12800/44000 | Loss: 3.1129 | LR: 4.10e-04 | 1258.2ms/step | 15.9 samples/s +Step 12810/44000 | Loss: 3.1140 | LR: 4.10e-04 | 1255.8ms/step | 15.9 samples/s +Step 12820/44000 | Loss: 3.0900 | LR: 4.10e-04 | 1254.8ms/step | 15.9 samples/s +Step 12830/44000 | Loss: 3.1008 | LR: 4.10e-04 | 1253.5ms/step | 16.0 samples/s +Step 12840/44000 | Loss: 3.0815 | LR: 4.10e-04 | 1254.6ms/step | 15.9 samples/s +Step 12850/44000 | Loss: 3.1013 | LR: 4.09e-04 | 1255.1ms/step | 15.9 samples/s +Step 12860/44000 | Loss: 3.0777 | LR: 4.09e-04 | 1255.6ms/step | 15.9 samples/s +Step 12870/44000 | Loss: 3.1067 | LR: 4.09e-04 | 1254.7ms/step | 15.9 samples/s +Step 12880/44000 | Loss: 3.0672 | LR: 4.09e-04 | 1255.7ms/step | 15.9 samples/s +Step 12890/44000 | Loss: 3.1217 | LR: 4.09e-04 | 1255.5ms/step | 15.9 samples/s +Step 12900/44000 | Loss: 3.0832 | LR: 4.09e-04 | 1257.9ms/step | 15.9 samples/s +Step 12910/44000 | Loss: 3.0996 | LR: 4.09e-04 | 1257.5ms/step | 15.9 samples/s +Step 12920/44000 | Loss: 3.1002 | LR: 4.09e-04 | 1255.8ms/step | 15.9 samples/s +Step 12930/44000 | Loss: 3.1008 | LR: 4.08e-04 | 1255.0ms/step | 15.9 samples/s +Step 12940/44000 | Loss: 3.1158 | LR: 4.08e-04 | 1255.7ms/step | 15.9 samples/s +Step 12950/44000 | Loss: 3.1004 | LR: 4.08e-04 | 1254.5ms/step | 15.9 samples/s +Step 12960/44000 | Loss: 3.1068 | LR: 4.08e-04 | 1254.0ms/step | 15.9 samples/s +Step 12970/44000 | Loss: 3.1147 | LR: 4.08e-04 | 1254.8ms/step | 15.9 samples/s +Step 12980/44000 | Loss: 3.1117 | LR: 4.08e-04 | 1254.3ms/step | 15.9 samples/s +Step 12990/44000 | Loss: 3.1031 | LR: 4.08e-04 | 1254.1ms/step | 15.9 samples/s +Step 13000/44000 | Loss: 3.0893 | LR: 4.07e-04 | 1256.0ms/step | 15.9 samples/s +Validation at step 13000: Loss = 3.1094, Perplexity = 22.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-13000 +Step 13010/44000 | Loss: 3.0968 | LR: 4.07e-04 | 3673.1ms/step | 5.4 samples/s +Step 13020/44000 | Loss: 3.0972 | LR: 4.07e-04 | 1258.3ms/step | 15.9 samples/s +Step 13030/44000 | Loss: 3.1023 | LR: 4.07e-04 | 1261.9ms/step | 15.8 samples/s +Step 13040/44000 | Loss: 3.0783 | LR: 4.07e-04 | 1263.8ms/step | 15.8 samples/s +Step 13050/44000 | Loss: 3.0635 | LR: 4.07e-04 | 1261.1ms/step | 15.9 samples/s +Step 13060/44000 | Loss: 3.1088 | LR: 4.07e-04 | 1259.8ms/step | 15.9 samples/s +Step 13070/44000 | Loss: 3.1012 | LR: 4.06e-04 | 1261.4ms/step | 15.9 samples/s +Step 13080/44000 | Loss: 3.1186 | LR: 4.06e-04 | 1260.6ms/step | 15.9 samples/s +Step 13090/44000 | Loss: 3.0862 | LR: 4.06e-04 | 1259.7ms/step | 15.9 samples/s +Step 13100/44000 | Loss: 3.0894 | LR: 4.06e-04 | 1260.0ms/step | 15.9 samples/s +Step 13110/44000 | Loss: 3.1024 | LR: 4.06e-04 | 1259.9ms/step | 15.9 samples/s +Step 13120/44000 | Loss: 3.1114 | LR: 4.06e-04 | 1260.0ms/step | 15.9 samples/s +Step 13130/44000 | Loss: 3.1117 | LR: 4.06e-04 | 1259.6ms/step | 15.9 samples/s +Step 13140/44000 | Loss: 3.1117 | LR: 4.05e-04 | 1260.3ms/step | 15.9 samples/s +Step 13150/44000 | Loss: 3.0957 | LR: 4.05e-04 | 1260.0ms/step | 15.9 samples/s +Step 13160/44000 | Loss: 3.0882 | LR: 4.05e-04 | 1259.7ms/step | 15.9 samples/s +Step 13170/44000 | Loss: 3.0923 | LR: 4.05e-04 | 1260.0ms/step | 15.9 samples/s +Step 13180/44000 | Loss: 3.1140 | LR: 4.05e-04 | 1260.2ms/step | 15.9 samples/s +Step 13190/44000 | Loss: 3.0897 | LR: 4.05e-04 | 1259.3ms/step | 15.9 samples/s +Step 13200/44000 | Loss: 3.1217 | LR: 4.05e-04 | 1260.7ms/step | 15.9 samples/s +Step 13210/44000 | Loss: 3.0769 | LR: 4.04e-04 | 1260.2ms/step | 15.9 samples/s +Step 13220/44000 | Loss: 3.0794 | LR: 4.04e-04 | 1261.0ms/step | 15.9 samples/s +Step 13230/44000 | Loss: 3.1089 | LR: 4.04e-04 | 1260.3ms/step | 15.9 samples/s +Step 13240/44000 | Loss: 3.0889 | LR: 4.04e-04 | 1260.2ms/step | 15.9 samples/s +Step 13250/44000 | Loss: 3.1070 | LR: 4.04e-04 | 1260.3ms/step | 15.9 samples/s +Validation at step 13250: Loss = 3.1056, Perplexity = 22.32 +Step 13260/44000 | Loss: 3.0989 | LR: 4.04e-04 | 3382.3ms/step | 5.9 samples/s +Step 13270/44000 | Loss: 3.0886 | LR: 4.04e-04 | 1261.7ms/step | 15.9 samples/s +Step 13280/44000 | Loss: 3.1143 | LR: 4.03e-04 | 1262.9ms/step | 15.8 samples/s +Step 13290/44000 | Loss: 3.1005 | LR: 4.03e-04 | 1264.3ms/step | 15.8 samples/s +Step 13300/44000 | Loss: 3.0609 | LR: 4.03e-04 | 1260.6ms/step | 15.9 samples/s +Step 13310/44000 | Loss: 3.0936 | LR: 4.03e-04 | 1259.7ms/step | 15.9 samples/s +Step 13320/44000 | Loss: 3.0743 | LR: 4.03e-04 | 1259.0ms/step | 15.9 samples/s +Step 13330/44000 | Loss: 3.0832 | LR: 4.03e-04 | 1258.8ms/step | 15.9 samples/s +Step 13340/44000 | Loss: 3.0912 | LR: 4.03e-04 | 1260.1ms/step | 15.9 samples/s +Step 13350/44000 | Loss: 3.0789 | LR: 4.02e-04 | 1260.6ms/step | 15.9 samples/s +Step 13360/44000 | Loss: 3.0919 | LR: 4.02e-04 | 1260.4ms/step | 15.9 samples/s +Step 13370/44000 | Loss: 3.1219 | LR: 4.02e-04 | 1261.5ms/step | 15.9 samples/s +Step 13380/44000 | Loss: 3.1151 | LR: 4.02e-04 | 1260.7ms/step | 15.9 samples/s +Step 13390/44000 | Loss: 3.0838 | LR: 4.02e-04 | 1259.3ms/step | 15.9 samples/s +Step 13400/44000 | Loss: 3.0839 | LR: 4.02e-04 | 1259.8ms/step | 15.9 samples/s +Step 13410/44000 | Loss: 3.0763 | LR: 4.02e-04 | 1259.7ms/step | 15.9 samples/s +Step 13420/44000 | Loss: 3.0462 | LR: 4.01e-04 | 1258.8ms/step | 15.9 samples/s +Step 13430/44000 | Loss: 3.1035 | LR: 4.01e-04 | 1259.5ms/step | 15.9 samples/s +Step 13440/44000 | Loss: 3.0936 | LR: 4.01e-04 | 1259.2ms/step | 15.9 samples/s +Step 13450/44000 | Loss: 3.1035 | LR: 4.01e-04 | 1259.3ms/step | 15.9 samples/s +Step 13460/44000 | Loss: 3.0805 | LR: 4.01e-04 | 1259.7ms/step | 15.9 samples/s +Step 13470/44000 | Loss: 3.0943 | LR: 4.01e-04 | 1259.7ms/step | 15.9 samples/s +Step 13480/44000 | Loss: 3.0684 | LR: 4.01e-04 | 1260.4ms/step | 15.9 samples/s +Step 13490/44000 | Loss: 3.0954 | LR: 4.00e-04 | 1259.4ms/step | 15.9 samples/s +Step 13500/44000 | Loss: 3.0961 | LR: 4.00e-04 | 1260.1ms/step | 15.9 samples/s +Validation at step 13500: Loss = 3.1013, Perplexity = 22.23 +Step 13510/44000 | Loss: 3.0833 | LR: 4.00e-04 | 3382.6ms/step | 5.9 samples/s +Step 13520/44000 | Loss: 3.0797 | LR: 4.00e-04 | 1259.7ms/step | 15.9 samples/s +Step 13530/44000 | Loss: 3.0771 | LR: 4.00e-04 | 1259.1ms/step | 15.9 samples/s +Step 13540/44000 | Loss: 3.0687 | LR: 4.00e-04 | 1260.2ms/step | 15.9 samples/s +Step 13550/44000 | Loss: 3.0863 | LR: 4.00e-04 | 1260.9ms/step | 15.9 samples/s +Step 13560/44000 | Loss: 3.0774 | LR: 3.99e-04 | 1261.3ms/step | 15.9 samples/s +Step 13570/44000 | Loss: 3.1141 | LR: 3.99e-04 | 1260.7ms/step | 15.9 samples/s +Step 13580/44000 | Loss: 3.0930 | LR: 3.99e-04 | 1260.1ms/step | 15.9 samples/s +Step 13590/44000 | Loss: 3.0707 | LR: 3.99e-04 | 1261.8ms/step | 15.9 samples/s +Step 13600/44000 | Loss: 3.0839 | LR: 3.99e-04 | 1263.3ms/step | 15.8 samples/s +Step 13610/44000 | Loss: 3.0737 | LR: 3.99e-04 | 1261.7ms/step | 15.9 samples/s +Step 13620/44000 | Loss: 3.1044 | LR: 3.98e-04 | 1261.2ms/step | 15.9 samples/s +Step 13630/44000 | Loss: 3.0961 | LR: 3.98e-04 | 1260.9ms/step | 15.9 samples/s +Step 13640/44000 | Loss: 3.0712 | LR: 3.98e-04 | 1260.8ms/step | 15.9 samples/s +Step 13650/44000 | Loss: 3.0892 | LR: 3.98e-04 | 1260.4ms/step | 15.9 samples/s +Step 13660/44000 | Loss: 3.0773 | LR: 3.98e-04 | 1260.5ms/step | 15.9 samples/s +Step 13670/44000 | Loss: 3.0630 | LR: 3.98e-04 | 1261.5ms/step | 15.9 samples/s +Step 13680/44000 | Loss: 3.0763 | LR: 3.98e-04 | 1261.4ms/step | 15.9 samples/s +Step 13690/44000 | Loss: 3.0931 | LR: 3.97e-04 | 1262.1ms/step | 15.8 samples/s +Step 13700/44000 | Loss: 3.0799 | LR: 3.97e-04 | 1262.0ms/step | 15.8 samples/s +Step 13710/44000 | Loss: 3.0896 | LR: 3.97e-04 | 1259.9ms/step | 15.9 samples/s +Step 13720/44000 | Loss: 3.0706 | LR: 3.97e-04 | 1258.7ms/step | 15.9 samples/s +Step 13730/44000 | Loss: 3.0890 | LR: 3.97e-04 | 1259.2ms/step | 15.9 samples/s +Step 13740/44000 | Loss: 3.0958 | LR: 3.97e-04 | 1260.4ms/step | 15.9 samples/s +Step 13750/44000 | Loss: 3.1264 | LR: 3.97e-04 | 1261.0ms/step | 15.9 samples/s +Validation at step 13750: Loss = 3.0979, Perplexity = 22.15 +Step 13760/44000 | Loss: 3.1162 | LR: 3.96e-04 | 3383.4ms/step | 5.9 samples/s +Step 13770/44000 | Loss: 3.0884 | LR: 3.96e-04 | 1261.4ms/step | 15.9 samples/s +Step 13780/44000 | Loss: 3.0818 | LR: 3.96e-04 | 1260.8ms/step | 15.9 samples/s +Step 13790/44000 | Loss: 3.1177 | LR: 3.96e-04 | 1261.2ms/step | 15.9 samples/s +Step 13800/44000 | Loss: 3.0749 | LR: 3.96e-04 | 1260.6ms/step | 15.9 samples/s +Step 13810/44000 | Loss: 3.0678 | LR: 3.96e-04 | 1261.7ms/step | 15.9 samples/s +Step 13820/44000 | Loss: 3.0831 | LR: 3.96e-04 | 1260.4ms/step | 15.9 samples/s +Step 13830/44000 | Loss: 3.0669 | LR: 3.95e-04 | 1261.8ms/step | 15.9 samples/s +Step 13840/44000 | Loss: 3.0582 | LR: 3.95e-04 | 1261.4ms/step | 15.9 samples/s +Step 13850/44000 | Loss: 3.0786 | LR: 3.95e-04 | 1259.7ms/step | 15.9 samples/s +Step 13860/44000 | Loss: 3.0853 | LR: 3.95e-04 | 1259.3ms/step | 15.9 samples/s +Step 13870/44000 | Loss: 3.0835 | LR: 3.95e-04 | 1258.9ms/step | 15.9 samples/s +Step 13880/44000 | Loss: 3.1021 | LR: 3.95e-04 | 1260.0ms/step | 15.9 samples/s +Step 13890/44000 | Loss: 3.0690 | LR: 3.95e-04 | 1259.7ms/step | 15.9 samples/s +Step 13900/44000 | Loss: 3.1193 | LR: 3.94e-04 | 1260.3ms/step | 15.9 samples/s +Step 13910/44000 | Loss: 3.0919 | LR: 3.94e-04 | 1259.9ms/step | 15.9 samples/s +Step 13920/44000 | Loss: 3.0716 | LR: 3.94e-04 | 1260.4ms/step | 15.9 samples/s +Step 13930/44000 | Loss: 3.0844 | LR: 3.94e-04 | 1259.9ms/step | 15.9 samples/s +Step 13940/44000 | Loss: 3.0642 | LR: 3.94e-04 | 1262.1ms/step | 15.8 samples/s +Step 13950/44000 | Loss: 3.0657 | LR: 3.94e-04 | 1261.1ms/step | 15.9 samples/s +Step 13960/44000 | Loss: 3.0855 | LR: 3.93e-04 | 1261.9ms/step | 15.8 samples/s +Step 13970/44000 | Loss: 3.0710 | LR: 3.93e-04 | 1264.8ms/step | 15.8 samples/s +Step 13980/44000 | Loss: 3.0700 | LR: 3.93e-04 | 1260.0ms/step | 15.9 samples/s +Step 13990/44000 | Loss: 3.0709 | LR: 3.93e-04 | 1261.9ms/step | 15.8 samples/s +Step 14000/44000 | Loss: 3.0897 | LR: 3.93e-04 | 1263.3ms/step | 15.8 samples/s +Validation at step 14000: Loss = 3.0931, Perplexity = 22.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-14000 +Step 14010/44000 | Loss: 3.0924 | LR: 3.93e-04 | 3675.7ms/step | 5.4 samples/s +Step 14020/44000 | Loss: 3.0666 | LR: 3.93e-04 | 1254.1ms/step | 15.9 samples/s +Step 14030/44000 | Loss: 3.0354 | LR: 3.92e-04 | 1254.8ms/step | 15.9 samples/s +Step 14040/44000 | Loss: 3.0524 | LR: 3.92e-04 | 1253.4ms/step | 16.0 samples/s +Step 14050/44000 | Loss: 3.0445 | LR: 3.92e-04 | 1254.4ms/step | 15.9 samples/s +Step 14060/44000 | Loss: 3.1013 | LR: 3.92e-04 | 1257.1ms/step | 15.9 samples/s +Step 14070/44000 | Loss: 3.0926 | LR: 3.92e-04 | 1257.2ms/step | 15.9 samples/s +Step 14080/44000 | Loss: 3.0832 | LR: 3.92e-04 | 1257.2ms/step | 15.9 samples/s +Step 14090/44000 | Loss: 3.0662 | LR: 3.92e-04 | 1257.3ms/step | 15.9 samples/s +Step 14100/44000 | Loss: 3.0948 | LR: 3.91e-04 | 1258.0ms/step | 15.9 samples/s +Step 14110/44000 | Loss: 3.0887 | LR: 3.91e-04 | 1256.9ms/step | 15.9 samples/s +Step 14120/44000 | Loss: 3.0811 | LR: 3.91e-04 | 1254.8ms/step | 15.9 samples/s +Step 14130/44000 | Loss: 3.1033 | LR: 3.91e-04 | 1253.2ms/step | 16.0 samples/s +Step 14140/44000 | Loss: 3.0903 | LR: 3.91e-04 | 1253.3ms/step | 16.0 samples/s +Step 14150/44000 | Loss: 3.0603 | LR: 3.91e-04 | 1252.9ms/step | 16.0 samples/s +Step 14160/44000 | Loss: 3.0724 | LR: 3.90e-04 | 1253.9ms/step | 15.9 samples/s +Step 14170/44000 | Loss: 3.0964 | LR: 3.90e-04 | 1254.0ms/step | 15.9 samples/s +Step 14180/44000 | Loss: 3.0844 | LR: 3.90e-04 | 1255.9ms/step | 15.9 samples/s +Step 14190/44000 | Loss: 3.0787 | LR: 3.90e-04 | 1254.8ms/step | 15.9 samples/s +Step 14200/44000 | Loss: 3.0892 | LR: 3.90e-04 | 1254.0ms/step | 15.9 samples/s +Step 14210/44000 | Loss: 3.0753 | LR: 3.90e-04 | 1253.9ms/step | 16.0 samples/s +Step 14220/44000 | Loss: 3.0748 | LR: 3.90e-04 | 1254.3ms/step | 15.9 samples/s +Step 14230/44000 | Loss: 3.0793 | LR: 3.89e-04 | 1251.6ms/step | 16.0 samples/s +Step 14240/44000 | Loss: 3.0602 | LR: 3.89e-04 | 1251.8ms/step | 16.0 samples/s +Step 14250/44000 | Loss: 3.0735 | LR: 3.89e-04 | 1251.4ms/step | 16.0 samples/s +Validation at step 14250: Loss = 3.0898, Perplexity = 21.97 +Step 14260/44000 | Loss: 3.1055 | LR: 3.89e-04 | 3359.7ms/step | 6.0 samples/s +Step 14270/44000 | Loss: 3.0975 | LR: 3.89e-04 | 1253.9ms/step | 16.0 samples/s +Step 14280/44000 | Loss: 3.0957 | LR: 3.89e-04 | 1254.3ms/step | 15.9 samples/s +Step 14290/44000 | Loss: 3.0824 | LR: 3.89e-04 | 1254.2ms/step | 15.9 samples/s +Step 14300/44000 | Loss: 3.0809 | LR: 3.88e-04 | 1253.8ms/step | 16.0 samples/s +Step 14310/44000 | Loss: 3.0802 | LR: 3.88e-04 | 1252.4ms/step | 16.0 samples/s +Step 14320/44000 | Loss: 3.0781 | LR: 3.88e-04 | 1254.5ms/step | 15.9 samples/s +Step 14330/44000 | Loss: 3.0926 | LR: 3.88e-04 | 1254.5ms/step | 15.9 samples/s +Step 14340/44000 | Loss: 3.0892 | LR: 3.88e-04 | 1253.3ms/step | 16.0 samples/s +Step 14350/44000 | Loss: 3.0732 | LR: 3.88e-04 | 1252.6ms/step | 16.0 samples/s +Step 14360/44000 | Loss: 3.0891 | LR: 3.87e-04 | 1253.8ms/step | 16.0 samples/s +Step 14370/44000 | Loss: 3.0872 | LR: 3.87e-04 | 1254.6ms/step | 15.9 samples/s +Step 14380/44000 | Loss: 3.0681 | LR: 3.87e-04 | 1254.4ms/step | 15.9 samples/s +Step 14390/44000 | Loss: 3.0729 | LR: 3.87e-04 | 1253.6ms/step | 16.0 samples/s +Step 14400/44000 | Loss: 3.0747 | LR: 3.87e-04 | 1254.3ms/step | 15.9 samples/s +Step 14410/44000 | Loss: 3.0553 | LR: 3.87e-04 | 1254.4ms/step | 15.9 samples/s +Step 14420/44000 | Loss: 3.0777 | LR: 3.87e-04 | 1254.1ms/step | 15.9 samples/s +Step 14430/44000 | Loss: 3.0453 | LR: 3.86e-04 | 1254.1ms/step | 15.9 samples/s +Step 14440/44000 | Loss: 3.0853 | LR: 3.86e-04 | 1253.5ms/step | 16.0 samples/s +Step 14450/44000 | Loss: 3.0788 | LR: 3.86e-04 | 1253.1ms/step | 16.0 samples/s +Step 14460/44000 | Loss: 3.0889 | LR: 3.86e-04 | 1252.9ms/step | 16.0 samples/s +Step 14470/44000 | Loss: 3.0858 | LR: 3.86e-04 | 1253.0ms/step | 16.0 samples/s +Step 14480/44000 | Loss: 3.0497 | LR: 3.86e-04 | 1253.2ms/step | 16.0 samples/s +Step 14490/44000 | Loss: 3.0642 | LR: 3.85e-04 | 1252.9ms/step | 16.0 samples/s +Step 14500/44000 | Loss: 3.0898 | LR: 3.85e-04 | 1255.6ms/step | 15.9 samples/s +Validation at step 14500: Loss = 3.0855, Perplexity = 21.88 +Step 14510/44000 | Loss: 3.0587 | LR: 3.85e-04 | 3379.3ms/step | 5.9 samples/s +Step 14520/44000 | Loss: 3.0925 | LR: 3.85e-04 | 1255.6ms/step | 15.9 samples/s +Step 14530/44000 | Loss: 3.0707 | LR: 3.85e-04 | 1253.3ms/step | 16.0 samples/s +Step 14540/44000 | Loss: 3.0792 | LR: 3.85e-04 | 1253.7ms/step | 16.0 samples/s +Step 14550/44000 | Loss: 3.0761 | LR: 3.85e-04 | 1254.5ms/step | 15.9 samples/s +Step 14560/44000 | Loss: 3.0870 | LR: 3.84e-04 | 1252.9ms/step | 16.0 samples/s +Step 14570/44000 | Loss: 3.0869 | LR: 3.84e-04 | 1253.4ms/step | 16.0 samples/s +Step 14580/44000 | Loss: 3.0743 | LR: 3.84e-04 | 1252.5ms/step | 16.0 samples/s +Step 14590/44000 | Loss: 3.0845 | LR: 3.84e-04 | 1252.5ms/step | 16.0 samples/s +Step 14600/44000 | Loss: 3.0775 | LR: 3.84e-04 | 1252.7ms/step | 16.0 samples/s +Step 14610/44000 | Loss: 3.0763 | LR: 3.84e-04 | 1253.2ms/step | 16.0 samples/s +Step 14620/44000 | Loss: 3.0893 | LR: 3.83e-04 | 1253.1ms/step | 16.0 samples/s +Step 14630/44000 | Loss: 3.0657 | LR: 3.83e-04 | 1252.3ms/step | 16.0 samples/s +Step 14640/44000 | Loss: 3.0642 | LR: 3.83e-04 | 1254.0ms/step | 15.9 samples/s +Step 14650/44000 | Loss: 3.0748 | LR: 3.83e-04 | 1251.7ms/step | 16.0 samples/s +Step 14660/44000 | Loss: 3.0882 | LR: 3.83e-04 | 1255.1ms/step | 15.9 samples/s +Step 14670/44000 | Loss: 3.0829 | LR: 3.83e-04 | 1254.7ms/step | 15.9 samples/s +Step 14680/44000 | Loss: 3.0487 | LR: 3.83e-04 | 1255.6ms/step | 15.9 samples/s +Step 14690/44000 | Loss: 3.0665 | LR: 3.82e-04 | 1254.6ms/step | 15.9 samples/s +Step 14700/44000 | Loss: 3.0838 | LR: 3.82e-04 | 1254.3ms/step | 15.9 samples/s +Step 14710/44000 | Loss: 3.0746 | LR: 3.82e-04 | 1254.6ms/step | 15.9 samples/s +Step 14720/44000 | Loss: 3.0652 | LR: 3.82e-04 | 1255.0ms/step | 15.9 samples/s +Step 14730/44000 | Loss: 3.0790 | LR: 3.82e-04 | 1254.5ms/step | 15.9 samples/s +Step 14740/44000 | Loss: 3.0684 | LR: 3.82e-04 | 1255.1ms/step | 15.9 samples/s +Step 14750/44000 | Loss: 3.0962 | LR: 3.81e-04 | 1255.0ms/step | 15.9 samples/s +Validation at step 14750: Loss = 3.0825, Perplexity = 21.81 +Step 14760/44000 | Loss: 3.0696 | LR: 3.81e-04 | 3378.2ms/step | 5.9 samples/s +Step 14770/44000 | Loss: 3.0817 | LR: 3.81e-04 | 1256.7ms/step | 15.9 samples/s +Step 14780/44000 | Loss: 3.0916 | LR: 3.81e-04 | 1253.1ms/step | 16.0 samples/s +Step 14790/44000 | Loss: 3.0755 | LR: 3.81e-04 | 1252.3ms/step | 16.0 samples/s +Step 14800/44000 | Loss: 3.0447 | LR: 3.81e-04 | 1253.5ms/step | 16.0 samples/s +Step 14810/44000 | Loss: 3.0633 | LR: 3.81e-04 | 1254.4ms/step | 15.9 samples/s +Step 14820/44000 | Loss: 3.0719 | LR: 3.80e-04 | 1256.0ms/step | 15.9 samples/s +Step 14830/44000 | Loss: 3.0784 | LR: 3.80e-04 | 1258.2ms/step | 15.9 samples/s +Step 14840/44000 | Loss: 3.0658 | LR: 3.80e-04 | 1257.5ms/step | 15.9 samples/s +Step 14850/44000 | Loss: 3.0754 | LR: 3.80e-04 | 1252.7ms/step | 16.0 samples/s +Step 14860/44000 | Loss: 3.0582 | LR: 3.80e-04 | 1254.4ms/step | 15.9 samples/s +Step 14870/44000 | Loss: 3.0606 | LR: 3.80e-04 | 1255.4ms/step | 15.9 samples/s +Step 14880/44000 | Loss: 3.0572 | LR: 3.79e-04 | 1254.9ms/step | 15.9 samples/s +Step 14890/44000 | Loss: 3.0565 | LR: 3.79e-04 | 1254.1ms/step | 15.9 samples/s +Step 14900/44000 | Loss: 3.0651 | LR: 3.79e-04 | 1252.3ms/step | 16.0 samples/s +Step 14910/44000 | Loss: 3.0863 | LR: 3.79e-04 | 1255.8ms/step | 15.9 samples/s +Step 14920/44000 | Loss: 3.0841 | LR: 3.79e-04 | 1256.4ms/step | 15.9 samples/s +Step 14930/44000 | Loss: 3.0887 | LR: 3.79e-04 | 1255.8ms/step | 15.9 samples/s +Step 14940/44000 | Loss: 3.0793 | LR: 3.79e-04 | 1255.9ms/step | 15.9 samples/s +Step 14950/44000 | Loss: 3.1058 | LR: 3.78e-04 | 1257.0ms/step | 15.9 samples/s +Step 14960/44000 | Loss: 3.0641 | LR: 3.78e-04 | 1256.6ms/step | 15.9 samples/s +Step 14970/44000 | Loss: 3.0684 | LR: 3.78e-04 | 1257.0ms/step | 15.9 samples/s +Step 14980/44000 | Loss: 3.0693 | LR: 3.78e-04 | 1257.1ms/step | 15.9 samples/s +Step 14990/44000 | Loss: 3.0554 | LR: 3.78e-04 | 1253.9ms/step | 16.0 samples/s +Step 15000/44000 | Loss: 3.0866 | LR: 3.78e-04 | 1254.5ms/step | 15.9 samples/s +Validation at step 15000: Loss = 3.0795, Perplexity = 21.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-15000 +Step 15010/44000 | Loss: 3.0443 | LR: 3.77e-04 | 3666.7ms/step | 5.5 samples/s +Step 15020/44000 | Loss: 3.0444 | LR: 3.77e-04 | 1250.3ms/step | 16.0 samples/s +Step 15030/44000 | Loss: 3.0535 | LR: 3.77e-04 | 1251.3ms/step | 16.0 samples/s +Step 15040/44000 | Loss: 3.0642 | LR: 3.77e-04 | 1252.0ms/step | 16.0 samples/s +Step 15050/44000 | Loss: 3.0747 | LR: 3.77e-04 | 1253.6ms/step | 16.0 samples/s +Step 15060/44000 | Loss: 3.0734 | LR: 3.77e-04 | 1252.0ms/step | 16.0 samples/s +Step 15070/44000 | Loss: 3.0504 | LR: 3.77e-04 | 1252.8ms/step | 16.0 samples/s +Step 15080/44000 | Loss: 3.0953 | LR: 3.76e-04 | 1252.6ms/step | 16.0 samples/s +Step 15090/44000 | Loss: 3.0630 | LR: 3.76e-04 | 1252.2ms/step | 16.0 samples/s +Step 15100/44000 | Loss: 3.0559 | LR: 3.76e-04 | 1253.3ms/step | 16.0 samples/s +Step 15110/44000 | Loss: 3.0678 | LR: 3.76e-04 | 1253.2ms/step | 16.0 samples/s +Step 15120/44000 | Loss: 3.0618 | LR: 3.76e-04 | 1253.8ms/step | 16.0 samples/s +Step 15130/44000 | Loss: 3.0619 | LR: 3.76e-04 | 1253.1ms/step | 16.0 samples/s +Step 15140/44000 | Loss: 3.0709 | LR: 3.75e-04 | 1252.9ms/step | 16.0 samples/s +Step 15150/44000 | Loss: 3.0787 | LR: 3.75e-04 | 1254.3ms/step | 15.9 samples/s +Step 15160/44000 | Loss: 3.0834 | LR: 3.75e-04 | 1253.8ms/step | 16.0 samples/s diff --git a/epfl_final/logs/l8k365m_fixed_s2_resume.log b/epfl_final/logs/l8k365m_fixed_s2_resume.log new file mode 100644 index 0000000000000000000000000000000000000000..6ad717e8362bbf04d4592cd64e6a1f59de206d9f --- /dev/null +++ b/epfl_final/logs/l8k365m_fixed_s2_resume.log @@ -0,0 +1,3281 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-15000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m fixed s2 8k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "fixed", + "seed2" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_fixed_s2", + "sparse_mask_seed": 2, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-15000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-15000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 15000 (epoch 0, micro_step 150000) +Resuming training from step 15000 +Resuming wandb run lgjoqt2r +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run lgjoqt2r +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_170941-lgjoqt2r +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m fixed s2 8k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/lgjoqt2r +Starting training... +Validation at step 15000: Loss = 3.0795, Perplexity = 21.75 +wandb: WARNING Tried to log to step 15000 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 15001: sparse_stochastic (configured sparse attention) +Step 15010/44000 | Loss: 3.0809 | LR: 3.77e-04 | 36938.1ms/step | 0.5 samples/s +wandb: WARNING Tried to log to step 15010 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15020/44000 | Loss: 3.0513 | LR: 3.77e-04 | 1280.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15020 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15030/44000 | Loss: 3.0678 | LR: 3.77e-04 | 1282.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15030 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15040/44000 | Loss: 3.0780 | LR: 3.77e-04 | 1284.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15040 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15050/44000 | Loss: 3.0537 | LR: 3.77e-04 | 1285.5ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15050 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15060/44000 | Loss: 3.0878 | LR: 3.77e-04 | 1282.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15060 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15070/44000 | Loss: 3.0499 | LR: 3.77e-04 | 1281.2ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15070 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15080/44000 | Loss: 3.0906 | LR: 3.76e-04 | 1284.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15080 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15090/44000 | Loss: 3.0651 | LR: 3.76e-04 | 1286.4ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 15090 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15100/44000 | Loss: 3.0404 | LR: 3.76e-04 | 1282.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15100 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15110/44000 | Loss: 3.0814 | LR: 3.76e-04 | 1284.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15110 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15120/44000 | Loss: 3.0863 | LR: 3.76e-04 | 1286.2ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 15120 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15130/44000 | Loss: 3.0576 | LR: 3.76e-04 | 1285.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15130 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15140/44000 | Loss: 3.0693 | LR: 3.75e-04 | 1285.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 15140 that is less than the current step 15141. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 15150/44000 | Loss: 3.0662 | LR: 3.75e-04 | 1286.1ms/step | 15.6 samples/s +Step 15160/44000 | Loss: 3.0902 | LR: 3.75e-04 | 1283.8ms/step | 15.6 samples/s +Step 15170/44000 | Loss: 3.0601 | LR: 3.75e-04 | 1283.1ms/step | 15.6 samples/s +Step 15180/44000 | Loss: 3.0941 | LR: 3.75e-04 | 1283.7ms/step | 15.6 samples/s +Step 15190/44000 | Loss: 3.0586 | LR: 3.75e-04 | 1285.3ms/step | 15.6 samples/s +Step 15200/44000 | Loss: 3.0952 | LR: 3.74e-04 | 1287.6ms/step | 15.5 samples/s +Step 15210/44000 | Loss: 3.0594 | LR: 3.74e-04 | 1286.0ms/step | 15.6 samples/s +Step 15220/44000 | Loss: 3.0519 | LR: 3.74e-04 | 1286.9ms/step | 15.5 samples/s +Step 15230/44000 | Loss: 3.0361 | LR: 3.74e-04 | 1287.1ms/step | 15.5 samples/s +Step 15240/44000 | Loss: 3.0145 | LR: 3.74e-04 | 1286.1ms/step | 15.6 samples/s +Step 15250/44000 | Loss: 3.0721 | LR: 3.74e-04 | 1284.2ms/step | 15.6 samples/s +Validation at step 15250: Loss = 3.0759, Perplexity = 21.67 +Step 15260/44000 | Loss: 3.0496 | LR: 3.74e-04 | 3447.7ms/step | 5.8 samples/s +Step 15270/44000 | Loss: 3.0471 | LR: 3.73e-04 | 1288.3ms/step | 15.5 samples/s +Step 15280/44000 | Loss: 3.0485 | LR: 3.73e-04 | 1288.8ms/step | 15.5 samples/s +Step 15290/44000 | Loss: 3.0566 | LR: 3.73e-04 | 1289.4ms/step | 15.5 samples/s +Step 15300/44000 | Loss: 3.0633 | LR: 3.73e-04 | 1289.6ms/step | 15.5 samples/s +Step 15310/44000 | Loss: 3.0764 | LR: 3.73e-04 | 1287.3ms/step | 15.5 samples/s +Step 15320/44000 | Loss: 3.0640 | LR: 3.73e-04 | 1288.3ms/step | 15.5 samples/s +Step 15330/44000 | Loss: 3.0484 | LR: 3.72e-04 | 1288.2ms/step | 15.5 samples/s +Step 15340/44000 | Loss: 3.0562 | LR: 3.72e-04 | 1290.0ms/step | 15.5 samples/s +Step 15350/44000 | Loss: 3.0451 | LR: 3.72e-04 | 1290.0ms/step | 15.5 samples/s +Step 15360/44000 | Loss: 3.0503 | LR: 3.72e-04 | 1287.1ms/step | 15.5 samples/s +Step 15370/44000 | Loss: 3.0528 | LR: 3.72e-04 | 1286.2ms/step | 15.6 samples/s +Step 15380/44000 | Loss: 3.0570 | LR: 3.72e-04 | 1288.9ms/step | 15.5 samples/s +Step 15390/44000 | Loss: 3.0238 | LR: 3.71e-04 | 1289.4ms/step | 15.5 samples/s +Step 15400/44000 | Loss: 3.0357 | LR: 3.71e-04 | 1290.0ms/step | 15.5 samples/s +Step 15410/44000 | Loss: 3.0298 | LR: 3.71e-04 | 1290.1ms/step | 15.5 samples/s +Step 15420/44000 | Loss: 3.0593 | LR: 3.71e-04 | 1289.0ms/step | 15.5 samples/s +Step 15430/44000 | Loss: 3.0459 | LR: 3.71e-04 | 1286.5ms/step | 15.5 samples/s +Step 15440/44000 | Loss: 3.0840 | LR: 3.71e-04 | 1287.4ms/step | 15.5 samples/s +Step 15450/44000 | Loss: 3.0627 | LR: 3.71e-04 | 1291.8ms/step | 15.5 samples/s +Step 15460/44000 | Loss: 3.0603 | LR: 3.70e-04 | 1291.5ms/step | 15.5 samples/s +Step 15470/44000 | Loss: 3.0562 | LR: 3.70e-04 | 1290.2ms/step | 15.5 samples/s +Step 15480/44000 | Loss: 3.0537 | LR: 3.70e-04 | 1287.8ms/step | 15.5 samples/s +Step 15490/44000 | Loss: 3.0470 | LR: 3.70e-04 | 1287.9ms/step | 15.5 samples/s +Step 15500/44000 | Loss: 3.0896 | LR: 3.70e-04 | 1287.4ms/step | 15.5 samples/s +Validation at step 15500: Loss = 3.0724, Perplexity = 21.59 +Step 15510/44000 | Loss: 3.0566 | LR: 3.70e-04 | 3448.0ms/step | 5.8 samples/s +Step 15520/44000 | Loss: 3.0792 | LR: 3.69e-04 | 1289.3ms/step | 15.5 samples/s +Step 15530/44000 | Loss: 3.0430 | LR: 3.69e-04 | 1289.1ms/step | 15.5 samples/s +Step 15540/44000 | Loss: 3.0305 | LR: 3.69e-04 | 1291.3ms/step | 15.5 samples/s +Step 15550/44000 | Loss: 3.0329 | LR: 3.69e-04 | 1289.2ms/step | 15.5 samples/s +Step 15560/44000 | Loss: 3.0472 | LR: 3.69e-04 | 1288.6ms/step | 15.5 samples/s +Step 15570/44000 | Loss: 3.0564 | LR: 3.69e-04 | 1289.9ms/step | 15.5 samples/s +Step 15580/44000 | Loss: 3.0622 | LR: 3.68e-04 | 1289.2ms/step | 15.5 samples/s +Step 15590/44000 | Loss: 3.0704 | LR: 3.68e-04 | 1289.8ms/step | 15.5 samples/s +Step 15600/44000 | Loss: 3.0530 | LR: 3.68e-04 | 1288.4ms/step | 15.5 samples/s +Step 15610/44000 | Loss: 3.0659 | LR: 3.68e-04 | 1290.4ms/step | 15.5 samples/s +Step 15620/44000 | Loss: 3.0747 | LR: 3.68e-04 | 1290.2ms/step | 15.5 samples/s +Step 15630/44000 | Loss: 3.0380 | LR: 3.68e-04 | 1288.8ms/step | 15.5 samples/s +Step 15640/44000 | Loss: 3.0578 | LR: 3.67e-04 | 1289.1ms/step | 15.5 samples/s +Step 15650/44000 | Loss: 3.0568 | LR: 3.67e-04 | 1288.0ms/step | 15.5 samples/s +Step 15660/44000 | Loss: 3.0317 | LR: 3.67e-04 | 1288.9ms/step | 15.5 samples/s +Step 15670/44000 | Loss: 3.0452 | LR: 3.67e-04 | 1288.6ms/step | 15.5 samples/s +Step 15680/44000 | Loss: 3.1042 | LR: 3.67e-04 | 1288.4ms/step | 15.5 samples/s +Step 15690/44000 | Loss: 3.0751 | LR: 3.67e-04 | 1290.7ms/step | 15.5 samples/s +Step 15700/44000 | Loss: 3.0887 | LR: 3.67e-04 | 1289.5ms/step | 15.5 samples/s +Step 15710/44000 | Loss: 3.0978 | LR: 3.66e-04 | 1290.8ms/step | 15.5 samples/s +Step 15720/44000 | Loss: 3.0385 | LR: 3.66e-04 | 1289.0ms/step | 15.5 samples/s +Step 15730/44000 | Loss: 3.0519 | LR: 3.66e-04 | 1289.6ms/step | 15.5 samples/s +Step 15740/44000 | Loss: 3.0456 | LR: 3.66e-04 | 1288.9ms/step | 15.5 samples/s +Step 15750/44000 | Loss: 3.0420 | LR: 3.66e-04 | 1288.3ms/step | 15.5 samples/s +Validation at step 15750: Loss = 3.0689, Perplexity = 21.52 +Step 15760/44000 | Loss: 3.0441 | LR: 3.66e-04 | 3446.6ms/step | 5.8 samples/s +Step 15770/44000 | Loss: 3.0343 | LR: 3.65e-04 | 1288.0ms/step | 15.5 samples/s +Step 15780/44000 | Loss: 3.0504 | LR: 3.65e-04 | 1289.8ms/step | 15.5 samples/s +Step 15790/44000 | Loss: 3.0571 | LR: 3.65e-04 | 1289.0ms/step | 15.5 samples/s +Step 15800/44000 | Loss: 3.0486 | LR: 3.65e-04 | 1288.7ms/step | 15.5 samples/s +Step 15810/44000 | Loss: 3.0345 | LR: 3.65e-04 | 1290.0ms/step | 15.5 samples/s +Step 15820/44000 | Loss: 3.0403 | LR: 3.65e-04 | 1289.4ms/step | 15.5 samples/s +Step 15830/44000 | Loss: 3.0631 | LR: 3.64e-04 | 1290.4ms/step | 15.5 samples/s +Step 15840/44000 | Loss: 3.0504 | LR: 3.64e-04 | 1289.7ms/step | 15.5 samples/s +Step 15850/44000 | Loss: 3.0377 | LR: 3.64e-04 | 1291.7ms/step | 15.5 samples/s +Step 15860/44000 | Loss: 3.0412 | LR: 3.64e-04 | 1289.9ms/step | 15.5 samples/s +Step 15870/44000 | Loss: 3.0515 | LR: 3.64e-04 | 1291.3ms/step | 15.5 samples/s +Step 15880/44000 | Loss: 3.0588 | LR: 3.64e-04 | 1288.6ms/step | 15.5 samples/s +Step 15890/44000 | Loss: 3.0241 | LR: 3.63e-04 | 1289.3ms/step | 15.5 samples/s +Step 15900/44000 | Loss: 3.0677 | LR: 3.63e-04 | 1288.6ms/step | 15.5 samples/s +Step 15910/44000 | Loss: 3.0632 | LR: 3.63e-04 | 1288.6ms/step | 15.5 samples/s +Step 15920/44000 | Loss: 3.0596 | LR: 3.63e-04 | 1287.6ms/step | 15.5 samples/s +Step 15930/44000 | Loss: 3.0166 | LR: 3.63e-04 | 1288.9ms/step | 15.5 samples/s +Step 15940/44000 | Loss: 3.0404 | LR: 3.63e-04 | 1289.3ms/step | 15.5 samples/s +Step 15950/44000 | Loss: 3.0209 | LR: 3.62e-04 | 1289.5ms/step | 15.5 samples/s +Step 15960/44000 | Loss: 3.0299 | LR: 3.62e-04 | 1289.3ms/step | 15.5 samples/s +Step 15970/44000 | Loss: 3.0438 | LR: 3.62e-04 | 1288.4ms/step | 15.5 samples/s +Step 15980/44000 | Loss: 3.0518 | LR: 3.62e-04 | 1288.2ms/step | 15.5 samples/s +Step 15990/44000 | Loss: 3.0565 | LR: 3.62e-04 | 1290.8ms/step | 15.5 samples/s +Step 16000/44000 | Loss: 3.0621 | LR: 3.62e-04 | 1287.3ms/step | 15.5 samples/s +Validation at step 16000: Loss = 3.0667, Perplexity = 21.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-16000 +Step 16010/44000 | Loss: 3.0142 | LR: 3.62e-04 | 3745.8ms/step | 5.3 samples/s +Step 16020/44000 | Loss: 3.0337 | LR: 3.61e-04 | 1286.9ms/step | 15.5 samples/s +Step 16030/44000 | Loss: 3.0489 | LR: 3.61e-04 | 1289.1ms/step | 15.5 samples/s +Step 16040/44000 | Loss: 3.0345 | LR: 3.61e-04 | 1289.9ms/step | 15.5 samples/s +Step 16050/44000 | Loss: 3.0685 | LR: 3.61e-04 | 1290.5ms/step | 15.5 samples/s +Step 16060/44000 | Loss: 3.0406 | LR: 3.61e-04 | 1290.9ms/step | 15.5 samples/s +Step 16070/44000 | Loss: 3.0603 | LR: 3.61e-04 | 1291.7ms/step | 15.5 samples/s +Step 16080/44000 | Loss: 3.0503 | LR: 3.60e-04 | 1292.0ms/step | 15.5 samples/s +Step 16090/44000 | Loss: 3.0165 | LR: 3.60e-04 | 1289.2ms/step | 15.5 samples/s +Step 16100/44000 | Loss: 3.0379 | LR: 3.60e-04 | 1291.3ms/step | 15.5 samples/s +Step 16110/44000 | Loss: 3.0535 | LR: 3.60e-04 | 1290.7ms/step | 15.5 samples/s +Step 16120/44000 | Loss: 3.0527 | LR: 3.60e-04 | 1289.6ms/step | 15.5 samples/s +Step 16130/44000 | Loss: 3.0371 | LR: 3.60e-04 | 1289.1ms/step | 15.5 samples/s +Step 16140/44000 | Loss: 3.0443 | LR: 3.59e-04 | 1289.2ms/step | 15.5 samples/s +Step 16150/44000 | Loss: 3.0231 | LR: 3.59e-04 | 1287.0ms/step | 15.5 samples/s +Step 16160/44000 | Loss: 3.0405 | LR: 3.59e-04 | 1291.2ms/step | 15.5 samples/s +Step 16170/44000 | Loss: 3.0631 | LR: 3.59e-04 | 1291.7ms/step | 15.5 samples/s +Step 16180/44000 | Loss: 3.0679 | LR: 3.59e-04 | 1292.9ms/step | 15.5 samples/s +Step 16190/44000 | Loss: 3.0601 | LR: 3.59e-04 | 1291.1ms/step | 15.5 samples/s +Step 16200/44000 | Loss: 3.0311 | LR: 3.58e-04 | 1292.0ms/step | 15.5 samples/s +Step 16210/44000 | Loss: 3.0307 | LR: 3.58e-04 | 1291.0ms/step | 15.5 samples/s +Step 16220/44000 | Loss: 3.0754 | LR: 3.58e-04 | 1290.3ms/step | 15.5 samples/s +Step 16230/44000 | Loss: 3.0128 | LR: 3.58e-04 | 1290.5ms/step | 15.5 samples/s +Step 16240/44000 | Loss: 3.0267 | LR: 3.58e-04 | 1290.4ms/step | 15.5 samples/s +Step 16250/44000 | Loss: 3.0704 | LR: 3.58e-04 | 1290.3ms/step | 15.5 samples/s +Validation at step 16250: Loss = 3.0625, Perplexity = 21.38 +Step 16260/44000 | Loss: 3.0290 | LR: 3.57e-04 | 3448.0ms/step | 5.8 samples/s +Step 16270/44000 | Loss: 3.0866 | LR: 3.57e-04 | 1288.7ms/step | 15.5 samples/s +Step 16280/44000 | Loss: 3.0442 | LR: 3.57e-04 | 1293.8ms/step | 15.5 samples/s +Step 16290/44000 | Loss: 3.0263 | LR: 3.57e-04 | 1293.3ms/step | 15.5 samples/s +Step 16300/44000 | Loss: 3.0727 | LR: 3.57e-04 | 1289.1ms/step | 15.5 samples/s +Step 16310/44000 | Loss: 3.0583 | LR: 3.57e-04 | 1289.6ms/step | 15.5 samples/s +Step 16320/44000 | Loss: 3.0155 | LR: 3.56e-04 | 1291.7ms/step | 15.5 samples/s +Step 16330/44000 | Loss: 3.0312 | LR: 3.56e-04 | 1288.2ms/step | 15.5 samples/s +Step 16340/44000 | Loss: 3.0561 | LR: 3.56e-04 | 1290.4ms/step | 15.5 samples/s +Step 16350/44000 | Loss: 3.0560 | LR: 3.56e-04 | 1289.3ms/step | 15.5 samples/s +Step 16360/44000 | Loss: 3.0605 | LR: 3.56e-04 | 1290.5ms/step | 15.5 samples/s +Step 16370/44000 | Loss: 3.0332 | LR: 3.56e-04 | 1289.1ms/step | 15.5 samples/s +Step 16380/44000 | Loss: 3.0467 | LR: 3.55e-04 | 1291.1ms/step | 15.5 samples/s +Step 16390/44000 | Loss: 3.0809 | LR: 3.55e-04 | 1290.9ms/step | 15.5 samples/s +Step 16400/44000 | Loss: 3.0274 | LR: 3.55e-04 | 1291.3ms/step | 15.5 samples/s +Step 16410/44000 | Loss: 3.0390 | LR: 3.55e-04 | 1290.3ms/step | 15.5 samples/s +Step 16420/44000 | Loss: 3.0338 | LR: 3.55e-04 | 1289.6ms/step | 15.5 samples/s +Step 16430/44000 | Loss: 3.0562 | LR: 3.55e-04 | 1290.3ms/step | 15.5 samples/s +Step 16440/44000 | Loss: 3.0411 | LR: 3.54e-04 | 1290.3ms/step | 15.5 samples/s +Step 16450/44000 | Loss: 3.0462 | LR: 3.54e-04 | 1292.4ms/step | 15.5 samples/s +Step 16460/44000 | Loss: 3.0578 | LR: 3.54e-04 | 1289.7ms/step | 15.5 samples/s +Step 16470/44000 | Loss: 3.0218 | LR: 3.54e-04 | 1290.2ms/step | 15.5 samples/s +Step 16480/44000 | Loss: 3.0267 | LR: 3.54e-04 | 1289.2ms/step | 15.5 samples/s +Step 16490/44000 | Loss: 3.0226 | LR: 3.54e-04 | 1291.9ms/step | 15.5 samples/s +Step 16500/44000 | Loss: 3.0377 | LR: 3.53e-04 | 1290.7ms/step | 15.5 samples/s +Validation at step 16500: Loss = 3.0593, Perplexity = 21.31 +Step 16510/44000 | Loss: 3.0585 | LR: 3.53e-04 | 3451.3ms/step | 5.8 samples/s +Step 16520/44000 | Loss: 3.0341 | LR: 3.53e-04 | 1290.9ms/step | 15.5 samples/s +Step 16530/44000 | Loss: 3.0110 | LR: 3.53e-04 | 1290.5ms/step | 15.5 samples/s +Step 16540/44000 | Loss: 3.0470 | LR: 3.53e-04 | 1290.7ms/step | 15.5 samples/s +Step 16550/44000 | Loss: 3.0003 | LR: 3.53e-04 | 1286.3ms/step | 15.5 samples/s +Step 16560/44000 | Loss: 3.0266 | LR: 3.52e-04 | 1290.4ms/step | 15.5 samples/s +Step 16570/44000 | Loss: 3.0601 | LR: 3.52e-04 | 1289.2ms/step | 15.5 samples/s +Step 16580/44000 | Loss: 3.0262 | LR: 3.52e-04 | 1289.1ms/step | 15.5 samples/s +Step 16590/44000 | Loss: 3.0370 | LR: 3.52e-04 | 1290.0ms/step | 15.5 samples/s +Step 16600/44000 | Loss: 3.0234 | LR: 3.52e-04 | 1290.2ms/step | 15.5 samples/s +Step 16610/44000 | Loss: 3.0421 | LR: 3.52e-04 | 1291.6ms/step | 15.5 samples/s +Step 16620/44000 | Loss: 3.0439 | LR: 3.51e-04 | 1291.9ms/step | 15.5 samples/s +Step 16630/44000 | Loss: 3.0219 | LR: 3.51e-04 | 1292.7ms/step | 15.5 samples/s +Step 16640/44000 | Loss: 3.0567 | LR: 3.51e-04 | 1289.4ms/step | 15.5 samples/s +Step 16650/44000 | Loss: 3.0205 | LR: 3.51e-04 | 1290.0ms/step | 15.5 samples/s +Step 16660/44000 | Loss: 3.0233 | LR: 3.51e-04 | 1289.2ms/step | 15.5 samples/s +Step 16670/44000 | Loss: 3.0492 | LR: 3.51e-04 | 1290.8ms/step | 15.5 samples/s +Step 16680/44000 | Loss: 3.0026 | LR: 3.50e-04 | 1290.2ms/step | 15.5 samples/s +Step 16690/44000 | Loss: 3.0310 | LR: 3.50e-04 | 1292.3ms/step | 15.5 samples/s +Step 16700/44000 | Loss: 3.0643 | LR: 3.50e-04 | 1294.4ms/step | 15.5 samples/s +Step 16710/44000 | Loss: 3.0039 | LR: 3.50e-04 | 1288.4ms/step | 15.5 samples/s +Step 16720/44000 | Loss: 3.0479 | LR: 3.50e-04 | 1288.8ms/step | 15.5 samples/s +Step 16730/44000 | Loss: 3.0167 | LR: 3.50e-04 | 1289.0ms/step | 15.5 samples/s +Step 16740/44000 | Loss: 3.0196 | LR: 3.49e-04 | 1292.2ms/step | 15.5 samples/s +Step 16750/44000 | Loss: 3.0159 | LR: 3.49e-04 | 1292.6ms/step | 15.5 samples/s +Validation at step 16750: Loss = 3.0567, Perplexity = 21.26 +Step 16760/44000 | Loss: 3.0598 | LR: 3.49e-04 | 3456.9ms/step | 5.8 samples/s +Step 16770/44000 | Loss: 3.0625 | LR: 3.49e-04 | 1289.1ms/step | 15.5 samples/s +Step 16780/44000 | Loss: 3.0517 | LR: 3.49e-04 | 1288.6ms/step | 15.5 samples/s +Step 16790/44000 | Loss: 3.0697 | LR: 3.49e-04 | 1290.4ms/step | 15.5 samples/s +Step 16800/44000 | Loss: 3.0446 | LR: 3.48e-04 | 1290.8ms/step | 15.5 samples/s +Step 16810/44000 | Loss: 3.0211 | LR: 3.48e-04 | 1289.7ms/step | 15.5 samples/s +Step 16820/44000 | Loss: 3.0668 | LR: 3.48e-04 | 1290.7ms/step | 15.5 samples/s +Step 16830/44000 | Loss: 3.0639 | LR: 3.48e-04 | 1292.0ms/step | 15.5 samples/s +Step 16840/44000 | Loss: 3.0352 | LR: 3.48e-04 | 1290.6ms/step | 15.5 samples/s +Step 16850/44000 | Loss: 3.0216 | LR: 3.48e-04 | 1291.9ms/step | 15.5 samples/s +Step 16860/44000 | Loss: 3.0599 | LR: 3.47e-04 | 1290.8ms/step | 15.5 samples/s +Step 16870/44000 | Loss: 3.0352 | LR: 3.47e-04 | 1289.9ms/step | 15.5 samples/s +Step 16880/44000 | Loss: 3.0553 | LR: 3.47e-04 | 1290.1ms/step | 15.5 samples/s +Step 16890/44000 | Loss: 3.0242 | LR: 3.47e-04 | 1291.0ms/step | 15.5 samples/s +Step 16900/44000 | Loss: 3.0484 | LR: 3.47e-04 | 1290.3ms/step | 15.5 samples/s +Step 16910/44000 | Loss: 3.0363 | LR: 3.47e-04 | 1289.5ms/step | 15.5 samples/s +Step 16920/44000 | Loss: 3.0499 | LR: 3.46e-04 | 1289.6ms/step | 15.5 samples/s +Step 16930/44000 | Loss: 3.0692 | LR: 3.46e-04 | 1288.2ms/step | 15.5 samples/s +Step 16940/44000 | Loss: 3.0319 | LR: 3.46e-04 | 1289.5ms/step | 15.5 samples/s +Step 16950/44000 | Loss: 3.0193 | LR: 3.46e-04 | 1289.4ms/step | 15.5 samples/s +Step 16960/44000 | Loss: 3.0334 | LR: 3.46e-04 | 1290.1ms/step | 15.5 samples/s +Step 16970/44000 | Loss: 3.0610 | LR: 3.46e-04 | 1288.7ms/step | 15.5 samples/s +Step 16980/44000 | Loss: 3.0687 | LR: 3.45e-04 | 1289.0ms/step | 15.5 samples/s +Step 16990/44000 | Loss: 3.0257 | LR: 3.45e-04 | 1290.6ms/step | 15.5 samples/s +Step 17000/44000 | Loss: 3.0401 | LR: 3.45e-04 | 1289.7ms/step | 15.5 samples/s +Validation at step 17000: Loss = 3.0556, Perplexity = 21.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-17000 +Step 17010/44000 | Loss: 3.0552 | LR: 3.45e-04 | 3735.3ms/step | 5.4 samples/s +Step 17020/44000 | Loss: 3.0368 | LR: 3.45e-04 | 1286.4ms/step | 15.5 samples/s +Step 17030/44000 | Loss: 3.0397 | LR: 3.45e-04 | 1289.8ms/step | 15.5 samples/s +Step 17040/44000 | Loss: 3.0733 | LR: 3.44e-04 | 1291.5ms/step | 15.5 samples/s +Step 17050/44000 | Loss: 3.0657 | LR: 3.44e-04 | 1291.0ms/step | 15.5 samples/s +Step 17060/44000 | Loss: 3.0185 | LR: 3.44e-04 | 1289.8ms/step | 15.5 samples/s +Step 17070/44000 | Loss: 3.0020 | LR: 3.44e-04 | 1291.7ms/step | 15.5 samples/s +Step 17080/44000 | Loss: 3.0503 | LR: 3.44e-04 | 1290.0ms/step | 15.5 samples/s +Step 17090/44000 | Loss: 3.0331 | LR: 3.44e-04 | 1290.3ms/step | 15.5 samples/s +Step 17100/44000 | Loss: 3.0387 | LR: 3.43e-04 | 1289.6ms/step | 15.5 samples/s +Step 17110/44000 | Loss: 3.0314 | LR: 3.43e-04 | 1287.6ms/step | 15.5 samples/s +Step 17120/44000 | Loss: 3.0379 | LR: 3.43e-04 | 1287.4ms/step | 15.5 samples/s +Step 17130/44000 | Loss: 3.0255 | LR: 3.43e-04 | 1286.2ms/step | 15.6 samples/s +Step 17140/44000 | Loss: 3.0533 | LR: 3.43e-04 | 1289.1ms/step | 15.5 samples/s +Step 17150/44000 | Loss: 3.0537 | LR: 3.43e-04 | 1292.3ms/step | 15.5 samples/s +Step 17160/44000 | Loss: 3.0173 | LR: 3.42e-04 | 1289.0ms/step | 15.5 samples/s +Step 17170/44000 | Loss: 3.0316 | LR: 3.42e-04 | 1289.1ms/step | 15.5 samples/s +Step 17180/44000 | Loss: 3.0137 | LR: 3.42e-04 | 1289.6ms/step | 15.5 samples/s +Step 17190/44000 | Loss: 3.0566 | LR: 3.42e-04 | 1289.3ms/step | 15.5 samples/s +Step 17200/44000 | Loss: 3.0297 | LR: 3.42e-04 | 1289.2ms/step | 15.5 samples/s +Step 17210/44000 | Loss: 3.0488 | LR: 3.42e-04 | 1290.5ms/step | 15.5 samples/s +Step 17220/44000 | Loss: 3.0377 | LR: 3.41e-04 | 1288.5ms/step | 15.5 samples/s +Step 17230/44000 | Loss: 3.0317 | LR: 3.41e-04 | 1286.9ms/step | 15.5 samples/s +Step 17240/44000 | Loss: 3.0334 | LR: 3.41e-04 | 1289.3ms/step | 15.5 samples/s +Step 17250/44000 | Loss: 3.0435 | LR: 3.41e-04 | 1290.1ms/step | 15.5 samples/s +Validation at step 17250: Loss = 3.0506, Perplexity = 21.13 +Step 17260/44000 | Loss: 3.0379 | LR: 3.41e-04 | 3446.6ms/step | 5.8 samples/s +Step 17270/44000 | Loss: 3.0466 | LR: 3.41e-04 | 1289.4ms/step | 15.5 samples/s +Step 17280/44000 | Loss: 3.0356 | LR: 3.40e-04 | 1291.4ms/step | 15.5 samples/s +Step 17290/44000 | Loss: 3.0346 | LR: 3.40e-04 | 1289.3ms/step | 15.5 samples/s +Step 17300/44000 | Loss: 3.0209 | LR: 3.40e-04 | 1290.7ms/step | 15.5 samples/s +Step 17310/44000 | Loss: 3.0298 | LR: 3.40e-04 | 1291.1ms/step | 15.5 samples/s +Step 17320/44000 | Loss: 3.0321 | LR: 3.40e-04 | 1292.5ms/step | 15.5 samples/s +Step 17330/44000 | Loss: 3.0382 | LR: 3.40e-04 | 1287.7ms/step | 15.5 samples/s +Step 17340/44000 | Loss: 3.0373 | LR: 3.39e-04 | 1291.0ms/step | 15.5 samples/s +Step 17350/44000 | Loss: 3.0339 | LR: 3.39e-04 | 1290.0ms/step | 15.5 samples/s +Step 17360/44000 | Loss: 3.0291 | LR: 3.39e-04 | 1289.1ms/step | 15.5 samples/s +Step 17370/44000 | Loss: 3.0053 | LR: 3.39e-04 | 1288.9ms/step | 15.5 samples/s +Step 17380/44000 | Loss: 3.0540 | LR: 3.39e-04 | 1290.2ms/step | 15.5 samples/s +Step 17390/44000 | Loss: 3.0260 | LR: 3.39e-04 | 1291.5ms/step | 15.5 samples/s +Step 17400/44000 | Loss: 3.0572 | LR: 3.38e-04 | 1290.7ms/step | 15.5 samples/s +Step 17410/44000 | Loss: 3.0360 | LR: 3.38e-04 | 1291.0ms/step | 15.5 samples/s +Step 17420/44000 | Loss: 3.0110 | LR: 3.38e-04 | 1290.7ms/step | 15.5 samples/s +Step 17430/44000 | Loss: 3.0394 | LR: 3.38e-04 | 1288.3ms/step | 15.5 samples/s +Step 17440/44000 | Loss: 3.0325 | LR: 3.38e-04 | 1288.8ms/step | 15.5 samples/s +Step 17450/44000 | Loss: 3.0070 | LR: 3.38e-04 | 1291.4ms/step | 15.5 samples/s +Step 17460/44000 | Loss: 3.0261 | LR: 3.37e-04 | 1290.8ms/step | 15.5 samples/s +Step 17470/44000 | Loss: 3.0277 | LR: 3.37e-04 | 1290.2ms/step | 15.5 samples/s +Step 17480/44000 | Loss: 3.0519 | LR: 3.37e-04 | 1290.8ms/step | 15.5 samples/s +Step 17490/44000 | Loss: 3.0055 | LR: 3.37e-04 | 1292.3ms/step | 15.5 samples/s +Step 17500/44000 | Loss: 3.0581 | LR: 3.37e-04 | 1292.5ms/step | 15.5 samples/s +Validation at step 17500: Loss = 3.0466, Perplexity = 21.04 +Step 17510/44000 | Loss: 3.0321 | LR: 3.37e-04 | 3454.1ms/step | 5.8 samples/s +Step 17520/44000 | Loss: 2.9930 | LR: 3.36e-04 | 1291.7ms/step | 15.5 samples/s +Step 17530/44000 | Loss: 3.0312 | LR: 3.36e-04 | 1291.8ms/step | 15.5 samples/s +Step 17540/44000 | Loss: 3.0565 | LR: 3.36e-04 | 1289.1ms/step | 15.5 samples/s +Step 17550/44000 | Loss: 3.0195 | LR: 3.36e-04 | 1290.9ms/step | 15.5 samples/s +Step 17560/44000 | Loss: 3.0203 | LR: 3.36e-04 | 1291.1ms/step | 15.5 samples/s +Step 17570/44000 | Loss: 3.0539 | LR: 3.35e-04 | 1291.0ms/step | 15.5 samples/s +Step 17580/44000 | Loss: 3.0295 | LR: 3.35e-04 | 1292.6ms/step | 15.5 samples/s +Step 17590/44000 | Loss: 3.0332 | LR: 3.35e-04 | 1293.6ms/step | 15.5 samples/s +Step 17600/44000 | Loss: 3.0426 | LR: 3.35e-04 | 1293.1ms/step | 15.5 samples/s +Step 17610/44000 | Loss: 3.0400 | LR: 3.35e-04 | 1291.8ms/step | 15.5 samples/s +Step 17620/44000 | Loss: 3.0893 | LR: 3.35e-04 | 1293.0ms/step | 15.5 samples/s +Step 17630/44000 | Loss: 3.0582 | LR: 3.34e-04 | 1291.8ms/step | 15.5 samples/s +Step 17640/44000 | Loss: 3.0446 | LR: 3.34e-04 | 1291.7ms/step | 15.5 samples/s +Step 17650/44000 | Loss: 3.0127 | LR: 3.34e-04 | 1292.7ms/step | 15.5 samples/s +Step 17660/44000 | Loss: 3.0435 | LR: 3.34e-04 | 1290.8ms/step | 15.5 samples/s +Step 17670/44000 | Loss: 3.0552 | LR: 3.34e-04 | 1293.6ms/step | 15.5 samples/s +Step 17680/44000 | Loss: 3.0043 | LR: 3.34e-04 | 1290.8ms/step | 15.5 samples/s +Step 17690/44000 | Loss: 3.0240 | LR: 3.33e-04 | 1292.9ms/step | 15.5 samples/s +Step 17700/44000 | Loss: 3.0493 | LR: 3.33e-04 | 1292.6ms/step | 15.5 samples/s +Step 17710/44000 | Loss: 3.0234 | LR: 3.33e-04 | 1291.1ms/step | 15.5 samples/s +Step 17720/44000 | Loss: 3.0158 | LR: 3.33e-04 | 1291.4ms/step | 15.5 samples/s +Step 17730/44000 | Loss: 2.9981 | LR: 3.33e-04 | 1289.5ms/step | 15.5 samples/s +Step 17740/44000 | Loss: 3.0109 | LR: 3.33e-04 | 1288.9ms/step | 15.5 samples/s +Step 17750/44000 | Loss: 3.0465 | LR: 3.32e-04 | 1289.5ms/step | 15.5 samples/s +Validation at step 17750: Loss = 3.0443, Perplexity = 20.99 +Step 17760/44000 | Loss: 3.0267 | LR: 3.32e-04 | 3444.7ms/step | 5.8 samples/s +Step 17770/44000 | Loss: 3.0424 | LR: 3.32e-04 | 1288.5ms/step | 15.5 samples/s +Step 17780/44000 | Loss: 3.0445 | LR: 3.32e-04 | 1288.4ms/step | 15.5 samples/s +Step 17790/44000 | Loss: 3.0239 | LR: 3.32e-04 | 1288.8ms/step | 15.5 samples/s +Step 17800/44000 | Loss: 3.0283 | LR: 3.32e-04 | 1290.3ms/step | 15.5 samples/s +Step 17810/44000 | Loss: 3.0285 | LR: 3.31e-04 | 1290.1ms/step | 15.5 samples/s +Step 17820/44000 | Loss: 3.0588 | LR: 3.31e-04 | 1290.8ms/step | 15.5 samples/s +Step 17830/44000 | Loss: 3.0197 | LR: 3.31e-04 | 1291.1ms/step | 15.5 samples/s +Step 17840/44000 | Loss: 3.0359 | LR: 3.31e-04 | 1290.8ms/step | 15.5 samples/s +Step 17850/44000 | Loss: 3.0249 | LR: 3.31e-04 | 1290.9ms/step | 15.5 samples/s +Step 17860/44000 | Loss: 3.0104 | LR: 3.31e-04 | 1291.1ms/step | 15.5 samples/s +Step 17870/44000 | Loss: 3.0266 | LR: 3.30e-04 | 1291.0ms/step | 15.5 samples/s +Step 17880/44000 | Loss: 3.0160 | LR: 3.30e-04 | 1287.5ms/step | 15.5 samples/s +Step 17890/44000 | Loss: 2.9993 | LR: 3.30e-04 | 1288.0ms/step | 15.5 samples/s +Step 17900/44000 | Loss: 3.0401 | LR: 3.30e-04 | 1291.9ms/step | 15.5 samples/s +Step 17910/44000 | Loss: 3.0480 | LR: 3.30e-04 | 1291.4ms/step | 15.5 samples/s +Step 17920/44000 | Loss: 3.0203 | LR: 3.29e-04 | 1289.8ms/step | 15.5 samples/s +Step 17930/44000 | Loss: 3.0246 | LR: 3.29e-04 | 1291.9ms/step | 15.5 samples/s +Step 17940/44000 | Loss: 3.0241 | LR: 3.29e-04 | 1290.3ms/step | 15.5 samples/s +Step 17950/44000 | Loss: 3.0217 | LR: 3.29e-04 | 1291.2ms/step | 15.5 samples/s +Step 17960/44000 | Loss: 3.0381 | LR: 3.29e-04 | 1289.3ms/step | 15.5 samples/s +Step 17970/44000 | Loss: 3.0410 | LR: 3.29e-04 | 1290.3ms/step | 15.5 samples/s +Step 17980/44000 | Loss: 2.9991 | LR: 3.28e-04 | 1286.9ms/step | 15.5 samples/s +Step 17990/44000 | Loss: 3.0332 | LR: 3.28e-04 | 1289.5ms/step | 15.5 samples/s +Step 18000/44000 | Loss: 3.0406 | LR: 3.28e-04 | 1288.9ms/step | 15.5 samples/s +Validation at step 18000: Loss = 3.0419, Perplexity = 20.94 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-18000 +Step 18010/44000 | Loss: 3.0200 | LR: 3.28e-04 | 3736.3ms/step | 5.4 samples/s +Step 18020/44000 | Loss: 3.0174 | LR: 3.28e-04 | 1290.7ms/step | 15.5 samples/s +Step 18030/44000 | Loss: 2.9952 | LR: 3.28e-04 | 1289.7ms/step | 15.5 samples/s +Step 18040/44000 | Loss: 3.0477 | LR: 3.27e-04 | 1291.3ms/step | 15.5 samples/s +Step 18050/44000 | Loss: 3.0131 | LR: 3.27e-04 | 1290.6ms/step | 15.5 samples/s +Step 18060/44000 | Loss: 3.0181 | LR: 3.27e-04 | 1291.2ms/step | 15.5 samples/s +Step 18070/44000 | Loss: 3.0029 | LR: 3.27e-04 | 1291.9ms/step | 15.5 samples/s +Step 18080/44000 | Loss: 3.0139 | LR: 3.27e-04 | 1287.9ms/step | 15.5 samples/s +Step 18090/44000 | Loss: 3.0259 | LR: 3.27e-04 | 1291.0ms/step | 15.5 samples/s +Step 18100/44000 | Loss: 3.0055 | LR: 3.26e-04 | 1292.7ms/step | 15.5 samples/s +Step 18110/44000 | Loss: 3.0431 | LR: 3.26e-04 | 1295.1ms/step | 15.4 samples/s +Step 18120/44000 | Loss: 3.0406 | LR: 3.26e-04 | 1296.8ms/step | 15.4 samples/s +Step 18130/44000 | Loss: 3.0283 | LR: 3.26e-04 | 1293.0ms/step | 15.5 samples/s +Step 18140/44000 | Loss: 3.0125 | LR: 3.26e-04 | 1290.6ms/step | 15.5 samples/s +Step 18150/44000 | Loss: 2.9991 | LR: 3.26e-04 | 1292.2ms/step | 15.5 samples/s +Step 18160/44000 | Loss: 3.0276 | LR: 3.25e-04 | 1290.8ms/step | 15.5 samples/s +Step 18170/44000 | Loss: 3.0281 | LR: 3.25e-04 | 1289.5ms/step | 15.5 samples/s +Step 18180/44000 | Loss: 3.0163 | LR: 3.25e-04 | 1289.0ms/step | 15.5 samples/s +Step 18190/44000 | Loss: 3.0406 | LR: 3.25e-04 | 1289.3ms/step | 15.5 samples/s +Step 18200/44000 | Loss: 3.0310 | LR: 3.25e-04 | 1287.9ms/step | 15.5 samples/s +Step 18210/44000 | Loss: 3.0249 | LR: 3.24e-04 | 1287.9ms/step | 15.5 samples/s +Step 18220/44000 | Loss: 3.0154 | LR: 3.24e-04 | 1287.0ms/step | 15.5 samples/s +Step 18230/44000 | Loss: 3.0165 | LR: 3.24e-04 | 1286.5ms/step | 15.5 samples/s +Step 18240/44000 | Loss: 3.0158 | LR: 3.24e-04 | 1289.6ms/step | 15.5 samples/s +Step 18250/44000 | Loss: 3.0096 | LR: 3.24e-04 | 1289.7ms/step | 15.5 samples/s +Validation at step 18250: Loss = 3.0385, Perplexity = 20.87 +Step 18260/44000 | Loss: 3.0259 | LR: 3.24e-04 | 3445.7ms/step | 5.8 samples/s +Step 18270/44000 | Loss: 3.0294 | LR: 3.23e-04 | 1290.3ms/step | 15.5 samples/s +Step 18280/44000 | Loss: 3.0377 | LR: 3.23e-04 | 1289.9ms/step | 15.5 samples/s +Step 18290/44000 | Loss: 3.0062 | LR: 3.23e-04 | 1288.5ms/step | 15.5 samples/s +Step 18300/44000 | Loss: 3.0168 | LR: 3.23e-04 | 1288.2ms/step | 15.5 samples/s +Step 18310/44000 | Loss: 3.0524 | LR: 3.23e-04 | 1291.1ms/step | 15.5 samples/s +Step 18320/44000 | Loss: 3.0050 | LR: 3.23e-04 | 1289.6ms/step | 15.5 samples/s +Step 18330/44000 | Loss: 3.0572 | LR: 3.22e-04 | 1289.2ms/step | 15.5 samples/s +Step 18340/44000 | Loss: 3.0354 | LR: 3.22e-04 | 1289.7ms/step | 15.5 samples/s +Step 18350/44000 | Loss: 3.0346 | LR: 3.22e-04 | 1289.6ms/step | 15.5 samples/s +Step 18360/44000 | Loss: 3.0532 | LR: 3.22e-04 | 1290.2ms/step | 15.5 samples/s +Step 18370/44000 | Loss: 3.0178 | LR: 3.22e-04 | 1289.4ms/step | 15.5 samples/s +Step 18380/44000 | Loss: 3.0118 | LR: 3.22e-04 | 1291.1ms/step | 15.5 samples/s +Step 18390/44000 | Loss: 3.0446 | LR: 3.21e-04 | 1292.5ms/step | 15.5 samples/s +Step 18400/44000 | Loss: 3.0392 | LR: 3.21e-04 | 1291.8ms/step | 15.5 samples/s +Step 18410/44000 | Loss: 3.0252 | LR: 3.21e-04 | 1292.6ms/step | 15.5 samples/s +Step 18420/44000 | Loss: 2.9991 | LR: 3.21e-04 | 1292.1ms/step | 15.5 samples/s +Step 18430/44000 | Loss: 3.0375 | LR: 3.21e-04 | 1291.0ms/step | 15.5 samples/s +Step 18440/44000 | Loss: 3.0161 | LR: 3.20e-04 | 1289.9ms/step | 15.5 samples/s +Step 18450/44000 | Loss: 3.0365 | LR: 3.20e-04 | 1288.6ms/step | 15.5 samples/s +Step 18460/44000 | Loss: 3.0508 | LR: 3.20e-04 | 1288.9ms/step | 15.5 samples/s +Step 18470/44000 | Loss: 2.9963 | LR: 3.20e-04 | 1288.5ms/step | 15.5 samples/s +Step 18480/44000 | Loss: 3.0299 | LR: 3.20e-04 | 1290.0ms/step | 15.5 samples/s +Step 18490/44000 | Loss: 3.0319 | LR: 3.20e-04 | 1293.1ms/step | 15.5 samples/s +Step 18500/44000 | Loss: 3.0246 | LR: 3.19e-04 | 1291.2ms/step | 15.5 samples/s +Validation at step 18500: Loss = 3.0361, Perplexity = 20.82 +Step 18510/44000 | Loss: 3.0006 | LR: 3.19e-04 | 3448.4ms/step | 5.8 samples/s +Step 18520/44000 | Loss: 3.0033 | LR: 3.19e-04 | 1288.0ms/step | 15.5 samples/s +Step 18530/44000 | Loss: 2.9981 | LR: 3.19e-04 | 1290.2ms/step | 15.5 samples/s +Step 18540/44000 | Loss: 3.0484 | LR: 3.19e-04 | 1289.4ms/step | 15.5 samples/s +Step 18550/44000 | Loss: 3.0216 | LR: 3.19e-04 | 1289.4ms/step | 15.5 samples/s +Step 18560/44000 | Loss: 2.9907 | LR: 3.18e-04 | 1291.3ms/step | 15.5 samples/s +Step 18570/44000 | Loss: 3.0249 | LR: 3.18e-04 | 1287.4ms/step | 15.5 samples/s +Step 18580/44000 | Loss: 2.9950 | LR: 3.18e-04 | 1288.7ms/step | 15.5 samples/s +Step 18590/44000 | Loss: 3.0117 | LR: 3.18e-04 | 1291.4ms/step | 15.5 samples/s +Step 18600/44000 | Loss: 3.0346 | LR: 3.18e-04 | 1289.6ms/step | 15.5 samples/s +Step 18610/44000 | Loss: 3.0460 | LR: 3.18e-04 | 1289.2ms/step | 15.5 samples/s +Step 18620/44000 | Loss: 3.0068 | LR: 3.17e-04 | 1290.6ms/step | 15.5 samples/s +Step 18630/44000 | Loss: 3.0093 | LR: 3.17e-04 | 1290.8ms/step | 15.5 samples/s +Step 18640/44000 | Loss: 3.0122 | LR: 3.17e-04 | 1291.8ms/step | 15.5 samples/s +Step 18650/44000 | Loss: 2.9996 | LR: 3.17e-04 | 1290.7ms/step | 15.5 samples/s +Step 18660/44000 | Loss: 2.9756 | LR: 3.17e-04 | 1291.1ms/step | 15.5 samples/s +Step 18670/44000 | Loss: 2.9948 | LR: 3.16e-04 | 1291.4ms/step | 15.5 samples/s +Step 18680/44000 | Loss: 3.0128 | LR: 3.16e-04 | 1290.9ms/step | 15.5 samples/s +Step 18690/44000 | Loss: 3.0143 | LR: 3.16e-04 | 1292.0ms/step | 15.5 samples/s +Step 18700/44000 | Loss: 2.9901 | LR: 3.16e-04 | 1290.5ms/step | 15.5 samples/s +Step 18710/44000 | Loss: 3.0270 | LR: 3.16e-04 | 1288.4ms/step | 15.5 samples/s +Step 18720/44000 | Loss: 3.0344 | LR: 3.16e-04 | 1288.0ms/step | 15.5 samples/s +Step 18730/44000 | Loss: 3.0336 | LR: 3.15e-04 | 1289.3ms/step | 15.5 samples/s +Step 18740/44000 | Loss: 3.0336 | LR: 3.15e-04 | 1288.1ms/step | 15.5 samples/s +Step 18750/44000 | Loss: 3.0380 | LR: 3.15e-04 | 1289.6ms/step | 15.5 samples/s +Validation at step 18750: Loss = 3.0328, Perplexity = 20.76 +Step 18760/44000 | Loss: 3.0186 | LR: 3.15e-04 | 3446.9ms/step | 5.8 samples/s +Step 18770/44000 | Loss: 3.0168 | LR: 3.15e-04 | 1290.5ms/step | 15.5 samples/s +Step 18780/44000 | Loss: 3.0107 | LR: 3.15e-04 | 1289.6ms/step | 15.5 samples/s +Step 18790/44000 | Loss: 3.0083 | LR: 3.14e-04 | 1290.9ms/step | 15.5 samples/s +Step 18800/44000 | Loss: 3.0313 | LR: 3.14e-04 | 1288.4ms/step | 15.5 samples/s +Step 18810/44000 | Loss: 3.0229 | LR: 3.14e-04 | 1290.3ms/step | 15.5 samples/s +Step 18820/44000 | Loss: 3.0139 | LR: 3.14e-04 | 1289.4ms/step | 15.5 samples/s +Step 18830/44000 | Loss: 3.0041 | LR: 3.14e-04 | 1290.3ms/step | 15.5 samples/s +Step 18840/44000 | Loss: 3.0295 | LR: 3.13e-04 | 1289.4ms/step | 15.5 samples/s +Step 18850/44000 | Loss: 2.9866 | LR: 3.13e-04 | 1291.7ms/step | 15.5 samples/s +Step 18860/44000 | Loss: 3.0180 | LR: 3.13e-04 | 1293.3ms/step | 15.5 samples/s +Step 18870/44000 | Loss: 3.0310 | LR: 3.13e-04 | 1293.0ms/step | 15.5 samples/s +Step 18880/44000 | Loss: 3.0011 | LR: 3.13e-04 | 1288.9ms/step | 15.5 samples/s +Step 18890/44000 | Loss: 3.0183 | LR: 3.13e-04 | 1289.7ms/step | 15.5 samples/s +Step 18900/44000 | Loss: 2.9801 | LR: 3.12e-04 | 1291.9ms/step | 15.5 samples/s +Step 18910/44000 | Loss: 3.0185 | LR: 3.12e-04 | 1292.4ms/step | 15.5 samples/s +Step 18920/44000 | Loss: 3.0241 | LR: 3.12e-04 | 1290.2ms/step | 15.5 samples/s +Step 18930/44000 | Loss: 3.0195 | LR: 3.12e-04 | 1289.4ms/step | 15.5 samples/s +Step 18940/44000 | Loss: 3.0276 | LR: 3.12e-04 | 1290.9ms/step | 15.5 samples/s +Step 18950/44000 | Loss: 2.9882 | LR: 3.12e-04 | 1290.1ms/step | 15.5 samples/s +Step 18960/44000 | Loss: 3.0041 | LR: 3.11e-04 | 1289.9ms/step | 15.5 samples/s +Step 18970/44000 | Loss: 3.0245 | LR: 3.11e-04 | 1292.3ms/step | 15.5 samples/s +Step 18980/44000 | Loss: 2.9983 | LR: 3.11e-04 | 1292.7ms/step | 15.5 samples/s +Step 18990/44000 | Loss: 3.0162 | LR: 3.11e-04 | 1292.9ms/step | 15.5 samples/s +Step 19000/44000 | Loss: 3.0129 | LR: 3.11e-04 | 1292.9ms/step | 15.5 samples/s +Validation at step 19000: Loss = 3.0291, Perplexity = 20.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-19000 +Step 19010/44000 | Loss: 2.9850 | LR: 3.11e-04 | 3743.0ms/step | 5.3 samples/s +Step 19020/44000 | Loss: 3.0227 | LR: 3.10e-04 | 1289.0ms/step | 15.5 samples/s +Step 19030/44000 | Loss: 3.0019 | LR: 3.10e-04 | 1290.5ms/step | 15.5 samples/s +Step 19040/44000 | Loss: 3.0135 | LR: 3.10e-04 | 1293.6ms/step | 15.5 samples/s +Step 19050/44000 | Loss: 3.0306 | LR: 3.10e-04 | 1290.2ms/step | 15.5 samples/s +Step 19060/44000 | Loss: 3.0444 | LR: 3.10e-04 | 1291.4ms/step | 15.5 samples/s +Step 19070/44000 | Loss: 3.0207 | LR: 3.09e-04 | 1290.4ms/step | 15.5 samples/s +Step 19080/44000 | Loss: 2.9886 | LR: 3.09e-04 | 1291.8ms/step | 15.5 samples/s +Step 19090/44000 | Loss: 3.0076 | LR: 3.09e-04 | 1291.4ms/step | 15.5 samples/s +Step 19100/44000 | Loss: 2.9918 | LR: 3.09e-04 | 1290.9ms/step | 15.5 samples/s +Step 19110/44000 | Loss: 3.0223 | LR: 3.09e-04 | 1289.1ms/step | 15.5 samples/s +Step 19120/44000 | Loss: 3.0162 | LR: 3.09e-04 | 1285.9ms/step | 15.6 samples/s +Step 19130/44000 | Loss: 3.0290 | LR: 3.08e-04 | 1289.4ms/step | 15.5 samples/s +Step 19140/44000 | Loss: 2.9953 | LR: 3.08e-04 | 1291.6ms/step | 15.5 samples/s +Step 19150/44000 | Loss: 3.0090 | LR: 3.08e-04 | 1289.8ms/step | 15.5 samples/s +Step 19160/44000 | Loss: 3.0103 | LR: 3.08e-04 | 1289.9ms/step | 15.5 samples/s +Step 19170/44000 | Loss: 2.9722 | LR: 3.08e-04 | 1290.3ms/step | 15.5 samples/s +Step 19180/44000 | Loss: 3.0336 | LR: 3.08e-04 | 1289.7ms/step | 15.5 samples/s +Step 19190/44000 | Loss: 2.9856 | LR: 3.07e-04 | 1290.8ms/step | 15.5 samples/s +Step 19200/44000 | Loss: 2.9902 | LR: 3.07e-04 | 1291.2ms/step | 15.5 samples/s +Step 19210/44000 | Loss: 3.0178 | LR: 3.07e-04 | 1288.6ms/step | 15.5 samples/s +Step 19220/44000 | Loss: 3.0039 | LR: 3.07e-04 | 1289.4ms/step | 15.5 samples/s +Step 19230/44000 | Loss: 3.0681 | LR: 3.07e-04 | 1289.9ms/step | 15.5 samples/s +Step 19240/44000 | Loss: 3.0183 | LR: 3.06e-04 | 1289.6ms/step | 15.5 samples/s +Step 19250/44000 | Loss: 3.0044 | LR: 3.06e-04 | 1290.5ms/step | 15.5 samples/s +Validation at step 19250: Loss = 3.0268, Perplexity = 20.63 +Step 19260/44000 | Loss: 3.0137 | LR: 3.06e-04 | 3451.2ms/step | 5.8 samples/s +Step 19270/44000 | Loss: 3.0009 | LR: 3.06e-04 | 1288.7ms/step | 15.5 samples/s +Step 19280/44000 | Loss: 2.9873 | LR: 3.06e-04 | 1288.7ms/step | 15.5 samples/s +Step 19290/44000 | Loss: 3.0088 | LR: 3.06e-04 | 1292.6ms/step | 15.5 samples/s +Step 19300/44000 | Loss: 2.9988 | LR: 3.05e-04 | 1291.8ms/step | 15.5 samples/s +Step 19310/44000 | Loss: 3.0258 | LR: 3.05e-04 | 1293.7ms/step | 15.5 samples/s +Step 19320/44000 | Loss: 3.0248 | LR: 3.05e-04 | 1293.3ms/step | 15.5 samples/s +Step 19330/44000 | Loss: 3.0334 | LR: 3.05e-04 | 1291.6ms/step | 15.5 samples/s +Step 19340/44000 | Loss: 3.0284 | LR: 3.05e-04 | 1292.8ms/step | 15.5 samples/s +Step 19350/44000 | Loss: 3.0140 | LR: 3.04e-04 | 1290.6ms/step | 15.5 samples/s +Step 19360/44000 | Loss: 3.0260 | LR: 3.04e-04 | 1289.8ms/step | 15.5 samples/s +Step 19370/44000 | Loss: 3.0200 | LR: 3.04e-04 | 1290.7ms/step | 15.5 samples/s +Step 19380/44000 | Loss: 2.9847 | LR: 3.04e-04 | 1289.4ms/step | 15.5 samples/s +Step 19390/44000 | Loss: 2.9916 | LR: 3.04e-04 | 1291.0ms/step | 15.5 samples/s +Step 19400/44000 | Loss: 3.0134 | LR: 3.04e-04 | 1289.6ms/step | 15.5 samples/s +Step 19410/44000 | Loss: 3.0023 | LR: 3.03e-04 | 1291.0ms/step | 15.5 samples/s +Step 19420/44000 | Loss: 3.0200 | LR: 3.03e-04 | 1293.5ms/step | 15.5 samples/s +Step 19430/44000 | Loss: 2.9788 | LR: 3.03e-04 | 1290.7ms/step | 15.5 samples/s +Step 19440/44000 | Loss: 3.0316 | LR: 3.03e-04 | 1292.7ms/step | 15.5 samples/s +Step 19450/44000 | Loss: 3.0038 | LR: 3.03e-04 | 1293.6ms/step | 15.5 samples/s +Step 19460/44000 | Loss: 3.0180 | LR: 3.03e-04 | 1293.2ms/step | 15.5 samples/s +Step 19470/44000 | Loss: 3.0268 | LR: 3.02e-04 | 1294.5ms/step | 15.4 samples/s +Step 19480/44000 | Loss: 2.9842 | LR: 3.02e-04 | 1294.0ms/step | 15.5 samples/s +Step 19490/44000 | Loss: 3.0335 | LR: 3.02e-04 | 1290.2ms/step | 15.5 samples/s +Step 19500/44000 | Loss: 2.9958 | LR: 3.02e-04 | 1291.3ms/step | 15.5 samples/s +Validation at step 19500: Loss = 3.0230, Perplexity = 20.55 +Step 19510/44000 | Loss: 3.0181 | LR: 3.02e-04 | 3448.3ms/step | 5.8 samples/s +Step 19520/44000 | Loss: 2.9995 | LR: 3.01e-04 | 1290.0ms/step | 15.5 samples/s +Step 19530/44000 | Loss: 2.9778 | LR: 3.01e-04 | 1290.4ms/step | 15.5 samples/s +Step 19540/44000 | Loss: 3.0095 | LR: 3.01e-04 | 1289.7ms/step | 15.5 samples/s +Step 19550/44000 | Loss: 2.9961 | LR: 3.01e-04 | 1291.7ms/step | 15.5 samples/s +Step 19560/44000 | Loss: 3.0029 | LR: 3.01e-04 | 1291.8ms/step | 15.5 samples/s +Step 19570/44000 | Loss: 3.0170 | LR: 3.01e-04 | 1292.1ms/step | 15.5 samples/s +Step 19580/44000 | Loss: 3.0269 | LR: 3.00e-04 | 1290.8ms/step | 15.5 samples/s +Step 19590/44000 | Loss: 3.0110 | LR: 3.00e-04 | 1291.7ms/step | 15.5 samples/s +Step 19600/44000 | Loss: 2.9878 | LR: 3.00e-04 | 1290.8ms/step | 15.5 samples/s +Step 19610/44000 | Loss: 3.0067 | LR: 3.00e-04 | 1291.2ms/step | 15.5 samples/s +Step 19620/44000 | Loss: 3.0040 | LR: 3.00e-04 | 1292.4ms/step | 15.5 samples/s +Step 19630/44000 | Loss: 3.0298 | LR: 3.00e-04 | 1291.5ms/step | 15.5 samples/s +Step 19640/44000 | Loss: 3.0246 | LR: 2.99e-04 | 1291.2ms/step | 15.5 samples/s +Step 19650/44000 | Loss: 2.9932 | LR: 2.99e-04 | 1289.9ms/step | 15.5 samples/s +Step 19660/44000 | Loss: 3.0074 | LR: 2.99e-04 | 1290.3ms/step | 15.5 samples/s +Step 19670/44000 | Loss: 3.0196 | LR: 2.99e-04 | 1291.3ms/step | 15.5 samples/s +Step 19680/44000 | Loss: 3.0334 | LR: 2.99e-04 | 1289.6ms/step | 15.5 samples/s +Step 19690/44000 | Loss: 2.9911 | LR: 2.98e-04 | 1290.9ms/step | 15.5 samples/s +Step 19700/44000 | Loss: 3.0256 | LR: 2.98e-04 | 1294.4ms/step | 15.5 samples/s +Step 19710/44000 | Loss: 3.0202 | LR: 2.98e-04 | 1294.3ms/step | 15.5 samples/s +Step 19720/44000 | Loss: 3.0080 | LR: 2.98e-04 | 1292.9ms/step | 15.5 samples/s +Step 19730/44000 | Loss: 3.0333 | LR: 2.98e-04 | 1291.4ms/step | 15.5 samples/s +Step 19740/44000 | Loss: 3.0066 | LR: 2.98e-04 | 1289.9ms/step | 15.5 samples/s +Step 19750/44000 | Loss: 2.9881 | LR: 2.97e-04 | 1287.2ms/step | 15.5 samples/s +Validation at step 19750: Loss = 3.0210, Perplexity = 20.51 +Step 19760/44000 | Loss: 3.0204 | LR: 2.97e-04 | 3445.3ms/step | 5.8 samples/s +Step 19770/44000 | Loss: 3.0298 | LR: 2.97e-04 | 1289.7ms/step | 15.5 samples/s +Step 19780/44000 | Loss: 3.0244 | LR: 2.97e-04 | 1288.2ms/step | 15.5 samples/s +Step 19790/44000 | Loss: 3.0033 | LR: 2.97e-04 | 1288.7ms/step | 15.5 samples/s +Step 19800/44000 | Loss: 3.0306 | LR: 2.96e-04 | 1288.7ms/step | 15.5 samples/s +Step 19810/44000 | Loss: 2.9915 | LR: 2.96e-04 | 1288.8ms/step | 15.5 samples/s +Step 19820/44000 | Loss: 2.9811 | LR: 2.96e-04 | 1290.0ms/step | 15.5 samples/s +Step 19830/44000 | Loss: 3.0299 | LR: 2.96e-04 | 1289.3ms/step | 15.5 samples/s +Step 19840/44000 | Loss: 3.0038 | LR: 2.96e-04 | 1287.9ms/step | 15.5 samples/s +Step 19850/44000 | Loss: 3.0025 | LR: 2.96e-04 | 1289.1ms/step | 15.5 samples/s +Step 19860/44000 | Loss: 3.0335 | LR: 2.95e-04 | 1290.8ms/step | 15.5 samples/s +Step 19870/44000 | Loss: 3.0269 | LR: 2.95e-04 | 1290.3ms/step | 15.5 samples/s +Step 19880/44000 | Loss: 3.0117 | LR: 2.95e-04 | 1286.8ms/step | 15.5 samples/s +Step 19890/44000 | Loss: 3.0137 | LR: 2.95e-04 | 1289.2ms/step | 15.5 samples/s +Step 19900/44000 | Loss: 3.0000 | LR: 2.95e-04 | 1289.1ms/step | 15.5 samples/s +Step 19910/44000 | Loss: 3.0104 | LR: 2.95e-04 | 1289.7ms/step | 15.5 samples/s +Step 19920/44000 | Loss: 3.0036 | LR: 2.94e-04 | 1290.8ms/step | 15.5 samples/s +Step 19930/44000 | Loss: 3.0158 | LR: 2.94e-04 | 1289.8ms/step | 15.5 samples/s +Step 19940/44000 | Loss: 2.9787 | LR: 2.94e-04 | 1285.2ms/step | 15.6 samples/s +Step 19950/44000 | Loss: 2.9989 | LR: 2.94e-04 | 1289.6ms/step | 15.5 samples/s +Step 19960/44000 | Loss: 3.0389 | LR: 2.94e-04 | 1290.1ms/step | 15.5 samples/s +Step 19970/44000 | Loss: 3.0258 | LR: 2.93e-04 | 1291.5ms/step | 15.5 samples/s +Step 19980/44000 | Loss: 3.0235 | LR: 2.93e-04 | 1290.3ms/step | 15.5 samples/s +Step 19990/44000 | Loss: 3.0119 | LR: 2.93e-04 | 1292.1ms/step | 15.5 samples/s +Step 20000/44000 | Loss: 3.0174 | LR: 2.93e-04 | 1294.1ms/step | 15.5 samples/s +Validation at step 20000: Loss = 3.0184, Perplexity = 20.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-20000 +Step 20010/44000 | Loss: 2.9929 | LR: 2.93e-04 | 3737.2ms/step | 5.4 samples/s +Step 20020/44000 | Loss: 3.0170 | LR: 2.93e-04 | 1288.4ms/step | 15.5 samples/s +Step 20030/44000 | Loss: 2.9921 | LR: 2.92e-04 | 1287.4ms/step | 15.5 samples/s +Step 20040/44000 | Loss: 2.9947 | LR: 2.92e-04 | 1288.0ms/step | 15.5 samples/s +Step 20050/44000 | Loss: 3.0094 | LR: 2.92e-04 | 1289.4ms/step | 15.5 samples/s +Step 20060/44000 | Loss: 2.9778 | LR: 2.92e-04 | 1288.3ms/step | 15.5 samples/s +Step 20070/44000 | Loss: 3.0050 | LR: 2.92e-04 | 1289.6ms/step | 15.5 samples/s +Step 20080/44000 | Loss: 2.9874 | LR: 2.91e-04 | 1288.9ms/step | 15.5 samples/s +Step 20090/44000 | Loss: 2.9965 | LR: 2.91e-04 | 1290.1ms/step | 15.5 samples/s +Step 20100/44000 | Loss: 3.0137 | LR: 2.91e-04 | 1288.1ms/step | 15.5 samples/s +Step 20110/44000 | Loss: 3.0051 | LR: 2.91e-04 | 1289.6ms/step | 15.5 samples/s +Step 20120/44000 | Loss: 3.0152 | LR: 2.91e-04 | 1289.6ms/step | 15.5 samples/s +Step 20130/44000 | Loss: 3.0119 | LR: 2.91e-04 | 1289.4ms/step | 15.5 samples/s +Step 20140/44000 | Loss: 3.0116 | LR: 2.90e-04 | 1290.0ms/step | 15.5 samples/s +Step 20150/44000 | Loss: 2.9997 | LR: 2.90e-04 | 1290.7ms/step | 15.5 samples/s +Step 20160/44000 | Loss: 3.0144 | LR: 2.90e-04 | 1291.8ms/step | 15.5 samples/s +Step 20170/44000 | Loss: 2.9708 | LR: 2.90e-04 | 1291.8ms/step | 15.5 samples/s +Step 20180/44000 | Loss: 2.9905 | LR: 2.90e-04 | 1291.1ms/step | 15.5 samples/s +Step 20190/44000 | Loss: 3.0212 | LR: 2.90e-04 | 1289.7ms/step | 15.5 samples/s +Step 20200/44000 | Loss: 2.9794 | LR: 2.89e-04 | 1290.0ms/step | 15.5 samples/s +Step 20210/44000 | Loss: 2.9706 | LR: 2.89e-04 | 1287.7ms/step | 15.5 samples/s +Step 20220/44000 | Loss: 3.0219 | LR: 2.89e-04 | 1289.2ms/step | 15.5 samples/s +Step 20230/44000 | Loss: 3.0086 | LR: 2.89e-04 | 1289.4ms/step | 15.5 samples/s +Step 20240/44000 | Loss: 3.0107 | LR: 2.89e-04 | 1288.2ms/step | 15.5 samples/s +Step 20250/44000 | Loss: 2.9949 | LR: 2.88e-04 | 1290.0ms/step | 15.5 samples/s +Validation at step 20250: Loss = 3.0160, Perplexity = 20.41 +Step 20260/44000 | Loss: 2.9628 | LR: 2.88e-04 | 3445.2ms/step | 5.8 samples/s +Step 20270/44000 | Loss: 3.0134 | LR: 2.88e-04 | 1286.7ms/step | 15.5 samples/s +Step 20280/44000 | Loss: 3.0090 | LR: 2.88e-04 | 1290.9ms/step | 15.5 samples/s +Step 20290/44000 | Loss: 3.0013 | LR: 2.88e-04 | 1291.2ms/step | 15.5 samples/s +Step 20300/44000 | Loss: 3.0238 | LR: 2.88e-04 | 1300.1ms/step | 15.4 samples/s +Step 20310/44000 | Loss: 2.9931 | LR: 2.87e-04 | 1293.7ms/step | 15.5 samples/s +Step 20320/44000 | Loss: 3.0002 | LR: 2.87e-04 | 1291.8ms/step | 15.5 samples/s +Step 20330/44000 | Loss: 2.9905 | LR: 2.87e-04 | 1293.4ms/step | 15.5 samples/s +Step 20340/44000 | Loss: 2.9796 | LR: 2.87e-04 | 1295.5ms/step | 15.4 samples/s +Step 20350/44000 | Loss: 3.0063 | LR: 2.87e-04 | 1300.5ms/step | 15.4 samples/s +Step 20360/44000 | Loss: 3.0195 | LR: 2.86e-04 | 1296.1ms/step | 15.4 samples/s +Step 20370/44000 | Loss: 2.9765 | LR: 2.86e-04 | 1293.7ms/step | 15.5 samples/s +Step 20380/44000 | Loss: 2.9852 | LR: 2.86e-04 | 1293.6ms/step | 15.5 samples/s +Step 20390/44000 | Loss: 3.0016 | LR: 2.86e-04 | 1299.1ms/step | 15.4 samples/s +Step 20400/44000 | Loss: 3.0171 | LR: 2.86e-04 | 1293.6ms/step | 15.5 samples/s +Step 20410/44000 | Loss: 2.9900 | LR: 2.86e-04 | 1292.6ms/step | 15.5 samples/s +Step 20420/44000 | Loss: 3.0147 | LR: 2.85e-04 | 1291.0ms/step | 15.5 samples/s +Step 20430/44000 | Loss: 2.9798 | LR: 2.85e-04 | 1291.2ms/step | 15.5 samples/s +Step 20440/44000 | Loss: 3.0080 | LR: 2.85e-04 | 1293.1ms/step | 15.5 samples/s +Step 20450/44000 | Loss: 3.0004 | LR: 2.85e-04 | 1290.0ms/step | 15.5 samples/s +Step 20460/44000 | Loss: 3.0025 | LR: 2.85e-04 | 1289.1ms/step | 15.5 samples/s +Step 20470/44000 | Loss: 3.0204 | LR: 2.84e-04 | 1288.4ms/step | 15.5 samples/s +Step 20480/44000 | Loss: 2.9974 | LR: 2.84e-04 | 1289.5ms/step | 15.5 samples/s +Step 20490/44000 | Loss: 2.9979 | LR: 2.84e-04 | 1288.2ms/step | 15.5 samples/s +Step 20500/44000 | Loss: 2.9781 | LR: 2.84e-04 | 1287.3ms/step | 15.5 samples/s +Validation at step 20500: Loss = 3.0126, Perplexity = 20.34 +Step 20510/44000 | Loss: 3.0247 | LR: 2.84e-04 | 3443.2ms/step | 5.8 samples/s +Step 20520/44000 | Loss: 2.9787 | LR: 2.84e-04 | 1290.2ms/step | 15.5 samples/s +Step 20530/44000 | Loss: 2.9709 | LR: 2.83e-04 | 1289.0ms/step | 15.5 samples/s +Step 20540/44000 | Loss: 2.9999 | LR: 2.83e-04 | 1289.2ms/step | 15.5 samples/s +Step 20550/44000 | Loss: 3.0119 | LR: 2.83e-04 | 1288.4ms/step | 15.5 samples/s +Step 20560/44000 | Loss: 3.0148 | LR: 2.83e-04 | 1289.0ms/step | 15.5 samples/s +Step 20570/44000 | Loss: 3.0102 | LR: 2.83e-04 | 1291.4ms/step | 15.5 samples/s +Step 20580/44000 | Loss: 2.9817 | LR: 2.83e-04 | 1292.1ms/step | 15.5 samples/s +Step 20590/44000 | Loss: 3.0212 | LR: 2.82e-04 | 1292.8ms/step | 15.5 samples/s +Step 20600/44000 | Loss: 2.9983 | LR: 2.82e-04 | 1291.9ms/step | 15.5 samples/s +Step 20610/44000 | Loss: 3.0043 | LR: 2.82e-04 | 1292.7ms/step | 15.5 samples/s +Step 20620/44000 | Loss: 2.9989 | LR: 2.82e-04 | 1291.8ms/step | 15.5 samples/s +Step 20630/44000 | Loss: 3.0061 | LR: 2.82e-04 | 1291.8ms/step | 15.5 samples/s +Step 20640/44000 | Loss: 2.9668 | LR: 2.81e-04 | 1291.4ms/step | 15.5 samples/s +Step 20650/44000 | Loss: 3.0135 | LR: 2.81e-04 | 1291.4ms/step | 15.5 samples/s +Step 20660/44000 | Loss: 2.9660 | LR: 2.81e-04 | 1291.7ms/step | 15.5 samples/s +Step 20670/44000 | Loss: 3.0134 | LR: 2.81e-04 | 1291.3ms/step | 15.5 samples/s +Step 20680/44000 | Loss: 3.0046 | LR: 2.81e-04 | 1288.3ms/step | 15.5 samples/s +Step 20690/44000 | Loss: 2.9931 | LR: 2.81e-04 | 1289.1ms/step | 15.5 samples/s +Step 20700/44000 | Loss: 3.0034 | LR: 2.80e-04 | 1289.4ms/step | 15.5 samples/s +Step 20710/44000 | Loss: 3.0096 | LR: 2.80e-04 | 1289.0ms/step | 15.5 samples/s +Step 20720/44000 | Loss: 3.0095 | LR: 2.80e-04 | 1289.9ms/step | 15.5 samples/s +Step 20730/44000 | Loss: 3.0238 | LR: 2.80e-04 | 1290.0ms/step | 15.5 samples/s +Step 20740/44000 | Loss: 3.0077 | LR: 2.80e-04 | 1289.7ms/step | 15.5 samples/s +Step 20750/44000 | Loss: 3.0014 | LR: 2.79e-04 | 1289.9ms/step | 15.5 samples/s +Validation at step 20750: Loss = 3.0097, Perplexity = 20.28 +Step 20760/44000 | Loss: 3.0086 | LR: 2.79e-04 | 3453.2ms/step | 5.8 samples/s +Step 20770/44000 | Loss: 2.9912 | LR: 2.79e-04 | 1290.0ms/step | 15.5 samples/s +Step 20780/44000 | Loss: 2.9693 | LR: 2.79e-04 | 1289.5ms/step | 15.5 samples/s +Step 20790/44000 | Loss: 2.9877 | LR: 2.79e-04 | 1290.5ms/step | 15.5 samples/s +Step 20800/44000 | Loss: 2.9895 | LR: 2.79e-04 | 1291.8ms/step | 15.5 samples/s +Step 20810/44000 | Loss: 3.0070 | LR: 2.78e-04 | 1291.9ms/step | 15.5 samples/s +Step 20820/44000 | Loss: 3.0023 | LR: 2.78e-04 | 1291.8ms/step | 15.5 samples/s +Step 20830/44000 | Loss: 3.0068 | LR: 2.78e-04 | 1293.1ms/step | 15.5 samples/s +Step 20840/44000 | Loss: 2.9940 | LR: 2.78e-04 | 1293.3ms/step | 15.5 samples/s +Step 20850/44000 | Loss: 2.9945 | LR: 2.78e-04 | 1290.0ms/step | 15.5 samples/s +Step 20860/44000 | Loss: 3.0134 | LR: 2.77e-04 | 1288.0ms/step | 15.5 samples/s +Step 20870/44000 | Loss: 3.0190 | LR: 2.77e-04 | 1287.9ms/step | 15.5 samples/s +Step 20880/44000 | Loss: 2.9901 | LR: 2.77e-04 | 1289.6ms/step | 15.5 samples/s +Step 20890/44000 | Loss: 2.9758 | LR: 2.77e-04 | 1291.1ms/step | 15.5 samples/s +Step 20900/44000 | Loss: 2.9775 | LR: 2.77e-04 | 1290.7ms/step | 15.5 samples/s +Step 20910/44000 | Loss: 3.0095 | LR: 2.77e-04 | 1289.3ms/step | 15.5 samples/s +Step 20920/44000 | Loss: 2.9720 | LR: 2.76e-04 | 1289.3ms/step | 15.5 samples/s +Step 20930/44000 | Loss: 2.9807 | LR: 2.76e-04 | 1291.6ms/step | 15.5 samples/s +Step 20940/44000 | Loss: 3.0043 | LR: 2.76e-04 | 1291.4ms/step | 15.5 samples/s +Step 20950/44000 | Loss: 2.9693 | LR: 2.76e-04 | 1291.9ms/step | 15.5 samples/s +Step 20960/44000 | Loss: 2.9861 | LR: 2.76e-04 | 1292.1ms/step | 15.5 samples/s +Step 20970/44000 | Loss: 2.9690 | LR: 2.75e-04 | 1287.8ms/step | 15.5 samples/s +Step 20980/44000 | Loss: 2.9735 | LR: 2.75e-04 | 1288.4ms/step | 15.5 samples/s +Step 20990/44000 | Loss: 2.9812 | LR: 2.75e-04 | 1289.9ms/step | 15.5 samples/s +Step 21000/44000 | Loss: 2.9801 | LR: 2.75e-04 | 1290.4ms/step | 15.5 samples/s +Validation at step 21000: Loss = 3.0081, Perplexity = 20.25 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-21000 +Step 21010/44000 | Loss: 2.9879 | LR: 2.75e-04 | 3739.3ms/step | 5.3 samples/s +Step 21020/44000 | Loss: 2.9887 | LR: 2.75e-04 | 1289.5ms/step | 15.5 samples/s +Step 21030/44000 | Loss: 3.0253 | LR: 2.74e-04 | 1290.9ms/step | 15.5 samples/s +Step 21040/44000 | Loss: 2.9702 | LR: 2.74e-04 | 1290.5ms/step | 15.5 samples/s +Step 21050/44000 | Loss: 2.9885 | LR: 2.74e-04 | 1288.9ms/step | 15.5 samples/s +Step 21060/44000 | Loss: 2.9724 | LR: 2.74e-04 | 1287.5ms/step | 15.5 samples/s +Step 21070/44000 | Loss: 3.0170 | LR: 2.74e-04 | 1288.1ms/step | 15.5 samples/s +Step 21080/44000 | Loss: 2.9596 | LR: 2.73e-04 | 1290.7ms/step | 15.5 samples/s +Step 21090/44000 | Loss: 3.0204 | LR: 2.73e-04 | 1287.3ms/step | 15.5 samples/s +Step 21100/44000 | Loss: 3.0035 | LR: 2.73e-04 | 1287.6ms/step | 15.5 samples/s +Step 21110/44000 | Loss: 2.9929 | LR: 2.73e-04 | 1287.8ms/step | 15.5 samples/s +Step 21120/44000 | Loss: 3.0050 | LR: 2.73e-04 | 1291.1ms/step | 15.5 samples/s +Step 21130/44000 | Loss: 3.0262 | LR: 2.73e-04 | 1291.4ms/step | 15.5 samples/s +Step 21140/44000 | Loss: 2.9871 | LR: 2.72e-04 | 1290.5ms/step | 15.5 samples/s +Step 21150/44000 | Loss: 2.9964 | LR: 2.72e-04 | 1291.5ms/step | 15.5 samples/s +Step 21160/44000 | Loss: 2.9785 | LR: 2.72e-04 | 1291.2ms/step | 15.5 samples/s +Step 21170/44000 | Loss: 2.9934 | LR: 2.72e-04 | 1288.1ms/step | 15.5 samples/s +Step 21180/44000 | Loss: 3.0007 | LR: 2.72e-04 | 1289.4ms/step | 15.5 samples/s +Step 21190/44000 | Loss: 2.9406 | LR: 2.71e-04 | 1290.6ms/step | 15.5 samples/s +Step 21200/44000 | Loss: 3.0114 | LR: 2.71e-04 | 1290.5ms/step | 15.5 samples/s +Step 21210/44000 | Loss: 2.9943 | LR: 2.71e-04 | 1289.8ms/step | 15.5 samples/s +Step 21220/44000 | Loss: 2.9835 | LR: 2.71e-04 | 1290.5ms/step | 15.5 samples/s +Step 21230/44000 | Loss: 2.9963 | LR: 2.71e-04 | 1290.5ms/step | 15.5 samples/s +Step 21240/44000 | Loss: 2.9865 | LR: 2.71e-04 | 1288.2ms/step | 15.5 samples/s +Step 21250/44000 | Loss: 2.9800 | LR: 2.70e-04 | 1287.0ms/step | 15.5 samples/s +Validation at step 21250: Loss = 3.0054, Perplexity = 20.19 +Step 21260/44000 | Loss: 3.0208 | LR: 2.70e-04 | 3443.3ms/step | 5.8 samples/s +Step 21270/44000 | Loss: 2.9992 | LR: 2.70e-04 | 1288.1ms/step | 15.5 samples/s +Step 21280/44000 | Loss: 2.9891 | LR: 2.70e-04 | 1289.7ms/step | 15.5 samples/s +Step 21290/44000 | Loss: 2.9854 | LR: 2.70e-04 | 1289.5ms/step | 15.5 samples/s +Step 21300/44000 | Loss: 2.9936 | LR: 2.70e-04 | 1289.0ms/step | 15.5 samples/s +Step 21310/44000 | Loss: 2.9954 | LR: 2.69e-04 | 1290.1ms/step | 15.5 samples/s +Step 21320/44000 | Loss: 3.0081 | LR: 2.69e-04 | 1290.3ms/step | 15.5 samples/s +Step 21330/44000 | Loss: 2.9906 | LR: 2.69e-04 | 1290.2ms/step | 15.5 samples/s +Step 21340/44000 | Loss: 2.9848 | LR: 2.69e-04 | 1291.1ms/step | 15.5 samples/s +Step 21350/44000 | Loss: 3.0195 | LR: 2.69e-04 | 1292.1ms/step | 15.5 samples/s +Step 21360/44000 | Loss: 2.9887 | LR: 2.68e-04 | 1291.3ms/step | 15.5 samples/s +Step 21370/44000 | Loss: 2.9927 | LR: 2.68e-04 | 1291.4ms/step | 15.5 samples/s +Step 21380/44000 | Loss: 2.9801 | LR: 2.68e-04 | 1291.9ms/step | 15.5 samples/s +Step 21390/44000 | Loss: 3.0114 | LR: 2.68e-04 | 1292.0ms/step | 15.5 samples/s +Step 21400/44000 | Loss: 2.9730 | LR: 2.68e-04 | 1291.2ms/step | 15.5 samples/s +Step 21410/44000 | Loss: 3.0127 | LR: 2.68e-04 | 1290.8ms/step | 15.5 samples/s +Step 21420/44000 | Loss: 2.9615 | LR: 2.67e-04 | 1292.8ms/step | 15.5 samples/s +Step 21430/44000 | Loss: 2.9771 | LR: 2.67e-04 | 1292.4ms/step | 15.5 samples/s +Step 21440/44000 | Loss: 2.9692 | LR: 2.67e-04 | 1289.1ms/step | 15.5 samples/s +Step 21450/44000 | Loss: 2.9874 | LR: 2.67e-04 | 1292.8ms/step | 15.5 samples/s +Step 21460/44000 | Loss: 2.9935 | LR: 2.67e-04 | 1289.6ms/step | 15.5 samples/s +Step 21470/44000 | Loss: 2.9984 | LR: 2.66e-04 | 1290.5ms/step | 15.5 samples/s +Step 21480/44000 | Loss: 2.9789 | LR: 2.66e-04 | 1290.0ms/step | 15.5 samples/s +Step 21490/44000 | Loss: 2.9951 | LR: 2.66e-04 | 1292.1ms/step | 15.5 samples/s +Step 21500/44000 | Loss: 2.9942 | LR: 2.66e-04 | 1289.8ms/step | 15.5 samples/s +Validation at step 21500: Loss = 3.0030, Perplexity = 20.15 +Step 21510/44000 | Loss: 2.9981 | LR: 2.66e-04 | 3446.0ms/step | 5.8 samples/s +Step 21520/44000 | Loss: 2.9840 | LR: 2.66e-04 | 1289.0ms/step | 15.5 samples/s +Step 21530/44000 | Loss: 2.9796 | LR: 2.65e-04 | 1288.5ms/step | 15.5 samples/s +Step 21540/44000 | Loss: 2.9913 | LR: 2.65e-04 | 1288.8ms/step | 15.5 samples/s +Step 21550/44000 | Loss: 2.9908 | LR: 2.65e-04 | 1291.0ms/step | 15.5 samples/s +Step 21560/44000 | Loss: 2.9953 | LR: 2.65e-04 | 1290.1ms/step | 15.5 samples/s +Step 21570/44000 | Loss: 2.9739 | LR: 2.65e-04 | 1291.5ms/step | 15.5 samples/s +Step 21580/44000 | Loss: 2.9666 | LR: 2.64e-04 | 1292.0ms/step | 15.5 samples/s +Step 21590/44000 | Loss: 2.9831 | LR: 2.64e-04 | 1290.2ms/step | 15.5 samples/s +Step 21600/44000 | Loss: 2.9844 | LR: 2.64e-04 | 1290.5ms/step | 15.5 samples/s +Step 21610/44000 | Loss: 2.9799 | LR: 2.64e-04 | 1290.2ms/step | 15.5 samples/s +Step 21620/44000 | Loss: 3.0011 | LR: 2.64e-04 | 1290.5ms/step | 15.5 samples/s +Step 21630/44000 | Loss: 2.9726 | LR: 2.64e-04 | 1290.9ms/step | 15.5 samples/s +Step 21640/44000 | Loss: 2.9641 | LR: 2.63e-04 | 1290.9ms/step | 15.5 samples/s +Step 21650/44000 | Loss: 2.9913 | LR: 2.63e-04 | 1290.4ms/step | 15.5 samples/s +Step 21660/44000 | Loss: 3.0257 | LR: 2.63e-04 | 1294.5ms/step | 15.4 samples/s +Step 21670/44000 | Loss: 2.9718 | LR: 2.63e-04 | 1297.3ms/step | 15.4 samples/s +Step 21680/44000 | Loss: 2.9945 | LR: 2.63e-04 | 1297.1ms/step | 15.4 samples/s +Step 21690/44000 | Loss: 3.0218 | LR: 2.62e-04 | 1294.1ms/step | 15.5 samples/s +Step 21700/44000 | Loss: 3.0031 | LR: 2.62e-04 | 1294.1ms/step | 15.5 samples/s +Step 21710/44000 | Loss: 2.9875 | LR: 2.62e-04 | 1294.0ms/step | 15.5 samples/s +Step 21720/44000 | Loss: 2.9597 | LR: 2.62e-04 | 1296.2ms/step | 15.4 samples/s +Step 21730/44000 | Loss: 2.9844 | LR: 2.62e-04 | 1293.5ms/step | 15.5 samples/s +Step 21740/44000 | Loss: 2.9750 | LR: 2.62e-04 | 1292.4ms/step | 15.5 samples/s +Step 21750/44000 | Loss: 2.9791 | LR: 2.61e-04 | 1293.0ms/step | 15.5 samples/s +Validation at step 21750: Loss = 3.0003, Perplexity = 20.09 +Step 21760/44000 | Loss: 3.0007 | LR: 2.61e-04 | 3451.7ms/step | 5.8 samples/s +Step 21770/44000 | Loss: 2.9850 | LR: 2.61e-04 | 1288.4ms/step | 15.5 samples/s +Step 21780/44000 | Loss: 2.9855 | LR: 2.61e-04 | 1286.4ms/step | 15.5 samples/s +Step 21790/44000 | Loss: 2.9508 | LR: 2.61e-04 | 1286.5ms/step | 15.5 samples/s +Step 21800/44000 | Loss: 2.9774 | LR: 2.60e-04 | 1290.9ms/step | 15.5 samples/s +Step 21810/44000 | Loss: 2.9725 | LR: 2.60e-04 | 1290.2ms/step | 15.5 samples/s +Step 21820/44000 | Loss: 2.9940 | LR: 2.60e-04 | 1292.5ms/step | 15.5 samples/s +Step 21830/44000 | Loss: 2.9652 | LR: 2.60e-04 | 1292.9ms/step | 15.5 samples/s +Step 21840/44000 | Loss: 2.9902 | LR: 2.60e-04 | 1290.2ms/step | 15.5 samples/s +Step 21850/44000 | Loss: 2.9836 | LR: 2.60e-04 | 1289.6ms/step | 15.5 samples/s +Step 21860/44000 | Loss: 2.9881 | LR: 2.59e-04 | 1288.2ms/step | 15.5 samples/s +Step 21870/44000 | Loss: 2.9996 | LR: 2.59e-04 | 1290.2ms/step | 15.5 samples/s +Step 21880/44000 | Loss: 2.9700 | LR: 2.59e-04 | 1290.9ms/step | 15.5 samples/s +Step 21890/44000 | Loss: 2.9717 | LR: 2.59e-04 | 1292.0ms/step | 15.5 samples/s +Step 21900/44000 | Loss: 2.9789 | LR: 2.59e-04 | 1294.1ms/step | 15.5 samples/s +Step 21910/44000 | Loss: 2.9787 | LR: 2.58e-04 | 1301.5ms/step | 15.4 samples/s +Step 21920/44000 | Loss: 2.9881 | LR: 2.58e-04 | 1291.1ms/step | 15.5 samples/s +Step 21930/44000 | Loss: 3.0128 | LR: 2.58e-04 | 1291.1ms/step | 15.5 samples/s +Step 21940/44000 | Loss: 2.9557 | LR: 2.58e-04 | 1290.3ms/step | 15.5 samples/s +Step 21950/44000 | Loss: 2.9871 | LR: 2.58e-04 | 1288.2ms/step | 15.5 samples/s +Step 21960/44000 | Loss: 2.9750 | LR: 2.58e-04 | 1290.0ms/step | 15.5 samples/s +Step 21970/44000 | Loss: 2.9720 | LR: 2.57e-04 | 1291.5ms/step | 15.5 samples/s +Step 21980/44000 | Loss: 2.9916 | LR: 2.57e-04 | 1292.2ms/step | 15.5 samples/s +Step 21990/44000 | Loss: 2.9945 | LR: 2.57e-04 | 1291.3ms/step | 15.5 samples/s +Step 22000/44000 | Loss: 2.9842 | LR: 2.57e-04 | 1289.4ms/step | 15.5 samples/s +Validation at step 22000: Loss = 2.9977, Perplexity = 20.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-22000 +Step 22010/44000 | Loss: 2.9840 | LR: 2.57e-04 | 3731.3ms/step | 5.4 samples/s +Step 22020/44000 | Loss: 2.9966 | LR: 2.56e-04 | 1288.6ms/step | 15.5 samples/s +Step 22030/44000 | Loss: 2.9929 | LR: 2.56e-04 | 1290.0ms/step | 15.5 samples/s +Step 22040/44000 | Loss: 3.0069 | LR: 2.56e-04 | 1289.1ms/step | 15.5 samples/s +Step 22050/44000 | Loss: 2.9783 | LR: 2.56e-04 | 1287.4ms/step | 15.5 samples/s +Step 22060/44000 | Loss: 2.9772 | LR: 2.56e-04 | 1289.7ms/step | 15.5 samples/s +Step 22070/44000 | Loss: 2.9830 | LR: 2.56e-04 | 1289.3ms/step | 15.5 samples/s +Step 22080/44000 | Loss: 2.9737 | LR: 2.55e-04 | 1289.7ms/step | 15.5 samples/s +Step 22090/44000 | Loss: 3.0001 | LR: 2.55e-04 | 1291.2ms/step | 15.5 samples/s +Step 22100/44000 | Loss: 2.9948 | LR: 2.55e-04 | 1289.1ms/step | 15.5 samples/s +Step 22110/44000 | Loss: 2.9788 | LR: 2.55e-04 | 1300.6ms/step | 15.4 samples/s +Step 22120/44000 | Loss: 2.9613 | LR: 2.55e-04 | 1295.6ms/step | 15.4 samples/s +Step 22130/44000 | Loss: 2.9705 | LR: 2.54e-04 | 1292.4ms/step | 15.5 samples/s +Step 22140/44000 | Loss: 2.9812 | LR: 2.54e-04 | 1292.5ms/step | 15.5 samples/s +Step 22150/44000 | Loss: 2.9611 | LR: 2.54e-04 | 1291.8ms/step | 15.5 samples/s +Step 22160/44000 | Loss: 2.9812 | LR: 2.54e-04 | 1292.3ms/step | 15.5 samples/s +Step 22170/44000 | Loss: 2.9999 | LR: 2.54e-04 | 1289.7ms/step | 15.5 samples/s +Step 22180/44000 | Loss: 2.9828 | LR: 2.54e-04 | 1291.1ms/step | 15.5 samples/s +Step 22190/44000 | Loss: 2.9816 | LR: 2.53e-04 | 1290.6ms/step | 15.5 samples/s +Step 22200/44000 | Loss: 2.9803 | LR: 2.53e-04 | 1291.4ms/step | 15.5 samples/s +Step 22210/44000 | Loss: 2.9833 | LR: 2.53e-04 | 1290.0ms/step | 15.5 samples/s +Step 22220/44000 | Loss: 2.9801 | LR: 2.53e-04 | 1290.0ms/step | 15.5 samples/s +Step 22230/44000 | Loss: 2.9975 | LR: 2.53e-04 | 1286.6ms/step | 15.5 samples/s +Step 22240/44000 | Loss: 3.0114 | LR: 2.52e-04 | 1290.2ms/step | 15.5 samples/s +Step 22250/44000 | Loss: 2.9968 | LR: 2.52e-04 | 1288.8ms/step | 15.5 samples/s +Validation at step 22250: Loss = 2.9953, Perplexity = 19.99 +Step 22260/44000 | Loss: 2.9630 | LR: 2.52e-04 | 3444.4ms/step | 5.8 samples/s +Step 22270/44000 | Loss: 2.9984 | LR: 2.52e-04 | 1289.4ms/step | 15.5 samples/s +Step 22280/44000 | Loss: 2.9772 | LR: 2.52e-04 | 1289.9ms/step | 15.5 samples/s +Step 22290/44000 | Loss: 2.9775 | LR: 2.52e-04 | 1289.4ms/step | 15.5 samples/s +Step 22300/44000 | Loss: 3.0129 | LR: 2.51e-04 | 1292.0ms/step | 15.5 samples/s +Step 22310/44000 | Loss: 2.9739 | LR: 2.51e-04 | 1292.1ms/step | 15.5 samples/s +Step 22320/44000 | Loss: 2.9600 | LR: 2.51e-04 | 1290.4ms/step | 15.5 samples/s +Step 22330/44000 | Loss: 2.9610 | LR: 2.51e-04 | 1290.3ms/step | 15.5 samples/s +Step 22340/44000 | Loss: 3.0105 | LR: 2.51e-04 | 1289.1ms/step | 15.5 samples/s +Step 22350/44000 | Loss: 2.9728 | LR: 2.50e-04 | 1290.6ms/step | 15.5 samples/s +Step 22360/44000 | Loss: 2.9719 | LR: 2.50e-04 | 1288.6ms/step | 15.5 samples/s +Step 22370/44000 | Loss: 2.9778 | LR: 2.50e-04 | 1291.0ms/step | 15.5 samples/s +Step 22380/44000 | Loss: 2.9806 | LR: 2.50e-04 | 1291.2ms/step | 15.5 samples/s +Step 22390/44000 | Loss: 2.9687 | LR: 2.50e-04 | 1290.8ms/step | 15.5 samples/s +Step 22400/44000 | Loss: 2.9532 | LR: 2.50e-04 | 1291.2ms/step | 15.5 samples/s +Step 22410/44000 | Loss: 2.9703 | LR: 2.49e-04 | 1289.2ms/step | 15.5 samples/s +Step 22420/44000 | Loss: 2.9721 | LR: 2.49e-04 | 1290.0ms/step | 15.5 samples/s +Step 22430/44000 | Loss: 2.9880 | LR: 2.49e-04 | 1290.6ms/step | 15.5 samples/s +Step 22440/44000 | Loss: 2.9775 | LR: 2.49e-04 | 1290.8ms/step | 15.5 samples/s +Step 22450/44000 | Loss: 2.9931 | LR: 2.49e-04 | 1291.7ms/step | 15.5 samples/s +Step 22460/44000 | Loss: 2.9718 | LR: 2.48e-04 | 1284.5ms/step | 15.6 samples/s +Step 22470/44000 | Loss: 2.9468 | LR: 2.48e-04 | 1286.8ms/step | 15.5 samples/s +Step 22480/44000 | Loss: 2.9614 | LR: 2.48e-04 | 1286.9ms/step | 15.5 samples/s +Step 22490/44000 | Loss: 2.9680 | LR: 2.48e-04 | 1287.4ms/step | 15.5 samples/s +Step 22500/44000 | Loss: 2.9535 | LR: 2.48e-04 | 1288.8ms/step | 15.5 samples/s +Validation at step 22500: Loss = 2.9925, Perplexity = 19.94 +Step 22510/44000 | Loss: 2.9610 | LR: 2.48e-04 | 3447.0ms/step | 5.8 samples/s +Step 22520/44000 | Loss: 3.0071 | LR: 2.47e-04 | 1290.0ms/step | 15.5 samples/s +Step 22530/44000 | Loss: 2.9454 | LR: 2.47e-04 | 1288.2ms/step | 15.5 samples/s +Step 22540/44000 | Loss: 2.9802 | LR: 2.47e-04 | 1287.7ms/step | 15.5 samples/s +Step 22550/44000 | Loss: 2.9485 | LR: 2.47e-04 | 1289.3ms/step | 15.5 samples/s +Step 22560/44000 | Loss: 2.9752 | LR: 2.47e-04 | 1289.9ms/step | 15.5 samples/s +Step 22570/44000 | Loss: 2.9509 | LR: 2.46e-04 | 1287.9ms/step | 15.5 samples/s +Step 22580/44000 | Loss: 2.9896 | LR: 2.46e-04 | 1288.6ms/step | 15.5 samples/s +Step 22590/44000 | Loss: 2.9857 | LR: 2.46e-04 | 1288.7ms/step | 15.5 samples/s +Step 22600/44000 | Loss: 2.9579 | LR: 2.46e-04 | 1289.5ms/step | 15.5 samples/s +Step 22610/44000 | Loss: 2.9730 | LR: 2.46e-04 | 1288.5ms/step | 15.5 samples/s +Step 22620/44000 | Loss: 2.9909 | LR: 2.46e-04 | 1288.6ms/step | 15.5 samples/s +Step 22630/44000 | Loss: 2.9705 | LR: 2.45e-04 | 1289.5ms/step | 15.5 samples/s +Step 22640/44000 | Loss: 2.9834 | LR: 2.45e-04 | 1290.9ms/step | 15.5 samples/s +Step 22650/44000 | Loss: 2.9727 | LR: 2.45e-04 | 1289.1ms/step | 15.5 samples/s +Step 22660/44000 | Loss: 3.0075 | LR: 2.45e-04 | 1288.3ms/step | 15.5 samples/s +Step 22670/44000 | Loss: 2.9809 | LR: 2.45e-04 | 1289.5ms/step | 15.5 samples/s +Step 22680/44000 | Loss: 2.9586 | LR: 2.44e-04 | 1287.0ms/step | 15.5 samples/s +Step 22690/44000 | Loss: 2.9890 | LR: 2.44e-04 | 1286.9ms/step | 15.5 samples/s +Step 22700/44000 | Loss: 2.9922 | LR: 2.44e-04 | 1286.7ms/step | 15.5 samples/s +Step 22710/44000 | Loss: 2.9481 | LR: 2.44e-04 | 1288.0ms/step | 15.5 samples/s +Step 22720/44000 | Loss: 2.9768 | LR: 2.44e-04 | 1289.4ms/step | 15.5 samples/s +Step 22730/44000 | Loss: 2.9921 | LR: 2.44e-04 | 1290.1ms/step | 15.5 samples/s +Step 22740/44000 | Loss: 2.9786 | LR: 2.43e-04 | 1288.4ms/step | 15.5 samples/s +Step 22750/44000 | Loss: 2.9551 | LR: 2.43e-04 | 1285.4ms/step | 15.6 samples/s +Validation at step 22750: Loss = 2.9902, Perplexity = 19.89 +Step 22760/44000 | Loss: 2.9883 | LR: 2.43e-04 | 3442.9ms/step | 5.8 samples/s +Step 22770/44000 | Loss: 2.9712 | LR: 2.43e-04 | 1288.6ms/step | 15.5 samples/s +Step 22780/44000 | Loss: 2.9814 | LR: 2.43e-04 | 1289.7ms/step | 15.5 samples/s +Step 22790/44000 | Loss: 2.9953 | LR: 2.42e-04 | 1291.5ms/step | 15.5 samples/s +Step 22800/44000 | Loss: 2.9971 | LR: 2.42e-04 | 1289.6ms/step | 15.5 samples/s +Step 22810/44000 | Loss: 2.9630 | LR: 2.42e-04 | 1286.9ms/step | 15.5 samples/s +Step 22820/44000 | Loss: 2.9765 | LR: 2.42e-04 | 1291.4ms/step | 15.5 samples/s +Step 22830/44000 | Loss: 3.0034 | LR: 2.42e-04 | 1290.4ms/step | 15.5 samples/s +Step 22840/44000 | Loss: 2.9853 | LR: 2.42e-04 | 1290.1ms/step | 15.5 samples/s +Step 22850/44000 | Loss: 2.9592 | LR: 2.41e-04 | 1290.6ms/step | 15.5 samples/s +Step 22860/44000 | Loss: 2.9975 | LR: 2.41e-04 | 1289.3ms/step | 15.5 samples/s +Step 22870/44000 | Loss: 2.9691 | LR: 2.41e-04 | 1289.3ms/step | 15.5 samples/s +Step 22880/44000 | Loss: 2.9708 | LR: 2.41e-04 | 1288.3ms/step | 15.5 samples/s +Step 22890/44000 | Loss: 2.9599 | LR: 2.41e-04 | 1286.1ms/step | 15.6 samples/s +Step 22900/44000 | Loss: 3.0049 | LR: 2.40e-04 | 1288.1ms/step | 15.5 samples/s +Step 22910/44000 | Loss: 2.9894 | LR: 2.40e-04 | 1290.7ms/step | 15.5 samples/s +Step 22920/44000 | Loss: 2.9778 | LR: 2.40e-04 | 1288.8ms/step | 15.5 samples/s +Step 22930/44000 | Loss: 2.9596 | LR: 2.40e-04 | 1290.5ms/step | 15.5 samples/s +Step 22940/44000 | Loss: 2.9624 | LR: 2.40e-04 | 1290.8ms/step | 15.5 samples/s +Step 22950/44000 | Loss: 2.9694 | LR: 2.40e-04 | 1288.7ms/step | 15.5 samples/s +Step 22960/44000 | Loss: 3.0098 | LR: 2.39e-04 | 1285.6ms/step | 15.6 samples/s +Step 22970/44000 | Loss: 2.9584 | LR: 2.39e-04 | 1285.7ms/step | 15.6 samples/s +Step 22980/44000 | Loss: 2.9736 | LR: 2.39e-04 | 1291.2ms/step | 15.5 samples/s +Step 22990/44000 | Loss: 2.9628 | LR: 2.39e-04 | 1292.4ms/step | 15.5 samples/s +Step 23000/44000 | Loss: 2.9589 | LR: 2.39e-04 | 1290.5ms/step | 15.5 samples/s +Validation at step 23000: Loss = 2.9883, Perplexity = 19.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-23000 +Step 23010/44000 | Loss: 2.9634 | LR: 2.38e-04 | 3742.3ms/step | 5.3 samples/s +Step 23020/44000 | Loss: 2.9740 | LR: 2.38e-04 | 1292.1ms/step | 15.5 samples/s +Step 23030/44000 | Loss: 2.9786 | LR: 2.38e-04 | 1298.1ms/step | 15.4 samples/s +Step 23040/44000 | Loss: 2.9931 | LR: 2.38e-04 | 1294.7ms/step | 15.4 samples/s +Step 23050/44000 | Loss: 3.0124 | LR: 2.38e-04 | 1294.2ms/step | 15.5 samples/s +Step 23060/44000 | Loss: 2.9757 | LR: 2.38e-04 | 1292.4ms/step | 15.5 samples/s +Step 23070/44000 | Loss: 2.9725 | LR: 2.37e-04 | 1292.1ms/step | 15.5 samples/s +Step 23080/44000 | Loss: 2.9751 | LR: 2.37e-04 | 1291.4ms/step | 15.5 samples/s +Step 23090/44000 | Loss: 2.9857 | LR: 2.37e-04 | 1294.2ms/step | 15.5 samples/s +Step 23100/44000 | Loss: 2.9732 | LR: 2.37e-04 | 1293.1ms/step | 15.5 samples/s +Step 23110/44000 | Loss: 2.9595 | LR: 2.37e-04 | 1294.3ms/step | 15.5 samples/s +Step 23120/44000 | Loss: 2.9673 | LR: 2.36e-04 | 1292.7ms/step | 15.5 samples/s +Step 23130/44000 | Loss: 2.9682 | LR: 2.36e-04 | 1291.3ms/step | 15.5 samples/s +Step 23140/44000 | Loss: 2.9475 | LR: 2.36e-04 | 1291.3ms/step | 15.5 samples/s +Step 23150/44000 | Loss: 2.9571 | LR: 2.36e-04 | 1290.3ms/step | 15.5 samples/s +Step 23160/44000 | Loss: 2.9787 | LR: 2.36e-04 | 1291.6ms/step | 15.5 samples/s +Step 23170/44000 | Loss: 2.9649 | LR: 2.36e-04 | 1291.3ms/step | 15.5 samples/s +Step 23180/44000 | Loss: 2.9802 | LR: 2.35e-04 | 1290.6ms/step | 15.5 samples/s +Step 23190/44000 | Loss: 2.9752 | LR: 2.35e-04 | 1289.5ms/step | 15.5 samples/s +Step 23200/44000 | Loss: 2.9735 | LR: 2.35e-04 | 1289.0ms/step | 15.5 samples/s +Step 23210/44000 | Loss: 2.9588 | LR: 2.35e-04 | 1290.7ms/step | 15.5 samples/s +Step 23220/44000 | Loss: 2.9771 | LR: 2.35e-04 | 1288.8ms/step | 15.5 samples/s +Step 23230/44000 | Loss: 2.9551 | LR: 2.34e-04 | 1290.8ms/step | 15.5 samples/s +Step 23240/44000 | Loss: 2.9763 | LR: 2.34e-04 | 1290.8ms/step | 15.5 samples/s +Step 23250/44000 | Loss: 2.9783 | LR: 2.34e-04 | 1289.7ms/step | 15.5 samples/s +Validation at step 23250: Loss = 2.9860, Perplexity = 19.81 +Step 23260/44000 | Loss: 2.9502 | LR: 2.34e-04 | 3446.8ms/step | 5.8 samples/s +Step 23270/44000 | Loss: 2.9732 | LR: 2.34e-04 | 1289.4ms/step | 15.5 samples/s +Step 23280/44000 | Loss: 2.9859 | LR: 2.34e-04 | 1289.5ms/step | 15.5 samples/s +Step 23290/44000 | Loss: 2.9659 | LR: 2.33e-04 | 1288.8ms/step | 15.5 samples/s +Step 23300/44000 | Loss: 2.9576 | LR: 2.33e-04 | 1291.0ms/step | 15.5 samples/s +Step 23310/44000 | Loss: 2.9373 | LR: 2.33e-04 | 1287.7ms/step | 15.5 samples/s +Step 23320/44000 | Loss: 2.9637 | LR: 2.33e-04 | 1286.9ms/step | 15.5 samples/s +Step 23330/44000 | Loss: 2.9550 | LR: 2.33e-04 | 1288.2ms/step | 15.5 samples/s +Step 23340/44000 | Loss: 2.9813 | LR: 2.32e-04 | 1289.9ms/step | 15.5 samples/s +Step 23350/44000 | Loss: 2.9667 | LR: 2.32e-04 | 1290.5ms/step | 15.5 samples/s +Step 23360/44000 | Loss: 3.0085 | LR: 2.32e-04 | 1289.6ms/step | 15.5 samples/s +Step 23370/44000 | Loss: 2.9590 | LR: 2.32e-04 | 1291.7ms/step | 15.5 samples/s +Step 23380/44000 | Loss: 2.9528 | LR: 2.32e-04 | 1292.5ms/step | 15.5 samples/s +Step 23390/44000 | Loss: 2.9396 | LR: 2.32e-04 | 1291.4ms/step | 15.5 samples/s +Step 23400/44000 | Loss: 2.9775 | LR: 2.31e-04 | 1292.6ms/step | 15.5 samples/s +Step 23410/44000 | Loss: 2.9666 | LR: 2.31e-04 | 1290.3ms/step | 15.5 samples/s +Step 23420/44000 | Loss: 2.9874 | LR: 2.31e-04 | 1290.8ms/step | 15.5 samples/s +Step 23430/44000 | Loss: 2.9567 | LR: 2.31e-04 | 1290.5ms/step | 15.5 samples/s +Step 23440/44000 | Loss: 2.9549 | LR: 2.31e-04 | 1290.9ms/step | 15.5 samples/s +Step 23450/44000 | Loss: 2.9748 | LR: 2.30e-04 | 1291.8ms/step | 15.5 samples/s +Step 23460/44000 | Loss: 2.9597 | LR: 2.30e-04 | 1293.3ms/step | 15.5 samples/s +Step 23470/44000 | Loss: 2.9804 | LR: 2.30e-04 | 1290.0ms/step | 15.5 samples/s +Step 23480/44000 | Loss: 2.9551 | LR: 2.30e-04 | 1291.1ms/step | 15.5 samples/s +Step 23490/44000 | Loss: 2.9700 | LR: 2.30e-04 | 1290.7ms/step | 15.5 samples/s +Step 23500/44000 | Loss: 2.9572 | LR: 2.30e-04 | 1292.4ms/step | 15.5 samples/s +Validation at step 23500: Loss = 2.9839, Perplexity = 19.76 +Step 23510/44000 | Loss: 2.9733 | LR: 2.29e-04 | 3458.4ms/step | 5.8 samples/s +Step 23520/44000 | Loss: 2.9645 | LR: 2.29e-04 | 1291.8ms/step | 15.5 samples/s +Step 23530/44000 | Loss: 2.9629 | LR: 2.29e-04 | 1289.3ms/step | 15.5 samples/s +Step 23540/44000 | Loss: 2.9453 | LR: 2.29e-04 | 1289.5ms/step | 15.5 samples/s +Step 23550/44000 | Loss: 2.9468 | LR: 2.29e-04 | 1290.5ms/step | 15.5 samples/s +Step 23560/44000 | Loss: 2.9639 | LR: 2.29e-04 | 1289.9ms/step | 15.5 samples/s +Step 23570/44000 | Loss: 2.9901 | LR: 2.28e-04 | 1289.0ms/step | 15.5 samples/s +Step 23580/44000 | Loss: 2.9403 | LR: 2.28e-04 | 1288.8ms/step | 15.5 samples/s +Step 23590/44000 | Loss: 2.9826 | LR: 2.28e-04 | 1288.4ms/step | 15.5 samples/s +Step 23600/44000 | Loss: 2.9972 | LR: 2.28e-04 | 1288.6ms/step | 15.5 samples/s +Step 23610/44000 | Loss: 2.9563 | LR: 2.28e-04 | 1287.5ms/step | 15.5 samples/s +Step 23620/44000 | Loss: 2.9625 | LR: 2.27e-04 | 1290.9ms/step | 15.5 samples/s +Step 23630/44000 | Loss: 2.9673 | LR: 2.27e-04 | 1288.7ms/step | 15.5 samples/s +Step 23640/44000 | Loss: 2.9830 | LR: 2.27e-04 | 1290.4ms/step | 15.5 samples/s +Step 23650/44000 | Loss: 2.9908 | LR: 2.27e-04 | 1288.1ms/step | 15.5 samples/s +Step 23660/44000 | Loss: 2.9696 | LR: 2.27e-04 | 1290.3ms/step | 15.5 samples/s +Step 23670/44000 | Loss: 2.9780 | LR: 2.27e-04 | 1292.2ms/step | 15.5 samples/s +Step 23680/44000 | Loss: 2.9762 | LR: 2.26e-04 | 1291.2ms/step | 15.5 samples/s +Step 23690/44000 | Loss: 2.9789 | LR: 2.26e-04 | 1291.4ms/step | 15.5 samples/s +Step 23700/44000 | Loss: 2.9620 | LR: 2.26e-04 | 1292.4ms/step | 15.5 samples/s +Step 23710/44000 | Loss: 2.9475 | LR: 2.26e-04 | 1291.8ms/step | 15.5 samples/s +Step 23720/44000 | Loss: 2.9541 | LR: 2.26e-04 | 1291.6ms/step | 15.5 samples/s +Step 23730/44000 | Loss: 2.9724 | LR: 2.25e-04 | 1291.6ms/step | 15.5 samples/s +Step 23740/44000 | Loss: 2.9682 | LR: 2.25e-04 | 1290.8ms/step | 15.5 samples/s +Step 23750/44000 | Loss: 2.9544 | LR: 2.25e-04 | 1289.2ms/step | 15.5 samples/s +Validation at step 23750: Loss = 2.9814, Perplexity = 19.72 +Step 23760/44000 | Loss: 2.9813 | LR: 2.25e-04 | 3451.0ms/step | 5.8 samples/s +Step 23770/44000 | Loss: 2.9552 | LR: 2.25e-04 | 1290.3ms/step | 15.5 samples/s +Step 23780/44000 | Loss: 2.9531 | LR: 2.25e-04 | 1289.3ms/step | 15.5 samples/s +Step 23790/44000 | Loss: 2.9472 | LR: 2.24e-04 | 1288.6ms/step | 15.5 samples/s +Step 23800/44000 | Loss: 2.9725 | LR: 2.24e-04 | 1289.8ms/step | 15.5 samples/s +Step 23810/44000 | Loss: 2.9469 | LR: 2.24e-04 | 1291.1ms/step | 15.5 samples/s +Step 23820/44000 | Loss: 2.9916 | LR: 2.24e-04 | 1290.4ms/step | 15.5 samples/s +Step 23830/44000 | Loss: 2.9589 | LR: 2.24e-04 | 1289.3ms/step | 15.5 samples/s +Step 23840/44000 | Loss: 2.9717 | LR: 2.23e-04 | 1288.4ms/step | 15.5 samples/s +Step 23850/44000 | Loss: 2.9754 | LR: 2.23e-04 | 1287.2ms/step | 15.5 samples/s +Step 23860/44000 | Loss: 2.9639 | LR: 2.23e-04 | 1289.4ms/step | 15.5 samples/s +Step 23870/44000 | Loss: 2.9778 | LR: 2.23e-04 | 1290.8ms/step | 15.5 samples/s +Step 23880/44000 | Loss: 2.9603 | LR: 2.23e-04 | 1288.7ms/step | 15.5 samples/s +Step 23890/44000 | Loss: 2.9599 | LR: 2.23e-04 | 1288.4ms/step | 15.5 samples/s +Step 23900/44000 | Loss: 2.9576 | LR: 2.22e-04 | 1290.5ms/step | 15.5 samples/s +Step 23910/44000 | Loss: 2.9545 | LR: 2.22e-04 | 1289.6ms/step | 15.5 samples/s +Step 23920/44000 | Loss: 2.9981 | LR: 2.22e-04 | 1288.8ms/step | 15.5 samples/s +Step 23930/44000 | Loss: 2.9712 | LR: 2.22e-04 | 1289.1ms/step | 15.5 samples/s +Step 23940/44000 | Loss: 2.9821 | LR: 2.22e-04 | 1289.8ms/step | 15.5 samples/s +Step 23950/44000 | Loss: 2.9772 | LR: 2.21e-04 | 1289.3ms/step | 15.5 samples/s +Step 23960/44000 | Loss: 2.9991 | LR: 2.21e-04 | 1287.6ms/step | 15.5 samples/s +Step 23970/44000 | Loss: 2.9779 | LR: 2.21e-04 | 1293.1ms/step | 15.5 samples/s +Step 23980/44000 | Loss: 2.9583 | LR: 2.21e-04 | 1290.7ms/step | 15.5 samples/s +Step 23990/44000 | Loss: 2.9376 | LR: 2.21e-04 | 1287.3ms/step | 15.5 samples/s +Step 24000/44000 | Loss: 2.9594 | LR: 2.21e-04 | 1291.0ms/step | 15.5 samples/s +Validation at step 24000: Loss = 2.9799, Perplexity = 19.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-24000 +Step 24010/44000 | Loss: 2.9696 | LR: 2.20e-04 | 3741.7ms/step | 5.3 samples/s +Step 24020/44000 | Loss: 2.9646 | LR: 2.20e-04 | 1289.2ms/step | 15.5 samples/s +Step 24030/44000 | Loss: 2.9874 | LR: 2.20e-04 | 1286.7ms/step | 15.5 samples/s +Step 24040/44000 | Loss: 2.9997 | LR: 2.20e-04 | 1287.2ms/step | 15.5 samples/s +Step 24050/44000 | Loss: 2.9903 | LR: 2.20e-04 | 1291.4ms/step | 15.5 samples/s +Step 24060/44000 | Loss: 2.9753 | LR: 2.19e-04 | 1293.7ms/step | 15.5 samples/s +Step 24070/44000 | Loss: 2.9907 | LR: 2.19e-04 | 1292.1ms/step | 15.5 samples/s +Step 24080/44000 | Loss: 2.9500 | LR: 2.19e-04 | 1290.8ms/step | 15.5 samples/s +Step 24090/44000 | Loss: 2.9494 | LR: 2.19e-04 | 1293.4ms/step | 15.5 samples/s +Step 24100/44000 | Loss: 2.9682 | LR: 2.19e-04 | 1288.8ms/step | 15.5 samples/s +Step 24110/44000 | Loss: 2.9777 | LR: 2.19e-04 | 1289.3ms/step | 15.5 samples/s +Step 24120/44000 | Loss: 2.9325 | LR: 2.18e-04 | 1289.3ms/step | 15.5 samples/s +Step 24130/44000 | Loss: 2.9811 | LR: 2.18e-04 | 1291.6ms/step | 15.5 samples/s +Step 24140/44000 | Loss: 2.9564 | LR: 2.18e-04 | 1291.0ms/step | 15.5 samples/s +Step 24150/44000 | Loss: 2.9836 | LR: 2.18e-04 | 1289.5ms/step | 15.5 samples/s +Step 24160/44000 | Loss: 2.9939 | LR: 2.18e-04 | 1290.4ms/step | 15.5 samples/s +Step 24170/44000 | Loss: 2.9500 | LR: 2.17e-04 | 1290.4ms/step | 15.5 samples/s +Step 24180/44000 | Loss: 2.9586 | LR: 2.17e-04 | 1289.9ms/step | 15.5 samples/s +Step 24190/44000 | Loss: 2.9796 | LR: 2.17e-04 | 1291.0ms/step | 15.5 samples/s +Step 24200/44000 | Loss: 2.9525 | LR: 2.17e-04 | 1287.8ms/step | 15.5 samples/s +Step 24210/44000 | Loss: 2.9447 | LR: 2.17e-04 | 1288.4ms/step | 15.5 samples/s +Step 24220/44000 | Loss: 2.9635 | LR: 2.17e-04 | 1290.0ms/step | 15.5 samples/s +Step 24230/44000 | Loss: 2.9855 | LR: 2.16e-04 | 1288.4ms/step | 15.5 samples/s +Step 24240/44000 | Loss: 2.9472 | LR: 2.16e-04 | 1288.1ms/step | 15.5 samples/s +Step 24250/44000 | Loss: 2.9733 | LR: 2.16e-04 | 1289.5ms/step | 15.5 samples/s +Validation at step 24250: Loss = 2.9782, Perplexity = 19.65 +Step 24260/44000 | Loss: 2.9427 | LR: 2.16e-04 | 3449.3ms/step | 5.8 samples/s +Step 24270/44000 | Loss: 2.9618 | LR: 2.16e-04 | 1290.0ms/step | 15.5 samples/s +Step 24280/44000 | Loss: 2.9868 | LR: 2.16e-04 | 1289.9ms/step | 15.5 samples/s +Step 24290/44000 | Loss: 2.9687 | LR: 2.15e-04 | 1291.0ms/step | 15.5 samples/s +Step 24300/44000 | Loss: 2.9537 | LR: 2.15e-04 | 1290.4ms/step | 15.5 samples/s +Step 24310/44000 | Loss: 2.9671 | LR: 2.15e-04 | 1290.6ms/step | 15.5 samples/s +Step 24320/44000 | Loss: 2.9652 | LR: 2.15e-04 | 1289.6ms/step | 15.5 samples/s +Step 24330/44000 | Loss: 2.9843 | LR: 2.15e-04 | 1288.6ms/step | 15.5 samples/s +Step 24340/44000 | Loss: 2.9486 | LR: 2.14e-04 | 1289.2ms/step | 15.5 samples/s +Step 24350/44000 | Loss: 2.9397 | LR: 2.14e-04 | 1294.3ms/step | 15.5 samples/s +Step 24360/44000 | Loss: 2.9648 | LR: 2.14e-04 | 1294.0ms/step | 15.5 samples/s +Step 24370/44000 | Loss: 2.9331 | LR: 2.14e-04 | 1291.8ms/step | 15.5 samples/s +Step 24380/44000 | Loss: 2.9281 | LR: 2.14e-04 | 1290.2ms/step | 15.5 samples/s +Step 24390/44000 | Loss: 2.9346 | LR: 2.14e-04 | 1293.8ms/step | 15.5 samples/s +Step 24400/44000 | Loss: 2.9658 | LR: 2.13e-04 | 1289.3ms/step | 15.5 samples/s +Step 24410/44000 | Loss: 2.9649 | LR: 2.13e-04 | 1289.6ms/step | 15.5 samples/s +Step 24420/44000 | Loss: 2.9792 | LR: 2.13e-04 | 1290.5ms/step | 15.5 samples/s +Step 24430/44000 | Loss: 2.9642 | LR: 2.13e-04 | 1288.9ms/step | 15.5 samples/s +Step 24440/44000 | Loss: 2.9682 | LR: 2.13e-04 | 1291.2ms/step | 15.5 samples/s +Step 24450/44000 | Loss: 2.9365 | LR: 2.12e-04 | 1295.6ms/step | 15.4 samples/s +Step 24460/44000 | Loss: 2.9427 | LR: 2.12e-04 | 1291.2ms/step | 15.5 samples/s +Step 24470/44000 | Loss: 2.9688 | LR: 2.12e-04 | 1293.4ms/step | 15.5 samples/s +Step 24480/44000 | Loss: 2.9635 | LR: 2.12e-04 | 1292.3ms/step | 15.5 samples/s +Step 24490/44000 | Loss: 2.9590 | LR: 2.12e-04 | 1291.6ms/step | 15.5 samples/s +Step 24500/44000 | Loss: 2.9749 | LR: 2.12e-04 | 1290.7ms/step | 15.5 samples/s +Validation at step 24500: Loss = 2.9760, Perplexity = 19.61 +Step 24510/44000 | Loss: 2.9648 | LR: 2.11e-04 | 3446.1ms/step | 5.8 samples/s +Step 24520/44000 | Loss: 2.9328 | LR: 2.11e-04 | 1290.4ms/step | 15.5 samples/s +Step 24530/44000 | Loss: 2.9583 | LR: 2.11e-04 | 1290.8ms/step | 15.5 samples/s +Step 24540/44000 | Loss: 2.9780 | LR: 2.11e-04 | 1289.9ms/step | 15.5 samples/s +Step 24550/44000 | Loss: 2.9920 | LR: 2.11e-04 | 1291.0ms/step | 15.5 samples/s +Step 24560/44000 | Loss: 2.9587 | LR: 2.10e-04 | 1290.6ms/step | 15.5 samples/s +Step 24570/44000 | Loss: 3.0256 | LR: 2.10e-04 | 1292.2ms/step | 15.5 samples/s +Step 24580/44000 | Loss: 2.9411 | LR: 2.10e-04 | 1290.9ms/step | 15.5 samples/s +Step 24590/44000 | Loss: 2.9881 | LR: 2.10e-04 | 1287.2ms/step | 15.5 samples/s +Step 24600/44000 | Loss: 2.9604 | LR: 2.10e-04 | 1283.1ms/step | 15.6 samples/s +Step 24610/44000 | Loss: 2.9760 | LR: 2.10e-04 | 1287.9ms/step | 15.5 samples/s +Step 24620/44000 | Loss: 2.9706 | LR: 2.09e-04 | 1289.0ms/step | 15.5 samples/s +Step 24630/44000 | Loss: 2.9946 | LR: 2.09e-04 | 1290.1ms/step | 15.5 samples/s +Step 24640/44000 | Loss: 2.9639 | LR: 2.09e-04 | 1288.3ms/step | 15.5 samples/s +Step 24650/44000 | Loss: 2.9644 | LR: 2.09e-04 | 1287.9ms/step | 15.5 samples/s +Step 24660/44000 | Loss: 2.9542 | LR: 2.09e-04 | 1289.4ms/step | 15.5 samples/s +Step 24670/44000 | Loss: 2.9558 | LR: 2.09e-04 | 1288.3ms/step | 15.5 samples/s +Step 24680/44000 | Loss: 2.9524 | LR: 2.08e-04 | 1289.8ms/step | 15.5 samples/s +Step 24690/44000 | Loss: 2.9580 | LR: 2.08e-04 | 1289.4ms/step | 15.5 samples/s +Step 24700/44000 | Loss: 2.9530 | LR: 2.08e-04 | 1290.4ms/step | 15.5 samples/s +Step 24710/44000 | Loss: 2.9562 | LR: 2.08e-04 | 1290.0ms/step | 15.5 samples/s +Step 24720/44000 | Loss: 2.9439 | LR: 2.08e-04 | 1291.1ms/step | 15.5 samples/s +Step 24730/44000 | Loss: 2.9543 | LR: 2.07e-04 | 1290.8ms/step | 15.5 samples/s +Step 24740/44000 | Loss: 3.0030 | LR: 2.07e-04 | 1290.0ms/step | 15.5 samples/s +Step 24750/44000 | Loss: 2.9378 | LR: 2.07e-04 | 1290.4ms/step | 15.5 samples/s +Validation at step 24750: Loss = 2.9744, Perplexity = 19.58 +Step 24760/44000 | Loss: 2.9597 | LR: 2.07e-04 | 3444.6ms/step | 5.8 samples/s +Step 24770/44000 | Loss: 2.9606 | LR: 2.07e-04 | 1292.2ms/step | 15.5 samples/s +Step 24780/44000 | Loss: 2.9831 | LR: 2.07e-04 | 1293.3ms/step | 15.5 samples/s +Step 24790/44000 | Loss: 2.9394 | LR: 2.06e-04 | 1290.6ms/step | 15.5 samples/s +Step 24800/44000 | Loss: 2.9440 | LR: 2.06e-04 | 1292.5ms/step | 15.5 samples/s +Step 24810/44000 | Loss: 2.9757 | LR: 2.06e-04 | 1289.0ms/step | 15.5 samples/s +Step 24820/44000 | Loss: 2.9589 | LR: 2.06e-04 | 1290.6ms/step | 15.5 samples/s +Step 24830/44000 | Loss: 2.9351 | LR: 2.06e-04 | 1291.1ms/step | 15.5 samples/s +Step 24840/44000 | Loss: 2.9513 | LR: 2.05e-04 | 1293.0ms/step | 15.5 samples/s +Step 24850/44000 | Loss: 2.9554 | LR: 2.05e-04 | 1294.2ms/step | 15.5 samples/s +Step 24860/44000 | Loss: 2.9556 | LR: 2.05e-04 | 1289.8ms/step | 15.5 samples/s +Step 24870/44000 | Loss: 2.9454 | LR: 2.05e-04 | 1296.0ms/step | 15.4 samples/s +Step 24880/44000 | Loss: 2.9707 | LR: 2.05e-04 | 1291.8ms/step | 15.5 samples/s +Step 24890/44000 | Loss: 2.9957 | LR: 2.05e-04 | 1290.6ms/step | 15.5 samples/s +Step 24900/44000 | Loss: 2.9511 | LR: 2.04e-04 | 1290.2ms/step | 15.5 samples/s +Step 24910/44000 | Loss: 2.9603 | LR: 2.04e-04 | 1290.2ms/step | 15.5 samples/s +Step 24920/44000 | Loss: 2.9674 | LR: 2.04e-04 | 1290.5ms/step | 15.5 samples/s +Step 24930/44000 | Loss: 2.9612 | LR: 2.04e-04 | 1290.0ms/step | 15.5 samples/s +Step 24940/44000 | Loss: 2.9704 | LR: 2.04e-04 | 1290.6ms/step | 15.5 samples/s +Step 24950/44000 | Loss: 2.9475 | LR: 2.04e-04 | 1290.4ms/step | 15.5 samples/s +Step 24960/44000 | Loss: 2.9961 | LR: 2.03e-04 | 1290.8ms/step | 15.5 samples/s +Step 24970/44000 | Loss: 2.9479 | LR: 2.03e-04 | 1292.4ms/step | 15.5 samples/s +Step 24980/44000 | Loss: 2.9811 | LR: 2.03e-04 | 1290.4ms/step | 15.5 samples/s +Step 24990/44000 | Loss: 2.9331 | LR: 2.03e-04 | 1289.0ms/step | 15.5 samples/s +Step 25000/44000 | Loss: 2.9765 | LR: 2.03e-04 | 1288.0ms/step | 15.5 samples/s +Validation at step 25000: Loss = 2.9719, Perplexity = 19.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-25000 +Step 25010/44000 | Loss: 2.9389 | LR: 2.02e-04 | 3745.1ms/step | 5.3 samples/s +Step 25020/44000 | Loss: 2.9688 | LR: 2.02e-04 | 1289.8ms/step | 15.5 samples/s +Step 25030/44000 | Loss: 2.9489 | LR: 2.02e-04 | 1288.7ms/step | 15.5 samples/s +Step 25040/44000 | Loss: 2.9629 | LR: 2.02e-04 | 1290.0ms/step | 15.5 samples/s +Step 25050/44000 | Loss: 2.9548 | LR: 2.02e-04 | 1291.1ms/step | 15.5 samples/s +Step 25060/44000 | Loss: 2.9519 | LR: 2.02e-04 | 1289.7ms/step | 15.5 samples/s +Step 25070/44000 | Loss: 2.9759 | LR: 2.01e-04 | 1291.7ms/step | 15.5 samples/s +Step 25080/44000 | Loss: 2.9389 | LR: 2.01e-04 | 1290.7ms/step | 15.5 samples/s +Step 25090/44000 | Loss: 2.9768 | LR: 2.01e-04 | 1292.0ms/step | 15.5 samples/s +Step 25100/44000 | Loss: 2.9376 | LR: 2.01e-04 | 1291.9ms/step | 15.5 samples/s +Step 25110/44000 | Loss: 2.9625 | LR: 2.01e-04 | 1290.6ms/step | 15.5 samples/s +Step 25120/44000 | Loss: 2.9348 | LR: 2.00e-04 | 1291.1ms/step | 15.5 samples/s +Step 25130/44000 | Loss: 2.9901 | LR: 2.00e-04 | 1291.5ms/step | 15.5 samples/s +Step 25140/44000 | Loss: 2.9983 | LR: 2.00e-04 | 1293.3ms/step | 15.5 samples/s +Step 25150/44000 | Loss: 2.9209 | LR: 2.00e-04 | 1294.8ms/step | 15.4 samples/s +Step 25160/44000 | Loss: 2.9360 | LR: 2.00e-04 | 1294.5ms/step | 15.4 samples/s +Step 25170/44000 | Loss: 2.9694 | LR: 2.00e-04 | 1293.1ms/step | 15.5 samples/s +Step 25180/44000 | Loss: 2.9101 | LR: 1.99e-04 | 1286.6ms/step | 15.5 samples/s +Step 25190/44000 | Loss: 2.9534 | LR: 1.99e-04 | 1286.2ms/step | 15.6 samples/s +Step 25200/44000 | Loss: 2.9746 | LR: 1.99e-04 | 1286.2ms/step | 15.6 samples/s +Step 25210/44000 | Loss: 2.9692 | LR: 1.99e-04 | 1290.0ms/step | 15.5 samples/s +Step 25220/44000 | Loss: 2.9393 | LR: 1.99e-04 | 1291.1ms/step | 15.5 samples/s +Step 25230/44000 | Loss: 2.9599 | LR: 1.99e-04 | 1290.2ms/step | 15.5 samples/s +Step 25240/44000 | Loss: 2.9413 | LR: 1.98e-04 | 1288.0ms/step | 15.5 samples/s +Step 25250/44000 | Loss: 2.9733 | LR: 1.98e-04 | 1286.7ms/step | 15.5 samples/s +Validation at step 25250: Loss = 2.9700, Perplexity = 19.49 +Step 25260/44000 | Loss: 2.9341 | LR: 1.98e-04 | 3442.0ms/step | 5.8 samples/s +Step 25270/44000 | Loss: 2.9645 | LR: 1.98e-04 | 1289.9ms/step | 15.5 samples/s +Step 25280/44000 | Loss: 2.9315 | LR: 1.98e-04 | 1290.5ms/step | 15.5 samples/s +Step 25290/44000 | Loss: 2.9519 | LR: 1.97e-04 | 1289.5ms/step | 15.5 samples/s +Step 25300/44000 | Loss: 2.9579 | LR: 1.97e-04 | 1289.3ms/step | 15.5 samples/s +Step 25310/44000 | Loss: 2.9073 | LR: 1.97e-04 | 1290.1ms/step | 15.5 samples/s +Step 25320/44000 | Loss: 2.9484 | LR: 1.97e-04 | 1288.9ms/step | 15.5 samples/s +Step 25330/44000 | Loss: 2.9583 | LR: 1.97e-04 | 1289.0ms/step | 15.5 samples/s +Step 25340/44000 | Loss: 2.9517 | LR: 1.97e-04 | 1287.6ms/step | 15.5 samples/s +Step 25350/44000 | Loss: 2.9437 | LR: 1.96e-04 | 1288.2ms/step | 15.5 samples/s +Step 25360/44000 | Loss: 2.9902 | LR: 1.96e-04 | 1289.9ms/step | 15.5 samples/s +Step 25370/44000 | Loss: 2.9386 | LR: 1.96e-04 | 1287.3ms/step | 15.5 samples/s +Step 25380/44000 | Loss: 2.9796 | LR: 1.96e-04 | 1288.6ms/step | 15.5 samples/s +Step 25390/44000 | Loss: 2.9509 | LR: 1.96e-04 | 1289.0ms/step | 15.5 samples/s +Step 25400/44000 | Loss: 2.9642 | LR: 1.96e-04 | 1288.1ms/step | 15.5 samples/s +Step 25410/44000 | Loss: 2.9941 | LR: 1.95e-04 | 1287.9ms/step | 15.5 samples/s +Step 25420/44000 | Loss: 2.9776 | LR: 1.95e-04 | 1290.3ms/step | 15.5 samples/s +Step 25430/44000 | Loss: 2.9567 | LR: 1.95e-04 | 1290.0ms/step | 15.5 samples/s +Step 25440/44000 | Loss: 2.9569 | LR: 1.95e-04 | 1288.8ms/step | 15.5 samples/s +Step 25450/44000 | Loss: 2.9923 | LR: 1.95e-04 | 1289.7ms/step | 15.5 samples/s +Step 25460/44000 | Loss: 2.9644 | LR: 1.94e-04 | 1290.3ms/step | 15.5 samples/s +Step 25470/44000 | Loss: 2.9442 | LR: 1.94e-04 | 1290.2ms/step | 15.5 samples/s +Step 25480/44000 | Loss: 2.9595 | LR: 1.94e-04 | 1291.2ms/step | 15.5 samples/s +Step 25490/44000 | Loss: 2.9595 | LR: 1.94e-04 | 1292.5ms/step | 15.5 samples/s +Step 25500/44000 | Loss: 2.9497 | LR: 1.94e-04 | 1293.2ms/step | 15.5 samples/s +Validation at step 25500: Loss = 2.9684, Perplexity = 19.46 +Step 25510/44000 | Loss: 2.9373 | LR: 1.94e-04 | 3447.5ms/step | 5.8 samples/s +Step 25520/44000 | Loss: 2.9377 | LR: 1.93e-04 | 1287.5ms/step | 15.5 samples/s +Step 25530/44000 | Loss: 2.9392 | LR: 1.93e-04 | 1289.0ms/step | 15.5 samples/s +Step 25540/44000 | Loss: 2.9467 | LR: 1.93e-04 | 1288.8ms/step | 15.5 samples/s +Step 25550/44000 | Loss: 2.9581 | LR: 1.93e-04 | 1289.5ms/step | 15.5 samples/s +Step 25560/44000 | Loss: 2.9664 | LR: 1.93e-04 | 1290.1ms/step | 15.5 samples/s +Step 25570/44000 | Loss: 2.9485 | LR: 1.92e-04 | 1291.2ms/step | 15.5 samples/s +Step 25580/44000 | Loss: 2.9827 | LR: 1.92e-04 | 1290.1ms/step | 15.5 samples/s +Step 25590/44000 | Loss: 2.9586 | LR: 1.92e-04 | 1290.8ms/step | 15.5 samples/s +Step 25600/44000 | Loss: 2.9272 | LR: 1.92e-04 | 1289.8ms/step | 15.5 samples/s +Step 25610/44000 | Loss: 2.9429 | LR: 1.92e-04 | 1289.3ms/step | 15.5 samples/s +Step 25620/44000 | Loss: 2.9309 | LR: 1.92e-04 | 1287.3ms/step | 15.5 samples/s +Step 25630/44000 | Loss: 2.9618 | LR: 1.91e-04 | 1289.5ms/step | 15.5 samples/s +Step 25640/44000 | Loss: 2.9456 | LR: 1.91e-04 | 1288.3ms/step | 15.5 samples/s +Step 25650/44000 | Loss: 2.9492 | LR: 1.91e-04 | 1289.1ms/step | 15.5 samples/s +Step 25660/44000 | Loss: 2.9711 | LR: 1.91e-04 | 1295.5ms/step | 15.4 samples/s +Step 25670/44000 | Loss: 2.9620 | LR: 1.91e-04 | 1289.0ms/step | 15.5 samples/s +Step 25680/44000 | Loss: 2.9281 | LR: 1.91e-04 | 1288.7ms/step | 15.5 samples/s +Step 25690/44000 | Loss: 2.9483 | LR: 1.90e-04 | 1290.7ms/step | 15.5 samples/s +Step 25700/44000 | Loss: 2.9314 | LR: 1.90e-04 | 1292.3ms/step | 15.5 samples/s +Step 25710/44000 | Loss: 2.9466 | LR: 1.90e-04 | 1290.2ms/step | 15.5 samples/s +Step 25720/44000 | Loss: 2.9401 | LR: 1.90e-04 | 1296.2ms/step | 15.4 samples/s +Step 25730/44000 | Loss: 2.9336 | LR: 1.90e-04 | 1294.9ms/step | 15.4 samples/s +Step 25740/44000 | Loss: 2.9477 | LR: 1.89e-04 | 1292.1ms/step | 15.5 samples/s +Step 25750/44000 | Loss: 2.9475 | LR: 1.89e-04 | 1292.3ms/step | 15.5 samples/s +Validation at step 25750: Loss = 2.9669, Perplexity = 19.43 +Step 25760/44000 | Loss: 2.9470 | LR: 1.89e-04 | 3472.7ms/step | 5.8 samples/s +Step 25770/44000 | Loss: 2.9711 | LR: 1.89e-04 | 1292.9ms/step | 15.5 samples/s +Step 25780/44000 | Loss: 2.9557 | LR: 1.89e-04 | 1293.1ms/step | 15.5 samples/s +Step 25790/44000 | Loss: 2.9916 | LR: 1.89e-04 | 1295.1ms/step | 15.4 samples/s +Step 25800/44000 | Loss: 2.9147 | LR: 1.88e-04 | 1302.6ms/step | 15.4 samples/s +Step 25810/44000 | Loss: 2.9487 | LR: 1.88e-04 | 1296.0ms/step | 15.4 samples/s +Step 25820/44000 | Loss: 2.9509 | LR: 1.88e-04 | 1292.1ms/step | 15.5 samples/s +Step 25830/44000 | Loss: 2.9527 | LR: 1.88e-04 | 1292.3ms/step | 15.5 samples/s +Step 25840/44000 | Loss: 2.9624 | LR: 1.88e-04 | 1291.0ms/step | 15.5 samples/s +Step 25850/44000 | Loss: 2.9273 | LR: 1.88e-04 | 1291.4ms/step | 15.5 samples/s +Step 25860/44000 | Loss: 2.9235 | LR: 1.87e-04 | 1292.9ms/step | 15.5 samples/s +Step 25870/44000 | Loss: 2.9274 | LR: 1.87e-04 | 1290.6ms/step | 15.5 samples/s +Step 25880/44000 | Loss: 2.9717 | LR: 1.87e-04 | 1293.6ms/step | 15.5 samples/s +Step 25890/44000 | Loss: 2.9515 | LR: 1.87e-04 | 1290.9ms/step | 15.5 samples/s +Step 25900/44000 | Loss: 2.9539 | LR: 1.87e-04 | 1289.9ms/step | 15.5 samples/s +Step 25910/44000 | Loss: 2.9395 | LR: 1.87e-04 | 1289.3ms/step | 15.5 samples/s +Step 25920/44000 | Loss: 2.9376 | LR: 1.86e-04 | 1289.8ms/step | 15.5 samples/s +Step 25930/44000 | Loss: 2.9677 | LR: 1.86e-04 | 1292.5ms/step | 15.5 samples/s +Step 25940/44000 | Loss: 2.9267 | LR: 1.86e-04 | 1290.2ms/step | 15.5 samples/s +Step 25950/44000 | Loss: 2.9284 | LR: 1.86e-04 | 1293.5ms/step | 15.5 samples/s +Step 25960/44000 | Loss: 2.9460 | LR: 1.86e-04 | 1293.3ms/step | 15.5 samples/s +Step 25970/44000 | Loss: 2.9229 | LR: 1.85e-04 | 1291.6ms/step | 15.5 samples/s +Step 25980/44000 | Loss: 2.9673 | LR: 1.85e-04 | 1290.4ms/step | 15.5 samples/s +Step 25990/44000 | Loss: 2.9606 | LR: 1.85e-04 | 1291.5ms/step | 15.5 samples/s +Step 26000/44000 | Loss: 2.9385 | LR: 1.85e-04 | 1289.4ms/step | 15.5 samples/s +Validation at step 26000: Loss = 2.9654, Perplexity = 19.40 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-26000 +Step 26010/44000 | Loss: 2.9538 | LR: 1.85e-04 | 3730.4ms/step | 5.4 samples/s +Step 26020/44000 | Loss: 2.9534 | LR: 1.85e-04 | 1287.8ms/step | 15.5 samples/s +Step 26030/44000 | Loss: 2.9393 | LR: 1.84e-04 | 1289.0ms/step | 15.5 samples/s +Step 26040/44000 | Loss: 2.9260 | LR: 1.84e-04 | 1286.1ms/step | 15.6 samples/s +Step 26050/44000 | Loss: 2.9624 | LR: 1.84e-04 | 1288.5ms/step | 15.5 samples/s +Step 26060/44000 | Loss: 2.9118 | LR: 1.84e-04 | 1290.3ms/step | 15.5 samples/s +Step 26070/44000 | Loss: 2.9433 | LR: 1.84e-04 | 1290.0ms/step | 15.5 samples/s +Step 26080/44000 | Loss: 2.9279 | LR: 1.84e-04 | 1289.4ms/step | 15.5 samples/s +Step 26090/44000 | Loss: 2.9419 | LR: 1.83e-04 | 1288.1ms/step | 15.5 samples/s +Step 26100/44000 | Loss: 2.9438 | LR: 1.83e-04 | 1289.8ms/step | 15.5 samples/s +Step 26110/44000 | Loss: 2.9598 | LR: 1.83e-04 | 1289.4ms/step | 15.5 samples/s +Step 26120/44000 | Loss: 2.9422 | LR: 1.83e-04 | 1285.9ms/step | 15.6 samples/s +Step 26130/44000 | Loss: 2.9620 | LR: 1.83e-04 | 1288.2ms/step | 15.5 samples/s +Step 26140/44000 | Loss: 2.9674 | LR: 1.82e-04 | 1290.2ms/step | 15.5 samples/s +Step 26150/44000 | Loss: 2.9577 | LR: 1.82e-04 | 1289.2ms/step | 15.5 samples/s +Step 26160/44000 | Loss: 2.9416 | LR: 1.82e-04 | 1289.8ms/step | 15.5 samples/s +Step 26170/44000 | Loss: 2.9446 | LR: 1.82e-04 | 1290.7ms/step | 15.5 samples/s +Step 26180/44000 | Loss: 2.9585 | LR: 1.82e-04 | 1294.5ms/step | 15.5 samples/s +Step 26190/44000 | Loss: 2.9323 | LR: 1.82e-04 | 1294.0ms/step | 15.5 samples/s +Step 26200/44000 | Loss: 2.9121 | LR: 1.81e-04 | 1292.6ms/step | 15.5 samples/s +Step 26210/44000 | Loss: 2.9768 | LR: 1.81e-04 | 1292.5ms/step | 15.5 samples/s +Step 26220/44000 | Loss: 2.9502 | LR: 1.81e-04 | 1290.9ms/step | 15.5 samples/s +Step 26230/44000 | Loss: 2.9584 | LR: 1.81e-04 | 1291.5ms/step | 15.5 samples/s +Step 26240/44000 | Loss: 2.9512 | LR: 1.81e-04 | 1292.9ms/step | 15.5 samples/s +Step 26250/44000 | Loss: 2.9595 | LR: 1.81e-04 | 1290.8ms/step | 15.5 samples/s +Validation at step 26250: Loss = 2.9641, Perplexity = 19.38 +Step 26260/44000 | Loss: 2.9426 | LR: 1.80e-04 | 3445.6ms/step | 5.8 samples/s +Step 26270/44000 | Loss: 2.9395 | LR: 1.80e-04 | 1289.2ms/step | 15.5 samples/s +Step 26280/44000 | Loss: 2.9591 | LR: 1.80e-04 | 1291.6ms/step | 15.5 samples/s +Step 26290/44000 | Loss: 2.9737 | LR: 1.80e-04 | 1290.1ms/step | 15.5 samples/s +Step 26300/44000 | Loss: 2.9545 | LR: 1.80e-04 | 1292.0ms/step | 15.5 samples/s +Step 26310/44000 | Loss: 2.9403 | LR: 1.80e-04 | 1290.5ms/step | 15.5 samples/s +Step 26320/44000 | Loss: 2.9591 | LR: 1.79e-04 | 1289.7ms/step | 15.5 samples/s +Step 26330/44000 | Loss: 2.9894 | LR: 1.79e-04 | 1291.3ms/step | 15.5 samples/s +Step 26340/44000 | Loss: 2.9443 | LR: 1.79e-04 | 1289.6ms/step | 15.5 samples/s +Step 26350/44000 | Loss: 2.9606 | LR: 1.79e-04 | 1287.3ms/step | 15.5 samples/s +Step 26360/44000 | Loss: 2.9359 | LR: 1.79e-04 | 1287.7ms/step | 15.5 samples/s +Step 26370/44000 | Loss: 2.9623 | LR: 1.78e-04 | 1288.7ms/step | 15.5 samples/s +Step 26380/44000 | Loss: 2.9647 | LR: 1.78e-04 | 1289.1ms/step | 15.5 samples/s +Step 26390/44000 | Loss: 2.9656 | LR: 1.78e-04 | 1288.2ms/step | 15.5 samples/s +Step 26400/44000 | Loss: 2.9667 | LR: 1.78e-04 | 1291.2ms/step | 15.5 samples/s +Step 26410/44000 | Loss: 2.9302 | LR: 1.78e-04 | 1290.4ms/step | 15.5 samples/s +Step 26420/44000 | Loss: 2.9255 | LR: 1.78e-04 | 1289.1ms/step | 15.5 samples/s +Step 26430/44000 | Loss: 2.9671 | LR: 1.77e-04 | 1288.7ms/step | 15.5 samples/s +Step 26440/44000 | Loss: 2.9242 | LR: 1.77e-04 | 1288.9ms/step | 15.5 samples/s +Step 26450/44000 | Loss: 2.9482 | LR: 1.77e-04 | 1290.2ms/step | 15.5 samples/s +Step 26460/44000 | Loss: 2.9514 | LR: 1.77e-04 | 1287.6ms/step | 15.5 samples/s +Step 26470/44000 | Loss: 2.9427 | LR: 1.77e-04 | 1291.8ms/step | 15.5 samples/s +Step 26480/44000 | Loss: 2.9415 | LR: 1.77e-04 | 1291.8ms/step | 15.5 samples/s +Step 26490/44000 | Loss: 2.9796 | LR: 1.76e-04 | 1288.9ms/step | 15.5 samples/s +Step 26500/44000 | Loss: 2.9612 | LR: 1.76e-04 | 1288.1ms/step | 15.5 samples/s +Validation at step 26500: Loss = 2.9626, Perplexity = 19.35 +Step 26510/44000 | Loss: 2.9715 | LR: 1.76e-04 | 3444.5ms/step | 5.8 samples/s +Step 26520/44000 | Loss: 2.9488 | LR: 1.76e-04 | 1288.1ms/step | 15.5 samples/s +Step 26530/44000 | Loss: 2.9508 | LR: 1.76e-04 | 1287.7ms/step | 15.5 samples/s +Step 26540/44000 | Loss: 2.9588 | LR: 1.76e-04 | 1287.6ms/step | 15.5 samples/s +Step 26550/44000 | Loss: 2.9646 | LR: 1.75e-04 | 1290.0ms/step | 15.5 samples/s +Step 26560/44000 | Loss: 2.9952 | LR: 1.75e-04 | 1293.7ms/step | 15.5 samples/s +Step 26570/44000 | Loss: 2.9219 | LR: 1.75e-04 | 1292.9ms/step | 15.5 samples/s +Step 26580/44000 | Loss: 2.9738 | LR: 1.75e-04 | 1290.1ms/step | 15.5 samples/s +Step 26590/44000 | Loss: 2.9773 | LR: 1.75e-04 | 1288.3ms/step | 15.5 samples/s +Step 26600/44000 | Loss: 2.9474 | LR: 1.74e-04 | 1286.3ms/step | 15.5 samples/s +Step 26610/44000 | Loss: 2.9544 | LR: 1.74e-04 | 1286.9ms/step | 15.5 samples/s +Step 26620/44000 | Loss: 2.9343 | LR: 1.74e-04 | 1286.3ms/step | 15.5 samples/s +Step 26630/44000 | Loss: 2.9424 | LR: 1.74e-04 | 1290.3ms/step | 15.5 samples/s +Step 26640/44000 | Loss: 2.9381 | LR: 1.74e-04 | 1289.0ms/step | 15.5 samples/s +Step 26650/44000 | Loss: 2.9686 | LR: 1.74e-04 | 1289.3ms/step | 15.5 samples/s +Step 26660/44000 | Loss: 2.9567 | LR: 1.73e-04 | 1285.9ms/step | 15.6 samples/s +Step 26670/44000 | Loss: 2.9450 | LR: 1.73e-04 | 1288.5ms/step | 15.5 samples/s +Step 26680/44000 | Loss: 2.9601 | LR: 1.73e-04 | 1288.1ms/step | 15.5 samples/s +Step 26690/44000 | Loss: 2.9683 | LR: 1.73e-04 | 1290.0ms/step | 15.5 samples/s +Step 26700/44000 | Loss: 2.9631 | LR: 1.73e-04 | 1290.9ms/step | 15.5 samples/s +Step 26710/44000 | Loss: 2.9326 | LR: 1.73e-04 | 1291.2ms/step | 15.5 samples/s +Step 26720/44000 | Loss: 2.9408 | LR: 1.72e-04 | 1290.9ms/step | 15.5 samples/s +Step 26730/44000 | Loss: 2.9290 | LR: 1.72e-04 | 1286.9ms/step | 15.5 samples/s +Step 26740/44000 | Loss: 2.9512 | LR: 1.72e-04 | 1286.9ms/step | 15.5 samples/s +Step 26750/44000 | Loss: 2.9895 | LR: 1.72e-04 | 1286.7ms/step | 15.5 samples/s +Validation at step 26750: Loss = 2.9611, Perplexity = 19.32 +Step 26760/44000 | Loss: 2.9414 | LR: 1.72e-04 | 3442.9ms/step | 5.8 samples/s +Step 26770/44000 | Loss: 2.9180 | LR: 1.72e-04 | 1288.5ms/step | 15.5 samples/s +Step 26780/44000 | Loss: 2.9312 | LR: 1.71e-04 | 1290.6ms/step | 15.5 samples/s +Step 26790/44000 | Loss: 2.9442 | LR: 1.71e-04 | 1291.9ms/step | 15.5 samples/s +Step 26800/44000 | Loss: 2.9542 | LR: 1.71e-04 | 1289.3ms/step | 15.5 samples/s +Step 26810/44000 | Loss: 2.9556 | LR: 1.71e-04 | 1289.9ms/step | 15.5 samples/s +Step 26820/44000 | Loss: 2.9718 | LR: 1.71e-04 | 1290.6ms/step | 15.5 samples/s +Step 26830/44000 | Loss: 2.9482 | LR: 1.71e-04 | 1290.5ms/step | 15.5 samples/s +Step 26840/44000 | Loss: 2.9685 | LR: 1.70e-04 | 1290.3ms/step | 15.5 samples/s +Step 26850/44000 | Loss: 2.9468 | LR: 1.70e-04 | 1293.9ms/step | 15.5 samples/s +Step 26860/44000 | Loss: 2.9315 | LR: 1.70e-04 | 1290.7ms/step | 15.5 samples/s +Step 26870/44000 | Loss: 2.9594 | LR: 1.70e-04 | 1292.7ms/step | 15.5 samples/s +Step 26880/44000 | Loss: 2.9224 | LR: 1.70e-04 | 1292.0ms/step | 15.5 samples/s +Step 26890/44000 | Loss: 2.9705 | LR: 1.69e-04 | 1291.7ms/step | 15.5 samples/s +Step 26900/44000 | Loss: 2.9190 | LR: 1.69e-04 | 1290.1ms/step | 15.5 samples/s +Step 26910/44000 | Loss: 2.9623 | LR: 1.69e-04 | 1290.0ms/step | 15.5 samples/s +Step 26920/44000 | Loss: 2.9368 | LR: 1.69e-04 | 1289.8ms/step | 15.5 samples/s +Step 26930/44000 | Loss: 2.9376 | LR: 1.69e-04 | 1290.0ms/step | 15.5 samples/s +Step 26940/44000 | Loss: 2.9515 | LR: 1.69e-04 | 1289.9ms/step | 15.5 samples/s +Step 26950/44000 | Loss: 2.9540 | LR: 1.68e-04 | 1292.0ms/step | 15.5 samples/s +Step 26960/44000 | Loss: 2.9365 | LR: 1.68e-04 | 1287.7ms/step | 15.5 samples/s +Step 26970/44000 | Loss: 2.9406 | LR: 1.68e-04 | 1287.3ms/step | 15.5 samples/s +Step 26980/44000 | Loss: 2.9720 | LR: 1.68e-04 | 1286.8ms/step | 15.5 samples/s +Step 26990/44000 | Loss: 2.9335 | LR: 1.68e-04 | 1287.8ms/step | 15.5 samples/s +Step 27000/44000 | Loss: 2.9468 | LR: 1.68e-04 | 1287.5ms/step | 15.5 samples/s +Validation at step 27000: Loss = 2.9597, Perplexity = 19.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-27000 +Step 27010/44000 | Loss: 2.9624 | LR: 1.67e-04 | 3741.0ms/step | 5.3 samples/s +Step 27020/44000 | Loss: 2.9433 | LR: 1.67e-04 | 1290.3ms/step | 15.5 samples/s +Step 27030/44000 | Loss: 2.9616 | LR: 1.67e-04 | 1290.7ms/step | 15.5 samples/s +Step 27040/44000 | Loss: 2.9504 | LR: 1.67e-04 | 1291.9ms/step | 15.5 samples/s +Step 27050/44000 | Loss: 2.9596 | LR: 1.67e-04 | 1289.3ms/step | 15.5 samples/s +Step 27060/44000 | Loss: 2.9187 | LR: 1.67e-04 | 1291.3ms/step | 15.5 samples/s +Step 27070/44000 | Loss: 2.9539 | LR: 1.66e-04 | 1295.1ms/step | 15.4 samples/s +Step 27080/44000 | Loss: 2.9567 | LR: 1.66e-04 | 1293.3ms/step | 15.5 samples/s +Step 27090/44000 | Loss: 2.9426 | LR: 1.66e-04 | 1292.3ms/step | 15.5 samples/s +Step 27100/44000 | Loss: 2.9298 | LR: 1.66e-04 | 1294.1ms/step | 15.5 samples/s +Step 27110/44000 | Loss: 2.9243 | LR: 1.66e-04 | 1300.7ms/step | 15.4 samples/s +Step 27120/44000 | Loss: 2.9319 | LR: 1.66e-04 | 1298.5ms/step | 15.4 samples/s +Step 27130/44000 | Loss: 2.9248 | LR: 1.65e-04 | 1291.9ms/step | 15.5 samples/s +Step 27140/44000 | Loss: 2.9391 | LR: 1.65e-04 | 1294.2ms/step | 15.5 samples/s +Step 27150/44000 | Loss: 2.9575 | LR: 1.65e-04 | 1291.4ms/step | 15.5 samples/s +Step 27160/44000 | Loss: 2.9228 | LR: 1.65e-04 | 1293.8ms/step | 15.5 samples/s +Step 27170/44000 | Loss: 2.9508 | LR: 1.65e-04 | 1293.6ms/step | 15.5 samples/s +Step 27180/44000 | Loss: 2.9578 | LR: 1.65e-04 | 1292.4ms/step | 15.5 samples/s +Step 27190/44000 | Loss: 2.9542 | LR: 1.64e-04 | 1292.3ms/step | 15.5 samples/s +Step 27200/44000 | Loss: 2.9598 | LR: 1.64e-04 | 1291.0ms/step | 15.5 samples/s +Step 27210/44000 | Loss: 2.9461 | LR: 1.64e-04 | 1292.3ms/step | 15.5 samples/s +Step 27220/44000 | Loss: 2.9284 | LR: 1.64e-04 | 1291.6ms/step | 15.5 samples/s +Step 27230/44000 | Loss: 2.9213 | LR: 1.64e-04 | 1291.4ms/step | 15.5 samples/s +Step 27240/44000 | Loss: 2.9530 | LR: 1.63e-04 | 1292.2ms/step | 15.5 samples/s +Step 27250/44000 | Loss: 2.9242 | LR: 1.63e-04 | 1291.7ms/step | 15.5 samples/s +Validation at step 27250: Loss = 2.9584, Perplexity = 19.27 +Step 27260/44000 | Loss: 2.9627 | LR: 1.63e-04 | 3447.3ms/step | 5.8 samples/s +Step 27270/44000 | Loss: 2.9749 | LR: 1.63e-04 | 1290.7ms/step | 15.5 samples/s +Step 27280/44000 | Loss: 2.9639 | LR: 1.63e-04 | 1292.1ms/step | 15.5 samples/s +Step 27290/44000 | Loss: 2.9353 | LR: 1.63e-04 | 1291.2ms/step | 15.5 samples/s +Step 27300/44000 | Loss: 2.9225 | LR: 1.62e-04 | 1290.7ms/step | 15.5 samples/s +Step 27310/44000 | Loss: 2.9530 | LR: 1.62e-04 | 1291.8ms/step | 15.5 samples/s +Step 27320/44000 | Loss: 2.9612 | LR: 1.62e-04 | 1290.0ms/step | 15.5 samples/s +Step 27330/44000 | Loss: 2.9617 | LR: 1.62e-04 | 1290.4ms/step | 15.5 samples/s +Step 27340/44000 | Loss: 2.9202 | LR: 1.62e-04 | 1291.1ms/step | 15.5 samples/s +Step 27350/44000 | Loss: 2.9174 | LR: 1.62e-04 | 1292.0ms/step | 15.5 samples/s +Step 27360/44000 | Loss: 2.9247 | LR: 1.61e-04 | 1292.0ms/step | 15.5 samples/s +Step 27370/44000 | Loss: 2.9317 | LR: 1.61e-04 | 1292.0ms/step | 15.5 samples/s +Step 27380/44000 | Loss: 2.9771 | LR: 1.61e-04 | 1292.1ms/step | 15.5 samples/s +Step 27390/44000 | Loss: 2.9339 | LR: 1.61e-04 | 1289.9ms/step | 15.5 samples/s +Step 27400/44000 | Loss: 2.9603 | LR: 1.61e-04 | 1288.1ms/step | 15.5 samples/s +Step 27410/44000 | Loss: 2.9432 | LR: 1.61e-04 | 1288.4ms/step | 15.5 samples/s +Step 27420/44000 | Loss: 2.9375 | LR: 1.60e-04 | 1286.4ms/step | 15.5 samples/s +Step 27430/44000 | Loss: 2.9602 | LR: 1.60e-04 | 1289.0ms/step | 15.5 samples/s +Step 27440/44000 | Loss: 2.9501 | LR: 1.60e-04 | 1290.4ms/step | 15.5 samples/s +Step 27450/44000 | Loss: 2.9172 | LR: 1.60e-04 | 1292.8ms/step | 15.5 samples/s +Step 27460/44000 | Loss: 2.9377 | LR: 1.60e-04 | 1293.6ms/step | 15.5 samples/s +Step 27470/44000 | Loss: 2.9689 | LR: 1.60e-04 | 1290.5ms/step | 15.5 samples/s +Step 27480/44000 | Loss: 2.9404 | LR: 1.59e-04 | 1292.7ms/step | 15.5 samples/s +Step 27490/44000 | Loss: 2.9591 | LR: 1.59e-04 | 1293.9ms/step | 15.5 samples/s +Step 27500/44000 | Loss: 2.9498 | LR: 1.59e-04 | 1299.8ms/step | 15.4 samples/s +Validation at step 27500: Loss = 2.9571, Perplexity = 19.24 +Step 27510/44000 | Loss: 2.9821 | LR: 1.59e-04 | 3459.8ms/step | 5.8 samples/s +Step 27520/44000 | Loss: 2.9414 | LR: 1.59e-04 | 1292.5ms/step | 15.5 samples/s +Step 27530/44000 | Loss: 2.9538 | LR: 1.59e-04 | 1289.4ms/step | 15.5 samples/s +Step 27540/44000 | Loss: 2.9122 | LR: 1.58e-04 | 1291.1ms/step | 15.5 samples/s +Step 27550/44000 | Loss: 2.9118 | LR: 1.58e-04 | 1289.8ms/step | 15.5 samples/s +Step 27560/44000 | Loss: 2.9274 | LR: 1.58e-04 | 1292.7ms/step | 15.5 samples/s +Step 27570/44000 | Loss: 2.9181 | LR: 1.58e-04 | 1290.9ms/step | 15.5 samples/s +Step 27580/44000 | Loss: 2.9692 | LR: 1.58e-04 | 1290.6ms/step | 15.5 samples/s +Step 27590/44000 | Loss: 2.9351 | LR: 1.58e-04 | 1290.2ms/step | 15.5 samples/s +Step 27600/44000 | Loss: 2.9494 | LR: 1.57e-04 | 1290.4ms/step | 15.5 samples/s +Step 27610/44000 | Loss: 2.9195 | LR: 1.57e-04 | 1293.9ms/step | 15.5 samples/s +Step 27620/44000 | Loss: 2.9505 | LR: 1.57e-04 | 1291.4ms/step | 15.5 samples/s +Step 27630/44000 | Loss: 2.9175 | LR: 1.57e-04 | 1291.0ms/step | 15.5 samples/s +Step 27640/44000 | Loss: 2.9782 | LR: 1.57e-04 | 1292.0ms/step | 15.5 samples/s +Step 27650/44000 | Loss: 2.9985 | LR: 1.57e-04 | 1290.7ms/step | 15.5 samples/s +Step 27660/44000 | Loss: 2.9478 | LR: 1.56e-04 | 1289.7ms/step | 15.5 samples/s +Step 27670/44000 | Loss: 2.9367 | LR: 1.56e-04 | 1290.2ms/step | 15.5 samples/s +Step 27680/44000 | Loss: 2.9528 | LR: 1.56e-04 | 1291.7ms/step | 15.5 samples/s +Step 27690/44000 | Loss: 2.9410 | LR: 1.56e-04 | 1290.5ms/step | 15.5 samples/s +Step 27700/44000 | Loss: 2.9338 | LR: 1.56e-04 | 1291.1ms/step | 15.5 samples/s +Step 27710/44000 | Loss: 2.9526 | LR: 1.56e-04 | 1290.0ms/step | 15.5 samples/s +Step 27720/44000 | Loss: 2.9373 | LR: 1.55e-04 | 1290.0ms/step | 15.5 samples/s +Step 27730/44000 | Loss: 2.9396 | LR: 1.55e-04 | 1288.2ms/step | 15.5 samples/s +Step 27740/44000 | Loss: 2.9363 | LR: 1.55e-04 | 1289.3ms/step | 15.5 samples/s +Step 27750/44000 | Loss: 2.9365 | LR: 1.55e-04 | 1289.3ms/step | 15.5 samples/s +Validation at step 27750: Loss = 2.9557, Perplexity = 19.22 +Step 27760/44000 | Loss: 2.9420 | LR: 1.55e-04 | 3445.5ms/step | 5.8 samples/s +Step 27770/44000 | Loss: 2.9370 | LR: 1.55e-04 | 1289.2ms/step | 15.5 samples/s +Step 27780/44000 | Loss: 2.9362 | LR: 1.54e-04 | 1288.7ms/step | 15.5 samples/s +Step 27790/44000 | Loss: 2.8992 | LR: 1.54e-04 | 1289.6ms/step | 15.5 samples/s +Step 27800/44000 | Loss: 2.9330 | LR: 1.54e-04 | 1290.3ms/step | 15.5 samples/s +Step 27810/44000 | Loss: 2.9044 | LR: 1.54e-04 | 1288.4ms/step | 15.5 samples/s +Step 27820/44000 | Loss: 2.9265 | LR: 1.54e-04 | 1288.9ms/step | 15.5 samples/s +Step 27830/44000 | Loss: 2.9442 | LR: 1.54e-04 | 1288.6ms/step | 15.5 samples/s +Step 27840/44000 | Loss: 2.9591 | LR: 1.53e-04 | 1287.0ms/step | 15.5 samples/s +Step 27850/44000 | Loss: 2.9267 | LR: 1.53e-04 | 1290.3ms/step | 15.5 samples/s +Step 27860/44000 | Loss: 2.9337 | LR: 1.53e-04 | 1289.8ms/step | 15.5 samples/s +Step 27870/44000 | Loss: 2.9651 | LR: 1.53e-04 | 1291.4ms/step | 15.5 samples/s +Step 27880/44000 | Loss: 2.9376 | LR: 1.53e-04 | 1291.9ms/step | 15.5 samples/s +Step 27890/44000 | Loss: 2.9163 | LR: 1.53e-04 | 1291.6ms/step | 15.5 samples/s +Step 27900/44000 | Loss: 2.9213 | LR: 1.52e-04 | 1291.6ms/step | 15.5 samples/s +Step 27910/44000 | Loss: 2.9352 | LR: 1.52e-04 | 1290.8ms/step | 15.5 samples/s +Step 27920/44000 | Loss: 2.9731 | LR: 1.52e-04 | 1289.8ms/step | 15.5 samples/s +Step 27930/44000 | Loss: 2.9491 | LR: 1.52e-04 | 1289.9ms/step | 15.5 samples/s +Step 27940/44000 | Loss: 2.9416 | LR: 1.52e-04 | 1293.0ms/step | 15.5 samples/s +Step 27950/44000 | Loss: 2.9613 | LR: 1.52e-04 | 1287.6ms/step | 15.5 samples/s +Step 27960/44000 | Loss: 2.9427 | LR: 1.51e-04 | 1285.8ms/step | 15.6 samples/s +Step 27970/44000 | Loss: 2.9366 | LR: 1.51e-04 | 1290.1ms/step | 15.5 samples/s +Step 27980/44000 | Loss: 2.9442 | LR: 1.51e-04 | 1290.2ms/step | 15.5 samples/s +Step 27990/44000 | Loss: 2.9344 | LR: 1.51e-04 | 1291.8ms/step | 15.5 samples/s +Step 28000/44000 | Loss: 2.9396 | LR: 1.51e-04 | 1289.8ms/step | 15.5 samples/s +Validation at step 28000: Loss = 2.9546, Perplexity = 19.19 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-28000 +Step 28010/44000 | Loss: 2.9387 | LR: 1.51e-04 | 3731.3ms/step | 5.4 samples/s +Step 28020/44000 | Loss: 2.9755 | LR: 1.50e-04 | 1287.6ms/step | 15.5 samples/s +Step 28030/44000 | Loss: 2.9324 | LR: 1.50e-04 | 1289.5ms/step | 15.5 samples/s +Step 28040/44000 | Loss: 2.9530 | LR: 1.50e-04 | 1287.1ms/step | 15.5 samples/s +Step 28050/44000 | Loss: 2.9571 | LR: 1.50e-04 | 1286.9ms/step | 15.5 samples/s +Step 28060/44000 | Loss: 2.9643 | LR: 1.50e-04 | 1291.5ms/step | 15.5 samples/s +Step 28070/44000 | Loss: 2.9358 | LR: 1.50e-04 | 1291.9ms/step | 15.5 samples/s +Step 28080/44000 | Loss: 2.9192 | LR: 1.49e-04 | 1290.9ms/step | 15.5 samples/s +Step 28090/44000 | Loss: 2.9452 | LR: 1.49e-04 | 1289.3ms/step | 15.5 samples/s +Step 28100/44000 | Loss: 2.9401 | LR: 1.49e-04 | 1291.0ms/step | 15.5 samples/s +Step 28110/44000 | Loss: 2.9182 | LR: 1.49e-04 | 1290.6ms/step | 15.5 samples/s +Step 28120/44000 | Loss: 2.9461 | LR: 1.49e-04 | 1287.8ms/step | 15.5 samples/s +Step 28130/44000 | Loss: 2.9468 | LR: 1.49e-04 | 1290.4ms/step | 15.5 samples/s +Step 28140/44000 | Loss: 2.9404 | LR: 1.48e-04 | 1290.2ms/step | 15.5 samples/s +Step 28150/44000 | Loss: 2.9336 | LR: 1.48e-04 | 1290.5ms/step | 15.5 samples/s +Step 28160/44000 | Loss: 2.9471 | LR: 1.48e-04 | 1290.6ms/step | 15.5 samples/s +Step 28170/44000 | Loss: 2.9225 | LR: 1.48e-04 | 1290.0ms/step | 15.5 samples/s +Step 28180/44000 | Loss: 2.9603 | LR: 1.48e-04 | 1291.3ms/step | 15.5 samples/s +Step 28190/44000 | Loss: 2.9522 | LR: 1.48e-04 | 1292.4ms/step | 15.5 samples/s +Step 28200/44000 | Loss: 2.9620 | LR: 1.47e-04 | 1292.1ms/step | 15.5 samples/s +Step 28210/44000 | Loss: 2.9084 | LR: 1.47e-04 | 1293.8ms/step | 15.5 samples/s +Step 28220/44000 | Loss: 2.9252 | LR: 1.47e-04 | 1291.1ms/step | 15.5 samples/s +Step 28230/44000 | Loss: 2.9468 | LR: 1.47e-04 | 1289.3ms/step | 15.5 samples/s +Step 28240/44000 | Loss: 2.9080 | LR: 1.47e-04 | 1286.6ms/step | 15.5 samples/s +Step 28250/44000 | Loss: 2.9224 | LR: 1.47e-04 | 1287.2ms/step | 15.5 samples/s +Validation at step 28250: Loss = 2.9539, Perplexity = 19.18 +Step 28260/44000 | Loss: 2.9430 | LR: 1.46e-04 | 3449.8ms/step | 5.8 samples/s +Step 28270/44000 | Loss: 2.9265 | LR: 1.46e-04 | 1287.5ms/step | 15.5 samples/s +Step 28280/44000 | Loss: 2.9681 | LR: 1.46e-04 | 1289.6ms/step | 15.5 samples/s +Step 28290/44000 | Loss: 2.8971 | LR: 1.46e-04 | 1292.4ms/step | 15.5 samples/s +Step 28300/44000 | Loss: 2.9379 | LR: 1.46e-04 | 1290.6ms/step | 15.5 samples/s +Step 28310/44000 | Loss: 2.9134 | LR: 1.46e-04 | 1291.0ms/step | 15.5 samples/s +Step 28320/44000 | Loss: 2.9760 | LR: 1.45e-04 | 1290.9ms/step | 15.5 samples/s +Step 28330/44000 | Loss: 2.9386 | LR: 1.45e-04 | 1290.7ms/step | 15.5 samples/s +Step 28340/44000 | Loss: 2.9499 | LR: 1.45e-04 | 1289.8ms/step | 15.5 samples/s +Step 28350/44000 | Loss: 2.9273 | LR: 1.45e-04 | 1289.2ms/step | 15.5 samples/s +Step 28360/44000 | Loss: 2.9357 | LR: 1.45e-04 | 1290.4ms/step | 15.5 samples/s +Step 28370/44000 | Loss: 2.9127 | LR: 1.45e-04 | 1291.1ms/step | 15.5 samples/s +Step 28380/44000 | Loss: 2.9213 | LR: 1.44e-04 | 1289.7ms/step | 15.5 samples/s +Step 28390/44000 | Loss: 2.9359 | LR: 1.44e-04 | 1288.3ms/step | 15.5 samples/s +Step 28400/44000 | Loss: 2.9358 | LR: 1.44e-04 | 1289.9ms/step | 15.5 samples/s +Step 28410/44000 | Loss: 2.9601 | LR: 1.44e-04 | 1292.9ms/step | 15.5 samples/s +Step 28420/44000 | Loss: 2.9824 | LR: 1.44e-04 | 1292.1ms/step | 15.5 samples/s +Step 28430/44000 | Loss: 2.9071 | LR: 1.44e-04 | 1294.3ms/step | 15.5 samples/s +Step 28440/44000 | Loss: 2.9470 | LR: 1.43e-04 | 1291.1ms/step | 15.5 samples/s +Step 28450/44000 | Loss: 2.9440 | LR: 1.43e-04 | 1291.7ms/step | 15.5 samples/s +Step 28460/44000 | Loss: 2.9513 | LR: 1.43e-04 | 1292.4ms/step | 15.5 samples/s +Step 28470/44000 | Loss: 2.9492 | LR: 1.43e-04 | 1291.8ms/step | 15.5 samples/s +Step 28480/44000 | Loss: 2.9298 | LR: 1.43e-04 | 1292.5ms/step | 15.5 samples/s +Step 28490/44000 | Loss: 2.9377 | LR: 1.43e-04 | 1292.9ms/step | 15.5 samples/s +Step 28500/44000 | Loss: 2.9386 | LR: 1.42e-04 | 1292.3ms/step | 15.5 samples/s +Validation at step 28500: Loss = 2.9525, Perplexity = 19.15 +Step 28510/44000 | Loss: 2.9636 | LR: 1.42e-04 | 3579.6ms/step | 5.6 samples/s +Step 28520/44000 | Loss: 2.9432 | LR: 1.42e-04 | 1288.3ms/step | 15.5 samples/s +Step 28530/44000 | Loss: 2.9305 | LR: 1.42e-04 | 1289.3ms/step | 15.5 samples/s +Step 28540/44000 | Loss: 2.9572 | LR: 1.42e-04 | 1288.2ms/step | 15.5 samples/s +Step 28550/44000 | Loss: 2.9753 | LR: 1.42e-04 | 1287.3ms/step | 15.5 samples/s +Step 28560/44000 | Loss: 2.9247 | LR: 1.41e-04 | 1289.3ms/step | 15.5 samples/s +Step 28570/44000 | Loss: 2.9177 | LR: 1.41e-04 | 1287.1ms/step | 15.5 samples/s +Step 28580/44000 | Loss: 2.9485 | LR: 1.41e-04 | 1285.1ms/step | 15.6 samples/s +Step 28590/44000 | Loss: 2.9654 | LR: 1.41e-04 | 1289.2ms/step | 15.5 samples/s +Step 28600/44000 | Loss: 2.9651 | LR: 1.41e-04 | 1290.8ms/step | 15.5 samples/s +Step 28610/44000 | Loss: 2.9298 | LR: 1.41e-04 | 1290.0ms/step | 15.5 samples/s +Step 28620/44000 | Loss: 2.9134 | LR: 1.40e-04 | 1288.9ms/step | 15.5 samples/s +Step 28630/44000 | Loss: 2.9251 | LR: 1.40e-04 | 1286.8ms/step | 15.5 samples/s +Step 28640/44000 | Loss: 2.9075 | LR: 1.40e-04 | 1285.9ms/step | 15.6 samples/s +Step 28650/44000 | Loss: 2.9307 | LR: 1.40e-04 | 1287.0ms/step | 15.5 samples/s +Step 28660/44000 | Loss: 2.9280 | LR: 1.40e-04 | 1290.3ms/step | 15.5 samples/s +Step 28670/44000 | Loss: 2.8975 | LR: 1.40e-04 | 1289.9ms/step | 15.5 samples/s +Step 28680/44000 | Loss: 2.9468 | LR: 1.39e-04 | 1288.4ms/step | 15.5 samples/s +Step 28690/44000 | Loss: 2.9430 | LR: 1.39e-04 | 1288.6ms/step | 15.5 samples/s +Step 28700/44000 | Loss: 2.9463 | LR: 1.39e-04 | 1287.8ms/step | 15.5 samples/s +Step 28710/44000 | Loss: 2.9507 | LR: 1.39e-04 | 1289.5ms/step | 15.5 samples/s +Step 28720/44000 | Loss: 2.9265 | LR: 1.39e-04 | 1288.6ms/step | 15.5 samples/s +Step 28730/44000 | Loss: 2.9284 | LR: 1.39e-04 | 1291.0ms/step | 15.5 samples/s +Step 28740/44000 | Loss: 2.9523 | LR: 1.38e-04 | 1291.1ms/step | 15.5 samples/s +Step 28750/44000 | Loss: 2.9157 | LR: 1.38e-04 | 1290.1ms/step | 15.5 samples/s +Validation at step 28750: Loss = 2.9513, Perplexity = 19.13 +Step 28760/44000 | Loss: 2.9428 | LR: 1.38e-04 | 3443.9ms/step | 5.8 samples/s +Step 28770/44000 | Loss: 2.9091 | LR: 1.38e-04 | 1288.9ms/step | 15.5 samples/s +Step 28780/44000 | Loss: 2.9435 | LR: 1.38e-04 | 1288.6ms/step | 15.5 samples/s +Step 28790/44000 | Loss: 2.9533 | LR: 1.38e-04 | 1287.9ms/step | 15.5 samples/s +Step 28800/44000 | Loss: 2.9555 | LR: 1.38e-04 | 1289.1ms/step | 15.5 samples/s +Step 28810/44000 | Loss: 2.9406 | LR: 1.37e-04 | 1288.0ms/step | 15.5 samples/s +Step 28820/44000 | Loss: 2.9156 | LR: 1.37e-04 | 1288.0ms/step | 15.5 samples/s +Step 28830/44000 | Loss: 2.9471 | LR: 1.37e-04 | 1288.9ms/step | 15.5 samples/s +Step 28840/44000 | Loss: 2.9422 | LR: 1.37e-04 | 1291.2ms/step | 15.5 samples/s +Step 28850/44000 | Loss: 2.9202 | LR: 1.37e-04 | 1289.7ms/step | 15.5 samples/s +Step 28860/44000 | Loss: 2.9515 | LR: 1.37e-04 | 1289.1ms/step | 15.5 samples/s +Step 28870/44000 | Loss: 2.9556 | LR: 1.36e-04 | 1290.6ms/step | 15.5 samples/s +Step 28880/44000 | Loss: 2.9296 | LR: 1.36e-04 | 1290.2ms/step | 15.5 samples/s +Step 28890/44000 | Loss: 2.9423 | LR: 1.36e-04 | 1290.2ms/step | 15.5 samples/s +Step 28900/44000 | Loss: 2.9369 | LR: 1.36e-04 | 1289.2ms/step | 15.5 samples/s +Step 28910/44000 | Loss: 2.9334 | LR: 1.36e-04 | 1289.4ms/step | 15.5 samples/s +Step 28920/44000 | Loss: 2.9466 | LR: 1.36e-04 | 1291.4ms/step | 15.5 samples/s +Step 28930/44000 | Loss: 2.9373 | LR: 1.35e-04 | 1290.0ms/step | 15.5 samples/s +Step 28940/44000 | Loss: 2.9321 | LR: 1.35e-04 | 1291.1ms/step | 15.5 samples/s +Step 28950/44000 | Loss: 2.9765 | LR: 1.35e-04 | 1289.5ms/step | 15.5 samples/s +Step 28960/44000 | Loss: 2.9355 | LR: 1.35e-04 | 1287.5ms/step | 15.5 samples/s +Step 28970/44000 | Loss: 2.9587 | LR: 1.35e-04 | 1286.9ms/step | 15.5 samples/s +Step 28980/44000 | Loss: 2.9432 | LR: 1.35e-04 | 1287.9ms/step | 15.5 samples/s +Step 28990/44000 | Loss: 2.9322 | LR: 1.34e-04 | 1289.8ms/step | 15.5 samples/s +Step 29000/44000 | Loss: 2.9310 | LR: 1.34e-04 | 1290.6ms/step | 15.5 samples/s +Validation at step 29000: Loss = 2.9503, Perplexity = 19.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-29000 +Step 29010/44000 | Loss: 2.9255 | LR: 1.34e-04 | 3727.2ms/step | 5.4 samples/s +Step 29020/44000 | Loss: 2.9377 | LR: 1.34e-04 | 1289.1ms/step | 15.5 samples/s +Step 29030/44000 | Loss: 2.9245 | LR: 1.34e-04 | 1287.0ms/step | 15.5 samples/s +Step 29040/44000 | Loss: 2.9118 | LR: 1.34e-04 | 1290.7ms/step | 15.5 samples/s +Step 29050/44000 | Loss: 2.9163 | LR: 1.33e-04 | 1291.4ms/step | 15.5 samples/s +Step 29060/44000 | Loss: 2.9463 | LR: 1.33e-04 | 1291.6ms/step | 15.5 samples/s +Step 29070/44000 | Loss: 2.9413 | LR: 1.33e-04 | 1291.2ms/step | 15.5 samples/s +Step 29080/44000 | Loss: 2.9114 | LR: 1.33e-04 | 1287.4ms/step | 15.5 samples/s +Step 29090/44000 | Loss: 2.9441 | LR: 1.33e-04 | 1287.8ms/step | 15.5 samples/s +Step 29100/44000 | Loss: 2.9547 | LR: 1.33e-04 | 1286.9ms/step | 15.5 samples/s +Step 29110/44000 | Loss: 2.9492 | LR: 1.33e-04 | 1288.2ms/step | 15.5 samples/s +Step 29120/44000 | Loss: 2.9191 | LR: 1.32e-04 | 1286.3ms/step | 15.5 samples/s +Step 29130/44000 | Loss: 2.9471 | LR: 1.32e-04 | 1288.7ms/step | 15.5 samples/s +Step 29140/44000 | Loss: 2.9403 | LR: 1.32e-04 | 1290.0ms/step | 15.5 samples/s +Step 29150/44000 | Loss: 2.9571 | LR: 1.32e-04 | 1289.6ms/step | 15.5 samples/s +Step 29160/44000 | Loss: 2.9335 | LR: 1.32e-04 | 1290.6ms/step | 15.5 samples/s +Step 29170/44000 | Loss: 2.9113 | LR: 1.32e-04 | 1289.7ms/step | 15.5 samples/s +Step 29180/44000 | Loss: 2.9135 | LR: 1.31e-04 | 1290.8ms/step | 15.5 samples/s +Step 29190/44000 | Loss: 2.9273 | LR: 1.31e-04 | 1290.6ms/step | 15.5 samples/s +Step 29200/44000 | Loss: 2.9344 | LR: 1.31e-04 | 1289.6ms/step | 15.5 samples/s +Step 29210/44000 | Loss: 2.8893 | LR: 1.31e-04 | 1288.9ms/step | 15.5 samples/s +Step 29220/44000 | Loss: 2.9305 | LR: 1.31e-04 | 1290.9ms/step | 15.5 samples/s +Step 29230/44000 | Loss: 2.9204 | LR: 1.31e-04 | 1290.7ms/step | 15.5 samples/s +Step 29240/44000 | Loss: 2.9463 | LR: 1.30e-04 | 1291.2ms/step | 15.5 samples/s +Step 29250/44000 | Loss: 2.9280 | LR: 1.30e-04 | 1292.4ms/step | 15.5 samples/s +Validation at step 29250: Loss = 2.9494, Perplexity = 19.09 +Step 29260/44000 | Loss: 2.9177 | LR: 1.30e-04 | 3445.6ms/step | 5.8 samples/s +Step 29270/44000 | Loss: 2.9349 | LR: 1.30e-04 | 1291.8ms/step | 15.5 samples/s +Step 29280/44000 | Loss: 2.9244 | LR: 1.30e-04 | 1288.2ms/step | 15.5 samples/s +Step 29290/44000 | Loss: 2.9401 | LR: 1.30e-04 | 1289.8ms/step | 15.5 samples/s +Step 29300/44000 | Loss: 2.9462 | LR: 1.29e-04 | 1289.1ms/step | 15.5 samples/s +Step 29310/44000 | Loss: 2.9489 | LR: 1.29e-04 | 1290.0ms/step | 15.5 samples/s +Step 29320/44000 | Loss: 2.9454 | LR: 1.29e-04 | 1293.4ms/step | 15.5 samples/s +Step 29330/44000 | Loss: 2.9507 | LR: 1.29e-04 | 1291.9ms/step | 15.5 samples/s +Step 29340/44000 | Loss: 2.9071 | LR: 1.29e-04 | 1291.7ms/step | 15.5 samples/s +Step 29350/44000 | Loss: 2.9502 | LR: 1.29e-04 | 1290.1ms/step | 15.5 samples/s +Step 29360/44000 | Loss: 2.9544 | LR: 1.29e-04 | 1289.6ms/step | 15.5 samples/s +Step 29370/44000 | Loss: 2.9254 | LR: 1.28e-04 | 1292.2ms/step | 15.5 samples/s +Step 29380/44000 | Loss: 2.9343 | LR: 1.28e-04 | 1292.2ms/step | 15.5 samples/s +Step 29390/44000 | Loss: 2.9509 | LR: 1.28e-04 | 1291.4ms/step | 15.5 samples/s +Step 29400/44000 | Loss: 2.9369 | LR: 1.28e-04 | 1292.7ms/step | 15.5 samples/s +Step 29410/44000 | Loss: 2.9256 | LR: 1.28e-04 | 1293.2ms/step | 15.5 samples/s +Step 29420/44000 | Loss: 2.9667 | LR: 1.28e-04 | 1291.2ms/step | 15.5 samples/s +Step 29430/44000 | Loss: 2.9400 | LR: 1.27e-04 | 1290.0ms/step | 15.5 samples/s +Step 29440/44000 | Loss: 2.9399 | LR: 1.27e-04 | 1289.6ms/step | 15.5 samples/s +Step 29450/44000 | Loss: 2.9586 | LR: 1.27e-04 | 1290.1ms/step | 15.5 samples/s +Step 29460/44000 | Loss: 2.9454 | LR: 1.27e-04 | 1287.7ms/step | 15.5 samples/s +Step 29470/44000 | Loss: 2.9409 | LR: 1.27e-04 | 1289.5ms/step | 15.5 samples/s +Step 29480/44000 | Loss: 2.9379 | LR: 1.27e-04 | 1289.4ms/step | 15.5 samples/s +Step 29490/44000 | Loss: 2.9284 | LR: 1.26e-04 | 1290.2ms/step | 15.5 samples/s +Step 29500/44000 | Loss: 2.9234 | LR: 1.26e-04 | 1290.8ms/step | 15.5 samples/s +Validation at step 29500: Loss = 2.9486, Perplexity = 19.08 +Step 29510/44000 | Loss: 2.9280 | LR: 1.26e-04 | 3447.3ms/step | 5.8 samples/s +Step 29520/44000 | Loss: 2.9410 | LR: 1.26e-04 | 1289.2ms/step | 15.5 samples/s +Step 29530/44000 | Loss: 2.9208 | LR: 1.26e-04 | 1290.3ms/step | 15.5 samples/s +Step 29540/44000 | Loss: 2.9169 | LR: 1.26e-04 | 1290.8ms/step | 15.5 samples/s +Step 29550/44000 | Loss: 2.9250 | LR: 1.26e-04 | 1290.9ms/step | 15.5 samples/s +Step 29560/44000 | Loss: 2.9240 | LR: 1.25e-04 | 1291.9ms/step | 15.5 samples/s +Step 29570/44000 | Loss: 2.9322 | LR: 1.25e-04 | 1292.8ms/step | 15.5 samples/s +Step 29580/44000 | Loss: 2.9227 | LR: 1.25e-04 | 1291.6ms/step | 15.5 samples/s +Step 29590/44000 | Loss: 2.9529 | LR: 1.25e-04 | 1289.7ms/step | 15.5 samples/s +Step 29600/44000 | Loss: 2.9420 | LR: 1.25e-04 | 1288.1ms/step | 15.5 samples/s +Step 29610/44000 | Loss: 2.9332 | LR: 1.25e-04 | 1290.9ms/step | 15.5 samples/s +Step 29620/44000 | Loss: 2.9166 | LR: 1.24e-04 | 1290.7ms/step | 15.5 samples/s +Step 29630/44000 | Loss: 2.9579 | LR: 1.24e-04 | 1290.7ms/step | 15.5 samples/s +Step 29640/44000 | Loss: 2.9282 | LR: 1.24e-04 | 1291.3ms/step | 15.5 samples/s +Step 29650/44000 | Loss: 2.9210 | LR: 1.24e-04 | 1291.6ms/step | 15.5 samples/s +Step 29660/44000 | Loss: 2.9763 | LR: 1.24e-04 | 1291.5ms/step | 15.5 samples/s +Step 29670/44000 | Loss: 2.9190 | LR: 1.24e-04 | 1292.3ms/step | 15.5 samples/s +Step 29680/44000 | Loss: 2.9321 | LR: 1.23e-04 | 1293.9ms/step | 15.5 samples/s +Step 29690/44000 | Loss: 2.9398 | LR: 1.23e-04 | 1293.3ms/step | 15.5 samples/s +Step 29700/44000 | Loss: 2.9422 | LR: 1.23e-04 | 1291.2ms/step | 15.5 samples/s +Step 29710/44000 | Loss: 2.9535 | LR: 1.23e-04 | 1291.3ms/step | 15.5 samples/s +Step 29720/44000 | Loss: 2.8934 | LR: 1.23e-04 | 1291.0ms/step | 15.5 samples/s +Step 29730/44000 | Loss: 2.9420 | LR: 1.23e-04 | 1288.9ms/step | 15.5 samples/s +Step 29740/44000 | Loss: 2.9080 | LR: 1.23e-04 | 1294.3ms/step | 15.5 samples/s +Step 29750/44000 | Loss: 2.9248 | LR: 1.22e-04 | 1291.6ms/step | 15.5 samples/s +Validation at step 29750: Loss = 2.9474, Perplexity = 19.06 +Step 29760/44000 | Loss: 2.9528 | LR: 1.22e-04 | 3462.5ms/step | 5.8 samples/s +Step 29770/44000 | Loss: 2.9259 | LR: 1.22e-04 | 1292.2ms/step | 15.5 samples/s +Step 29780/44000 | Loss: 2.9348 | LR: 1.22e-04 | 1294.9ms/step | 15.4 samples/s +Step 29790/44000 | Loss: 2.9030 | LR: 1.22e-04 | 1294.8ms/step | 15.4 samples/s +Step 29800/44000 | Loss: 2.9111 | LR: 1.22e-04 | 1292.8ms/step | 15.5 samples/s +Step 29810/44000 | Loss: 2.9197 | LR: 1.21e-04 | 1292.9ms/step | 15.5 samples/s +Step 29820/44000 | Loss: 2.9629 | LR: 1.21e-04 | 1293.5ms/step | 15.5 samples/s +Step 29830/44000 | Loss: 2.9510 | LR: 1.21e-04 | 1291.4ms/step | 15.5 samples/s +Step 29840/44000 | Loss: 2.8994 | LR: 1.21e-04 | 1295.6ms/step | 15.4 samples/s +Step 29850/44000 | Loss: 2.9235 | LR: 1.21e-04 | 1292.7ms/step | 15.5 samples/s +Step 29860/44000 | Loss: 2.9383 | LR: 1.21e-04 | 1291.6ms/step | 15.5 samples/s +Step 29870/44000 | Loss: 2.9072 | LR: 1.21e-04 | 1293.5ms/step | 15.5 samples/s +Step 29880/44000 | Loss: 2.9257 | LR: 1.20e-04 | 1290.8ms/step | 15.5 samples/s +Step 29890/44000 | Loss: 2.9311 | LR: 1.20e-04 | 1290.9ms/step | 15.5 samples/s +Step 29900/44000 | Loss: 2.9714 | LR: 1.20e-04 | 1291.7ms/step | 15.5 samples/s +Step 29910/44000 | Loss: 2.9275 | LR: 1.20e-04 | 1287.9ms/step | 15.5 samples/s +Step 29920/44000 | Loss: 2.9180 | LR: 1.20e-04 | 1292.5ms/step | 15.5 samples/s +Step 29930/44000 | Loss: 2.9236 | LR: 1.20e-04 | 1293.9ms/step | 15.5 samples/s +Step 29940/44000 | Loss: 2.9199 | LR: 1.19e-04 | 1289.3ms/step | 15.5 samples/s +Step 29950/44000 | Loss: 2.9310 | LR: 1.19e-04 | 1287.9ms/step | 15.5 samples/s +Step 29960/44000 | Loss: 2.9115 | LR: 1.19e-04 | 1290.6ms/step | 15.5 samples/s +Step 29970/44000 | Loss: 2.9242 | LR: 1.19e-04 | 1290.1ms/step | 15.5 samples/s +Step 29980/44000 | Loss: 2.9223 | LR: 1.19e-04 | 1286.9ms/step | 15.5 samples/s +Step 29990/44000 | Loss: 2.9372 | LR: 1.19e-04 | 1286.9ms/step | 15.5 samples/s +Step 30000/44000 | Loss: 2.9146 | LR: 1.19e-04 | 1289.4ms/step | 15.5 samples/s +Validation at step 30000: Loss = 2.9465, Perplexity = 19.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-30000 +Step 30010/44000 | Loss: 2.9421 | LR: 1.18e-04 | 3732.0ms/step | 5.4 samples/s +Step 30020/44000 | Loss: 2.9543 | LR: 1.18e-04 | 1288.3ms/step | 15.5 samples/s +Step 30030/44000 | Loss: 2.9220 | LR: 1.18e-04 | 1288.3ms/step | 15.5 samples/s +Step 30040/44000 | Loss: 2.9410 | LR: 1.18e-04 | 1287.4ms/step | 15.5 samples/s +Step 30050/44000 | Loss: 2.9133 | LR: 1.18e-04 | 1287.6ms/step | 15.5 samples/s +Step 30060/44000 | Loss: 2.9393 | LR: 1.18e-04 | 1288.3ms/step | 15.5 samples/s +Step 30070/44000 | Loss: 2.8926 | LR: 1.17e-04 | 1288.9ms/step | 15.5 samples/s +Step 30080/44000 | Loss: 2.9659 | LR: 1.17e-04 | 1292.3ms/step | 15.5 samples/s +Step 30090/44000 | Loss: 2.9416 | LR: 1.17e-04 | 1291.3ms/step | 15.5 samples/s +Step 30100/44000 | Loss: 2.9411 | LR: 1.17e-04 | 1289.4ms/step | 15.5 samples/s +Step 30110/44000 | Loss: 2.9282 | LR: 1.17e-04 | 1290.8ms/step | 15.5 samples/s +Step 30120/44000 | Loss: 2.9168 | LR: 1.17e-04 | 1290.8ms/step | 15.5 samples/s +Step 30130/44000 | Loss: 2.9456 | LR: 1.17e-04 | 1289.9ms/step | 15.5 samples/s +Step 30140/44000 | Loss: 2.9348 | LR: 1.16e-04 | 1290.8ms/step | 15.5 samples/s +Step 30150/44000 | Loss: 2.9180 | LR: 1.16e-04 | 1290.2ms/step | 15.5 samples/s +Step 30160/44000 | Loss: 2.9043 | LR: 1.16e-04 | 1289.6ms/step | 15.5 samples/s +Step 30170/44000 | Loss: 2.9070 | LR: 1.16e-04 | 1290.5ms/step | 15.5 samples/s +Step 30180/44000 | Loss: 2.9419 | LR: 1.16e-04 | 1291.4ms/step | 15.5 samples/s +Step 30190/44000 | Loss: 2.9196 | LR: 1.16e-04 | 1291.0ms/step | 15.5 samples/s +Step 30200/44000 | Loss: 2.9278 | LR: 1.15e-04 | 1289.8ms/step | 15.5 samples/s +Step 30210/44000 | Loss: 2.9292 | LR: 1.15e-04 | 1287.0ms/step | 15.5 samples/s +Step 30220/44000 | Loss: 2.9045 | LR: 1.15e-04 | 1289.6ms/step | 15.5 samples/s +Step 30230/44000 | Loss: 2.9488 | LR: 1.15e-04 | 1295.4ms/step | 15.4 samples/s +Step 30240/44000 | Loss: 2.9317 | LR: 1.15e-04 | 1293.5ms/step | 15.5 samples/s +Step 30250/44000 | Loss: 2.9194 | LR: 1.15e-04 | 1293.6ms/step | 15.5 samples/s +Validation at step 30250: Loss = 2.9456, Perplexity = 19.02 +Step 30260/44000 | Loss: 2.9233 | LR: 1.15e-04 | 3453.3ms/step | 5.8 samples/s +Step 30270/44000 | Loss: 2.9115 | LR: 1.14e-04 | 1290.9ms/step | 15.5 samples/s +Step 30280/44000 | Loss: 2.9545 | LR: 1.14e-04 | 1291.1ms/step | 15.5 samples/s +Step 30290/44000 | Loss: 2.9127 | LR: 1.14e-04 | 1292.8ms/step | 15.5 samples/s +Step 30300/44000 | Loss: 2.8968 | LR: 1.14e-04 | 1290.1ms/step | 15.5 samples/s +Step 30310/44000 | Loss: 2.9067 | LR: 1.14e-04 | 1288.1ms/step | 15.5 samples/s +Step 30320/44000 | Loss: 2.9057 | LR: 1.14e-04 | 1288.5ms/step | 15.5 samples/s +Step 30330/44000 | Loss: 2.9236 | LR: 1.13e-04 | 1292.1ms/step | 15.5 samples/s +Step 30340/44000 | Loss: 2.9304 | LR: 1.13e-04 | 1289.2ms/step | 15.5 samples/s +Step 30350/44000 | Loss: 2.8931 | LR: 1.13e-04 | 1290.4ms/step | 15.5 samples/s +Step 30360/44000 | Loss: 2.9296 | LR: 1.13e-04 | 1291.0ms/step | 15.5 samples/s +Step 30370/44000 | Loss: 2.9428 | LR: 1.13e-04 | 1289.9ms/step | 15.5 samples/s +Step 30380/44000 | Loss: 2.9510 | LR: 1.13e-04 | 1291.5ms/step | 15.5 samples/s +Step 30390/44000 | Loss: 2.9227 | LR: 1.13e-04 | 1289.2ms/step | 15.5 samples/s +Step 30400/44000 | Loss: 2.9249 | LR: 1.12e-04 | 1287.9ms/step | 15.5 samples/s +Step 30410/44000 | Loss: 2.9331 | LR: 1.12e-04 | 1288.9ms/step | 15.5 samples/s +Step 30420/44000 | Loss: 2.9275 | LR: 1.12e-04 | 1290.5ms/step | 15.5 samples/s +Step 30430/44000 | Loss: 2.9292 | LR: 1.12e-04 | 1290.6ms/step | 15.5 samples/s +Step 30440/44000 | Loss: 2.9297 | LR: 1.12e-04 | 1291.4ms/step | 15.5 samples/s +Step 30450/44000 | Loss: 2.9019 | LR: 1.12e-04 | 1292.7ms/step | 15.5 samples/s +Step 30460/44000 | Loss: 2.9313 | LR: 1.11e-04 | 1292.2ms/step | 15.5 samples/s +Step 30470/44000 | Loss: 2.9425 | LR: 1.11e-04 | 1290.8ms/step | 15.5 samples/s +Step 30480/44000 | Loss: 2.9312 | LR: 1.11e-04 | 1290.5ms/step | 15.5 samples/s +Step 30490/44000 | Loss: 2.9246 | LR: 1.11e-04 | 1291.6ms/step | 15.5 samples/s +Step 30500/44000 | Loss: 2.9204 | LR: 1.11e-04 | 1289.5ms/step | 15.5 samples/s +Validation at step 30500: Loss = 2.9448, Perplexity = 19.01 +Step 30510/44000 | Loss: 2.9184 | LR: 1.11e-04 | 3449.7ms/step | 5.8 samples/s +Step 30520/44000 | Loss: 2.9251 | LR: 1.11e-04 | 1292.3ms/step | 15.5 samples/s +Step 30530/44000 | Loss: 2.9670 | LR: 1.10e-04 | 1289.8ms/step | 15.5 samples/s +Step 30540/44000 | Loss: 2.9623 | LR: 1.10e-04 | 1290.3ms/step | 15.5 samples/s +Step 30550/44000 | Loss: 2.9346 | LR: 1.10e-04 | 1289.0ms/step | 15.5 samples/s +Step 30560/44000 | Loss: 2.9340 | LR: 1.10e-04 | 1290.8ms/step | 15.5 samples/s +Step 30570/44000 | Loss: 2.9466 | LR: 1.10e-04 | 1290.2ms/step | 15.5 samples/s +Step 30580/44000 | Loss: 2.9423 | LR: 1.10e-04 | 1291.6ms/step | 15.5 samples/s +Step 30590/44000 | Loss: 2.9307 | LR: 1.10e-04 | 1292.3ms/step | 15.5 samples/s +Step 30600/44000 | Loss: 2.9260 | LR: 1.09e-04 | 1293.7ms/step | 15.5 samples/s +Step 30610/44000 | Loss: 2.9271 | LR: 1.09e-04 | 1290.7ms/step | 15.5 samples/s +Step 30620/44000 | Loss: 2.9395 | LR: 1.09e-04 | 1289.7ms/step | 15.5 samples/s +Step 30630/44000 | Loss: 2.9123 | LR: 1.09e-04 | 1288.3ms/step | 15.5 samples/s +Step 30640/44000 | Loss: 2.9410 | LR: 1.09e-04 | 1289.2ms/step | 15.5 samples/s +Step 30650/44000 | Loss: 2.9500 | LR: 1.09e-04 | 1290.5ms/step | 15.5 samples/s +Step 30660/44000 | Loss: 2.9671 | LR: 1.08e-04 | 1291.4ms/step | 15.5 samples/s +Step 30670/44000 | Loss: 2.9543 | LR: 1.08e-04 | 1289.9ms/step | 15.5 samples/s +Step 30680/44000 | Loss: 2.9355 | LR: 1.08e-04 | 1289.6ms/step | 15.5 samples/s +Step 30690/44000 | Loss: 2.9212 | LR: 1.08e-04 | 1288.1ms/step | 15.5 samples/s +Step 30700/44000 | Loss: 2.9429 | LR: 1.08e-04 | 1290.5ms/step | 15.5 samples/s +Step 30710/44000 | Loss: 2.9237 | LR: 1.08e-04 | 1289.8ms/step | 15.5 samples/s +Step 30720/44000 | Loss: 2.9410 | LR: 1.08e-04 | 1294.8ms/step | 15.4 samples/s +Step 30730/44000 | Loss: 2.9252 | LR: 1.07e-04 | 1291.1ms/step | 15.5 samples/s +Step 30740/44000 | Loss: 2.9430 | LR: 1.07e-04 | 1290.3ms/step | 15.5 samples/s +Step 30750/44000 | Loss: 2.9263 | LR: 1.07e-04 | 1290.1ms/step | 15.5 samples/s +Validation at step 30750: Loss = 2.9439, Perplexity = 18.99 +Step 30760/44000 | Loss: 2.9308 | LR: 1.07e-04 | 3442.9ms/step | 5.8 samples/s +Step 30770/44000 | Loss: 2.9719 | LR: 1.07e-04 | 1287.5ms/step | 15.5 samples/s +Step 30780/44000 | Loss: 2.9229 | LR: 1.07e-04 | 1287.5ms/step | 15.5 samples/s +Step 30790/44000 | Loss: 2.9285 | LR: 1.07e-04 | 1287.9ms/step | 15.5 samples/s +Step 30800/44000 | Loss: 2.9370 | LR: 1.06e-04 | 1288.7ms/step | 15.5 samples/s +Step 30810/44000 | Loss: 2.9047 | LR: 1.06e-04 | 1289.8ms/step | 15.5 samples/s +Step 30820/44000 | Loss: 2.9354 | LR: 1.06e-04 | 1290.1ms/step | 15.5 samples/s +Step 30830/44000 | Loss: 2.9741 | LR: 1.06e-04 | 1290.2ms/step | 15.5 samples/s +Step 30840/44000 | Loss: 2.9446 | LR: 1.06e-04 | 1289.9ms/step | 15.5 samples/s +Step 30850/44000 | Loss: 2.9172 | LR: 1.06e-04 | 1290.1ms/step | 15.5 samples/s +Step 30860/44000 | Loss: 2.9464 | LR: 1.05e-04 | 1291.4ms/step | 15.5 samples/s +Step 30870/44000 | Loss: 2.9059 | LR: 1.05e-04 | 1289.6ms/step | 15.5 samples/s +Step 30880/44000 | Loss: 2.9234 | LR: 1.05e-04 | 1287.7ms/step | 15.5 samples/s +Step 30890/44000 | Loss: 2.8778 | LR: 1.05e-04 | 1289.2ms/step | 15.5 samples/s +Step 30900/44000 | Loss: 2.9071 | LR: 1.05e-04 | 1289.1ms/step | 15.5 samples/s +Step 30910/44000 | Loss: 2.9186 | LR: 1.05e-04 | 1289.9ms/step | 15.5 samples/s +Step 30920/44000 | Loss: 2.9344 | LR: 1.05e-04 | 1293.1ms/step | 15.5 samples/s +Step 30930/44000 | Loss: 2.9196 | LR: 1.04e-04 | 1290.4ms/step | 15.5 samples/s +Step 30940/44000 | Loss: 2.9062 | LR: 1.04e-04 | 1292.0ms/step | 15.5 samples/s +Step 30950/44000 | Loss: 2.9293 | LR: 1.04e-04 | 1288.6ms/step | 15.5 samples/s +Step 30960/44000 | Loss: 2.9231 | LR: 1.04e-04 | 1289.0ms/step | 15.5 samples/s +Step 30970/44000 | Loss: 2.9633 | LR: 1.04e-04 | 1289.9ms/step | 15.5 samples/s +Step 30980/44000 | Loss: 2.9096 | LR: 1.04e-04 | 1290.8ms/step | 15.5 samples/s +Step 30990/44000 | Loss: 2.9148 | LR: 1.04e-04 | 1290.6ms/step | 15.5 samples/s +Step 31000/44000 | Loss: 2.9030 | LR: 1.03e-04 | 1291.0ms/step | 15.5 samples/s +Validation at step 31000: Loss = 2.9433, Perplexity = 18.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-31000 +Step 31010/44000 | Loss: 2.9126 | LR: 1.03e-04 | 3736.7ms/step | 5.4 samples/s +Step 31020/44000 | Loss: 2.9413 | LR: 1.03e-04 | 1285.0ms/step | 15.6 samples/s +Step 31030/44000 | Loss: 2.8942 | LR: 1.03e-04 | 1289.1ms/step | 15.5 samples/s +Step 31040/44000 | Loss: 2.9249 | LR: 1.03e-04 | 1288.5ms/step | 15.5 samples/s +Step 31050/44000 | Loss: 2.9365 | LR: 1.03e-04 | 1286.5ms/step | 15.5 samples/s +Step 31060/44000 | Loss: 2.9444 | LR: 1.03e-04 | 1287.6ms/step | 15.5 samples/s +Step 31070/44000 | Loss: 2.9310 | LR: 1.02e-04 | 1288.6ms/step | 15.5 samples/s +Step 31080/44000 | Loss: 2.9230 | LR: 1.02e-04 | 1289.4ms/step | 15.5 samples/s +Step 31090/44000 | Loss: 2.9190 | LR: 1.02e-04 | 1288.9ms/step | 15.5 samples/s +Step 31100/44000 | Loss: 2.9110 | LR: 1.02e-04 | 1290.1ms/step | 15.5 samples/s +Step 31110/44000 | Loss: 2.9133 | LR: 1.02e-04 | 1292.2ms/step | 15.5 samples/s +Step 31120/44000 | Loss: 2.9317 | LR: 1.02e-04 | 1295.3ms/step | 15.4 samples/s +Step 31130/44000 | Loss: 2.9614 | LR: 1.02e-04 | 1293.3ms/step | 15.5 samples/s +Step 31140/44000 | Loss: 2.9149 | LR: 1.01e-04 | 1297.8ms/step | 15.4 samples/s +Step 31150/44000 | Loss: 2.9598 | LR: 1.01e-04 | 1298.4ms/step | 15.4 samples/s +Step 31160/44000 | Loss: 2.9325 | LR: 1.01e-04 | 1298.7ms/step | 15.4 samples/s +Step 31170/44000 | Loss: 2.9252 | LR: 1.01e-04 | 1292.0ms/step | 15.5 samples/s +Step 31180/44000 | Loss: 2.9338 | LR: 1.01e-04 | 1291.1ms/step | 15.5 samples/s +Step 31190/44000 | Loss: 2.9053 | LR: 1.01e-04 | 1292.4ms/step | 15.5 samples/s +Step 31200/44000 | Loss: 2.9186 | LR: 1.00e-04 | 1292.6ms/step | 15.5 samples/s +Step 31210/44000 | Loss: 2.9364 | LR: 1.00e-04 | 1293.2ms/step | 15.5 samples/s +Step 31220/44000 | Loss: 2.9247 | LR: 1.00e-04 | 1290.9ms/step | 15.5 samples/s +Step 31230/44000 | Loss: 2.9587 | LR: 1.00e-04 | 1290.6ms/step | 15.5 samples/s +Step 31240/44000 | Loss: 2.9255 | LR: 9.99e-05 | 1290.0ms/step | 15.5 samples/s +Step 31250/44000 | Loss: 2.9099 | LR: 9.98e-05 | 1289.7ms/step | 15.5 samples/s +Validation at step 31250: Loss = 2.9427, Perplexity = 18.97 +Step 31260/44000 | Loss: 2.9400 | LR: 9.96e-05 | 3447.7ms/step | 5.8 samples/s +Step 31270/44000 | Loss: 2.9530 | LR: 9.95e-05 | 1291.2ms/step | 15.5 samples/s +Step 31280/44000 | Loss: 2.9139 | LR: 9.93e-05 | 1289.9ms/step | 15.5 samples/s +Step 31290/44000 | Loss: 2.9413 | LR: 9.92e-05 | 1288.4ms/step | 15.5 samples/s +Step 31300/44000 | Loss: 2.9392 | LR: 9.90e-05 | 1291.8ms/step | 15.5 samples/s +Step 31310/44000 | Loss: 2.9290 | LR: 9.89e-05 | 1291.3ms/step | 15.5 samples/s +Step 31320/44000 | Loss: 2.9244 | LR: 9.88e-05 | 1289.0ms/step | 15.5 samples/s +Step 31330/44000 | Loss: 2.9528 | LR: 9.86e-05 | 1290.6ms/step | 15.5 samples/s +Step 31340/44000 | Loss: 2.9202 | LR: 9.85e-05 | 1290.3ms/step | 15.5 samples/s +Step 31350/44000 | Loss: 2.8994 | LR: 9.83e-05 | 1289.4ms/step | 15.5 samples/s +Step 31360/44000 | Loss: 2.9081 | LR: 9.82e-05 | 1290.6ms/step | 15.5 samples/s +Step 31370/44000 | Loss: 2.9405 | LR: 9.80e-05 | 1290.3ms/step | 15.5 samples/s +Step 31380/44000 | Loss: 2.9543 | LR: 9.79e-05 | 1291.1ms/step | 15.5 samples/s +Step 31390/44000 | Loss: 2.8925 | LR: 9.77e-05 | 1291.4ms/step | 15.5 samples/s +Step 31400/44000 | Loss: 2.9263 | LR: 9.76e-05 | 1291.9ms/step | 15.5 samples/s +Step 31410/44000 | Loss: 2.8981 | LR: 9.75e-05 | 1290.4ms/step | 15.5 samples/s +Step 31420/44000 | Loss: 2.9167 | LR: 9.73e-05 | 1289.5ms/step | 15.5 samples/s +Step 31430/44000 | Loss: 2.9393 | LR: 9.72e-05 | 1288.1ms/step | 15.5 samples/s +Step 31440/44000 | Loss: 2.9243 | LR: 9.70e-05 | 1287.8ms/step | 15.5 samples/s +Step 31450/44000 | Loss: 2.9340 | LR: 9.69e-05 | 1291.6ms/step | 15.5 samples/s +Step 31460/44000 | Loss: 2.9137 | LR: 9.67e-05 | 1290.4ms/step | 15.5 samples/s +Step 31470/44000 | Loss: 2.9425 | LR: 9.66e-05 | 1290.2ms/step | 15.5 samples/s +Step 31480/44000 | Loss: 2.9056 | LR: 9.65e-05 | 1290.5ms/step | 15.5 samples/s +Step 31490/44000 | Loss: 2.9155 | LR: 9.63e-05 | 1290.1ms/step | 15.5 samples/s +Step 31500/44000 | Loss: 2.9280 | LR: 9.62e-05 | 1288.3ms/step | 15.5 samples/s +Validation at step 31500: Loss = 2.9420, Perplexity = 18.95 +Step 31510/44000 | Loss: 2.9089 | LR: 9.60e-05 | 3444.4ms/step | 5.8 samples/s +Step 31520/44000 | Loss: 2.9197 | LR: 9.59e-05 | 1286.5ms/step | 15.5 samples/s +Step 31530/44000 | Loss: 2.9333 | LR: 9.57e-05 | 1291.3ms/step | 15.5 samples/s +Step 31540/44000 | Loss: 2.9145 | LR: 9.56e-05 | 1288.9ms/step | 15.5 samples/s +Step 31550/44000 | Loss: 2.9399 | LR: 9.54e-05 | 1290.8ms/step | 15.5 samples/s +Step 31560/44000 | Loss: 2.9258 | LR: 9.53e-05 | 1291.8ms/step | 15.5 samples/s +Step 31570/44000 | Loss: 2.9156 | LR: 9.52e-05 | 1291.6ms/step | 15.5 samples/s +Step 31580/44000 | Loss: 2.9391 | LR: 9.50e-05 | 1290.9ms/step | 15.5 samples/s +Step 31590/44000 | Loss: 2.9413 | LR: 9.49e-05 | 1296.4ms/step | 15.4 samples/s +Step 31600/44000 | Loss: 2.9229 | LR: 9.47e-05 | 1290.5ms/step | 15.5 samples/s +Step 31610/44000 | Loss: 2.9126 | LR: 9.46e-05 | 1291.2ms/step | 15.5 samples/s +Step 31620/44000 | Loss: 2.9357 | LR: 9.45e-05 | 1292.0ms/step | 15.5 samples/s +Step 31630/44000 | Loss: 2.9195 | LR: 9.43e-05 | 1290.6ms/step | 15.5 samples/s +Step 31640/44000 | Loss: 2.8933 | LR: 9.42e-05 | 1291.1ms/step | 15.5 samples/s +Step 31650/44000 | Loss: 2.8981 | LR: 9.40e-05 | 1291.1ms/step | 15.5 samples/s +Step 31660/44000 | Loss: 2.9141 | LR: 9.39e-05 | 1290.4ms/step | 15.5 samples/s +Step 31670/44000 | Loss: 2.9081 | LR: 9.37e-05 | 1289.7ms/step | 15.5 samples/s +Step 31680/44000 | Loss: 2.9184 | LR: 9.36e-05 | 1288.3ms/step | 15.5 samples/s +Step 31690/44000 | Loss: 2.9299 | LR: 9.35e-05 | 1291.0ms/step | 15.5 samples/s +Step 31700/44000 | Loss: 2.9437 | LR: 9.33e-05 | 1292.5ms/step | 15.5 samples/s +Step 31710/44000 | Loss: 2.9204 | LR: 9.32e-05 | 1293.5ms/step | 15.5 samples/s +Step 31720/44000 | Loss: 2.9194 | LR: 9.30e-05 | 1292.5ms/step | 15.5 samples/s +Step 31730/44000 | Loss: 2.8933 | LR: 9.29e-05 | 1291.1ms/step | 15.5 samples/s +Step 31740/44000 | Loss: 2.9275 | LR: 9.28e-05 | 1289.5ms/step | 15.5 samples/s +Step 31750/44000 | Loss: 2.9361 | LR: 9.26e-05 | 1288.1ms/step | 15.5 samples/s +Validation at step 31750: Loss = 2.9413, Perplexity = 18.94 +Step 31760/44000 | Loss: 2.9103 | LR: 9.25e-05 | 3451.0ms/step | 5.8 samples/s +Step 31770/44000 | Loss: 2.9358 | LR: 9.23e-05 | 1293.2ms/step | 15.5 samples/s +Step 31780/44000 | Loss: 2.9268 | LR: 9.22e-05 | 1290.8ms/step | 15.5 samples/s +Step 31790/44000 | Loss: 2.9286 | LR: 9.20e-05 | 1293.5ms/step | 15.5 samples/s +Step 31800/44000 | Loss: 2.8990 | LR: 9.19e-05 | 1291.7ms/step | 15.5 samples/s +Step 31810/44000 | Loss: 2.8989 | LR: 9.18e-05 | 1289.4ms/step | 15.5 samples/s +Step 31820/44000 | Loss: 2.9167 | LR: 9.16e-05 | 1291.7ms/step | 15.5 samples/s +Step 31830/44000 | Loss: 2.9218 | LR: 9.15e-05 | 1290.5ms/step | 15.5 samples/s +Step 31840/44000 | Loss: 2.9349 | LR: 9.13e-05 | 1289.7ms/step | 15.5 samples/s +Step 31850/44000 | Loss: 2.9378 | LR: 9.12e-05 | 1290.2ms/step | 15.5 samples/s +Step 31860/44000 | Loss: 2.9446 | LR: 9.11e-05 | 1289.0ms/step | 15.5 samples/s +Step 31870/44000 | Loss: 2.9210 | LR: 9.09e-05 | 1290.1ms/step | 15.5 samples/s +Step 31880/44000 | Loss: 2.9552 | LR: 9.08e-05 | 1289.5ms/step | 15.5 samples/s +Step 31890/44000 | Loss: 2.8964 | LR: 9.06e-05 | 1289.1ms/step | 15.5 samples/s +Step 31900/44000 | Loss: 2.9149 | LR: 9.05e-05 | 1289.4ms/step | 15.5 samples/s +Step 31910/44000 | Loss: 2.9472 | LR: 9.04e-05 | 1292.9ms/step | 15.5 samples/s +Step 31920/44000 | Loss: 2.8747 | LR: 9.02e-05 | 1290.1ms/step | 15.5 samples/s +Step 31930/44000 | Loss: 2.9303 | LR: 9.01e-05 | 1289.1ms/step | 15.5 samples/s +Step 31940/44000 | Loss: 2.9390 | LR: 8.99e-05 | 1289.5ms/step | 15.5 samples/s +Step 31950/44000 | Loss: 2.9278 | LR: 8.98e-05 | 1287.4ms/step | 15.5 samples/s +Step 31960/44000 | Loss: 2.9427 | LR: 8.97e-05 | 1288.9ms/step | 15.5 samples/s +Step 31970/44000 | Loss: 2.9099 | LR: 8.95e-05 | 1291.1ms/step | 15.5 samples/s +Step 31980/44000 | Loss: 2.9231 | LR: 8.94e-05 | 1291.6ms/step | 15.5 samples/s +Step 31990/44000 | Loss: 2.8895 | LR: 8.92e-05 | 1290.4ms/step | 15.5 samples/s +Step 32000/44000 | Loss: 2.9131 | LR: 8.91e-05 | 1288.5ms/step | 15.5 samples/s +Validation at step 32000: Loss = 2.9406, Perplexity = 18.93 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-32000 +Step 32010/44000 | Loss: 2.9355 | LR: 8.90e-05 | 3733.8ms/step | 5.4 samples/s +Step 32020/44000 | Loss: 2.9239 | LR: 8.88e-05 | 1289.7ms/step | 15.5 samples/s +Step 32030/44000 | Loss: 2.9314 | LR: 8.87e-05 | 1291.5ms/step | 15.5 samples/s +Step 32040/44000 | Loss: 2.9333 | LR: 8.86e-05 | 1290.7ms/step | 15.5 samples/s +Step 32050/44000 | Loss: 2.9091 | LR: 8.84e-05 | 1291.9ms/step | 15.5 samples/s +Step 32060/44000 | Loss: 2.8935 | LR: 8.83e-05 | 1290.3ms/step | 15.5 samples/s +Step 32070/44000 | Loss: 2.9186 | LR: 8.81e-05 | 1292.3ms/step | 15.5 samples/s +Step 32080/44000 | Loss: 2.8941 | LR: 8.80e-05 | 1291.3ms/step | 15.5 samples/s +Step 32090/44000 | Loss: 2.9281 | LR: 8.79e-05 | 1291.0ms/step | 15.5 samples/s +Step 32100/44000 | Loss: 2.9302 | LR: 8.77e-05 | 1291.1ms/step | 15.5 samples/s +Step 32110/44000 | Loss: 2.9127 | LR: 8.76e-05 | 1291.8ms/step | 15.5 samples/s +Step 32120/44000 | Loss: 2.9169 | LR: 8.74e-05 | 1291.4ms/step | 15.5 samples/s +Step 32130/44000 | Loss: 2.9108 | LR: 8.73e-05 | 1293.6ms/step | 15.5 samples/s +Step 32140/44000 | Loss: 2.9084 | LR: 8.72e-05 | 1290.8ms/step | 15.5 samples/s +Step 32150/44000 | Loss: 2.9257 | LR: 8.70e-05 | 1288.8ms/step | 15.5 samples/s +Step 32160/44000 | Loss: 2.8958 | LR: 8.69e-05 | 1287.9ms/step | 15.5 samples/s +Step 32170/44000 | Loss: 2.9197 | LR: 8.68e-05 | 1289.0ms/step | 15.5 samples/s +Step 32180/44000 | Loss: 2.9523 | LR: 8.66e-05 | 1288.2ms/step | 15.5 samples/s +Step 32190/44000 | Loss: 2.9242 | LR: 8.65e-05 | 1290.2ms/step | 15.5 samples/s +Step 32200/44000 | Loss: 2.9379 | LR: 8.63e-05 | 1286.8ms/step | 15.5 samples/s +Step 32210/44000 | Loss: 2.9432 | LR: 8.62e-05 | 1287.1ms/step | 15.5 samples/s +Step 32220/44000 | Loss: 2.8983 | LR: 8.61e-05 | 1291.7ms/step | 15.5 samples/s +Step 32230/44000 | Loss: 2.9270 | LR: 8.59e-05 | 1292.4ms/step | 15.5 samples/s +Step 32240/44000 | Loss: 2.9163 | LR: 8.58e-05 | 1292.2ms/step | 15.5 samples/s +Step 32250/44000 | Loss: 2.9350 | LR: 8.57e-05 | 1289.0ms/step | 15.5 samples/s +Validation at step 32250: Loss = 2.9401, Perplexity = 18.92 +Step 32260/44000 | Loss: 2.9485 | LR: 8.55e-05 | 3445.1ms/step | 5.8 samples/s +Step 32270/44000 | Loss: 2.9232 | LR: 8.54e-05 | 1290.0ms/step | 15.5 samples/s +Step 32280/44000 | Loss: 2.9448 | LR: 8.53e-05 | 1289.8ms/step | 15.5 samples/s +Step 32290/44000 | Loss: 2.9133 | LR: 8.51e-05 | 1290.8ms/step | 15.5 samples/s +Step 32300/44000 | Loss: 2.9094 | LR: 8.50e-05 | 1290.0ms/step | 15.5 samples/s +Step 32310/44000 | Loss: 2.9330 | LR: 8.48e-05 | 1291.5ms/step | 15.5 samples/s +Step 32320/44000 | Loss: 2.9499 | LR: 8.47e-05 | 1292.1ms/step | 15.5 samples/s +Step 32330/44000 | Loss: 2.9196 | LR: 8.46e-05 | 1289.6ms/step | 15.5 samples/s +Step 32340/44000 | Loss: 2.9166 | LR: 8.44e-05 | 1293.2ms/step | 15.5 samples/s +Step 32350/44000 | Loss: 2.9395 | LR: 8.43e-05 | 1290.8ms/step | 15.5 samples/s +Step 32360/44000 | Loss: 2.9187 | LR: 8.42e-05 | 1287.5ms/step | 15.5 samples/s +Step 32370/44000 | Loss: 2.9176 | LR: 8.40e-05 | 1288.6ms/step | 15.5 samples/s +Step 32380/44000 | Loss: 2.9180 | LR: 8.39e-05 | 1289.9ms/step | 15.5 samples/s +Step 32390/44000 | Loss: 2.9197 | LR: 8.38e-05 | 1291.4ms/step | 15.5 samples/s +Step 32400/44000 | Loss: 2.9259 | LR: 8.36e-05 | 1292.2ms/step | 15.5 samples/s +Step 32410/44000 | Loss: 2.9213 | LR: 8.35e-05 | 1293.1ms/step | 15.5 samples/s +Step 32420/44000 | Loss: 2.9179 | LR: 8.33e-05 | 1289.9ms/step | 15.5 samples/s +Step 32430/44000 | Loss: 2.9670 | LR: 8.32e-05 | 1289.0ms/step | 15.5 samples/s +Step 32440/44000 | Loss: 2.9329 | LR: 8.31e-05 | 1290.0ms/step | 15.5 samples/s +Step 32450/44000 | Loss: 2.9248 | LR: 8.29e-05 | 1290.2ms/step | 15.5 samples/s +Step 32460/44000 | Loss: 2.9459 | LR: 8.28e-05 | 1294.0ms/step | 15.5 samples/s +Step 32470/44000 | Loss: 2.9021 | LR: 8.27e-05 | 1293.5ms/step | 15.5 samples/s +Step 32480/44000 | Loss: 2.9350 | LR: 8.25e-05 | 1292.6ms/step | 15.5 samples/s +Step 32490/44000 | Loss: 2.9118 | LR: 8.24e-05 | 1292.0ms/step | 15.5 samples/s +Step 32500/44000 | Loss: 2.9299 | LR: 8.23e-05 | 1291.6ms/step | 15.5 samples/s +Validation at step 32500: Loss = 2.9396, Perplexity = 18.91 +Step 32510/44000 | Loss: 2.8983 | LR: 8.21e-05 | 3454.8ms/step | 5.8 samples/s +Step 32520/44000 | Loss: 2.9227 | LR: 8.20e-05 | 1294.6ms/step | 15.4 samples/s +Step 32530/44000 | Loss: 2.9527 | LR: 8.19e-05 | 1288.6ms/step | 15.5 samples/s +Step 32540/44000 | Loss: 2.9090 | LR: 8.17e-05 | 1291.9ms/step | 15.5 samples/s +Step 32550/44000 | Loss: 2.8793 | LR: 8.16e-05 | 1294.5ms/step | 15.4 samples/s +Step 32560/44000 | Loss: 2.9444 | LR: 8.15e-05 | 1294.7ms/step | 15.4 samples/s +Step 32570/44000 | Loss: 2.9259 | LR: 8.13e-05 | 1293.8ms/step | 15.5 samples/s +Step 32580/44000 | Loss: 2.9311 | LR: 8.12e-05 | 1291.7ms/step | 15.5 samples/s +Step 32590/44000 | Loss: 2.9548 | LR: 8.11e-05 | 1290.6ms/step | 15.5 samples/s +Step 32600/44000 | Loss: 2.9297 | LR: 8.09e-05 | 1289.5ms/step | 15.5 samples/s +Step 32610/44000 | Loss: 2.9320 | LR: 8.08e-05 | 1291.2ms/step | 15.5 samples/s +Step 32620/44000 | Loss: 2.9408 | LR: 8.07e-05 | 1291.4ms/step | 15.5 samples/s +Step 32630/44000 | Loss: 2.8997 | LR: 8.05e-05 | 1290.4ms/step | 15.5 samples/s +Step 32640/44000 | Loss: 2.9619 | LR: 8.04e-05 | 1289.9ms/step | 15.5 samples/s +Step 32650/44000 | Loss: 2.9399 | LR: 8.03e-05 | 1290.5ms/step | 15.5 samples/s +Step 32660/44000 | Loss: 2.9266 | LR: 8.01e-05 | 1289.6ms/step | 15.5 samples/s +Step 32670/44000 | Loss: 2.9644 | LR: 8.00e-05 | 1291.4ms/step | 15.5 samples/s +Step 32680/44000 | Loss: 2.9201 | LR: 7.99e-05 | 1289.2ms/step | 15.5 samples/s +Step 32690/44000 | Loss: 2.9319 | LR: 7.97e-05 | 1290.5ms/step | 15.5 samples/s +Step 32700/44000 | Loss: 2.9118 | LR: 7.96e-05 | 1289.9ms/step | 15.5 samples/s +Step 32710/44000 | Loss: 2.9400 | LR: 7.95e-05 | 1289.7ms/step | 15.5 samples/s +Step 32720/44000 | Loss: 2.9392 | LR: 7.93e-05 | 1288.9ms/step | 15.5 samples/s +Step 32730/44000 | Loss: 2.9168 | LR: 7.92e-05 | 1288.2ms/step | 15.5 samples/s +Step 32740/44000 | Loss: 2.9065 | LR: 7.91e-05 | 1289.6ms/step | 15.5 samples/s +Step 32750/44000 | Loss: 2.8985 | LR: 7.89e-05 | 1289.1ms/step | 15.5 samples/s +Validation at step 32750: Loss = 2.9392, Perplexity = 18.90 +Step 32760/44000 | Loss: 2.9321 | LR: 7.88e-05 | 3451.7ms/step | 5.8 samples/s +Step 32770/44000 | Loss: 2.9281 | LR: 7.87e-05 | 1288.8ms/step | 15.5 samples/s +Step 32780/44000 | Loss: 2.9539 | LR: 7.85e-05 | 1292.0ms/step | 15.5 samples/s +Step 32790/44000 | Loss: 2.9599 | LR: 7.84e-05 | 1289.5ms/step | 15.5 samples/s +Step 32800/44000 | Loss: 2.9148 | LR: 7.83e-05 | 1290.4ms/step | 15.5 samples/s +Step 32810/44000 | Loss: 2.9244 | LR: 7.81e-05 | 1290.3ms/step | 15.5 samples/s +Step 32820/44000 | Loss: 2.9430 | LR: 7.80e-05 | 1289.2ms/step | 15.5 samples/s +Step 32830/44000 | Loss: 2.9396 | LR: 7.79e-05 | 1291.9ms/step | 15.5 samples/s +Step 32840/44000 | Loss: 2.8914 | LR: 7.77e-05 | 1290.3ms/step | 15.5 samples/s +Step 32850/44000 | Loss: 2.9211 | LR: 7.76e-05 | 1288.8ms/step | 15.5 samples/s +Step 32860/44000 | Loss: 2.9195 | LR: 7.75e-05 | 1288.8ms/step | 15.5 samples/s +Step 32870/44000 | Loss: 2.9361 | LR: 7.73e-05 | 1290.0ms/step | 15.5 samples/s +Step 32880/44000 | Loss: 2.8981 | LR: 7.72e-05 | 1290.8ms/step | 15.5 samples/s +Step 32890/44000 | Loss: 2.9342 | LR: 7.71e-05 | 1292.1ms/step | 15.5 samples/s +Step 32900/44000 | Loss: 2.9301 | LR: 7.70e-05 | 1297.0ms/step | 15.4 samples/s +Step 32910/44000 | Loss: 2.9206 | LR: 7.68e-05 | 1290.3ms/step | 15.5 samples/s +Step 32920/44000 | Loss: 2.9453 | LR: 7.67e-05 | 1290.8ms/step | 15.5 samples/s +Step 32930/44000 | Loss: 2.9179 | LR: 7.66e-05 | 1290.0ms/step | 15.5 samples/s +Step 32940/44000 | Loss: 2.9038 | LR: 7.64e-05 | 1290.6ms/step | 15.5 samples/s +Step 32950/44000 | Loss: 2.9097 | LR: 7.63e-05 | 1292.0ms/step | 15.5 samples/s +Step 32960/44000 | Loss: 2.9167 | LR: 7.62e-05 | 1291.9ms/step | 15.5 samples/s +Step 32970/44000 | Loss: 2.9225 | LR: 7.60e-05 | 1288.9ms/step | 15.5 samples/s +Step 32980/44000 | Loss: 2.9250 | LR: 7.59e-05 | 1289.7ms/step | 15.5 samples/s +Step 32990/44000 | Loss: 2.9269 | LR: 7.58e-05 | 1292.9ms/step | 15.5 samples/s +Step 33000/44000 | Loss: 2.9486 | LR: 7.56e-05 | 1296.5ms/step | 15.4 samples/s +Validation at step 33000: Loss = 2.9387, Perplexity = 18.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-33000 +Step 33010/44000 | Loss: 2.9365 | LR: 7.55e-05 | 3757.7ms/step | 5.3 samples/s +Step 33020/44000 | Loss: 2.9008 | LR: 7.54e-05 | 1287.5ms/step | 15.5 samples/s +Step 33030/44000 | Loss: 2.9444 | LR: 7.53e-05 | 1289.0ms/step | 15.5 samples/s +Step 33040/44000 | Loss: 2.9293 | LR: 7.51e-05 | 1288.2ms/step | 15.5 samples/s +Step 33050/44000 | Loss: 2.9381 | LR: 7.50e-05 | 1286.6ms/step | 15.5 samples/s +Step 33060/44000 | Loss: 2.9083 | LR: 7.49e-05 | 1290.6ms/step | 15.5 samples/s +Step 33070/44000 | Loss: 2.9376 | LR: 7.47e-05 | 1290.8ms/step | 15.5 samples/s +Step 33080/44000 | Loss: 2.9381 | LR: 7.46e-05 | 1289.7ms/step | 15.5 samples/s +Step 33090/44000 | Loss: 2.9276 | LR: 7.45e-05 | 1291.4ms/step | 15.5 samples/s +Step 33100/44000 | Loss: 2.9017 | LR: 7.44e-05 | 1291.4ms/step | 15.5 samples/s +Step 33110/44000 | Loss: 2.8783 | LR: 7.42e-05 | 1288.7ms/step | 15.5 samples/s +Step 33120/44000 | Loss: 2.9098 | LR: 7.41e-05 | 1289.2ms/step | 15.5 samples/s +Step 33130/44000 | Loss: 2.9275 | LR: 7.40e-05 | 1287.9ms/step | 15.5 samples/s +Step 33140/44000 | Loss: 2.9452 | LR: 7.38e-05 | 1285.8ms/step | 15.6 samples/s +Step 33150/44000 | Loss: 2.9360 | LR: 7.37e-05 | 1286.0ms/step | 15.6 samples/s +Step 33160/44000 | Loss: 2.9118 | LR: 7.36e-05 | 1288.3ms/step | 15.5 samples/s +Step 33170/44000 | Loss: 2.9187 | LR: 7.34e-05 | 1288.3ms/step | 15.5 samples/s +Step 33180/44000 | Loss: 2.9432 | LR: 7.33e-05 | 1287.9ms/step | 15.5 samples/s +Step 33190/44000 | Loss: 2.9427 | LR: 7.32e-05 | 1288.9ms/step | 15.5 samples/s +Step 33200/44000 | Loss: 2.9314 | LR: 7.31e-05 | 1291.7ms/step | 15.5 samples/s +Step 33210/44000 | Loss: 2.9186 | LR: 7.29e-05 | 1290.4ms/step | 15.5 samples/s +Step 33220/44000 | Loss: 2.9372 | LR: 7.28e-05 | 1289.0ms/step | 15.5 samples/s +Step 33230/44000 | Loss: 2.9199 | LR: 7.27e-05 | 1288.9ms/step | 15.5 samples/s +Step 33240/44000 | Loss: 2.9142 | LR: 7.26e-05 | 1287.6ms/step | 15.5 samples/s +Step 33250/44000 | Loss: 2.9228 | LR: 7.24e-05 | 1288.1ms/step | 15.5 samples/s +Validation at step 33250: Loss = 2.9384, Perplexity = 18.89 +Step 33260/44000 | Loss: 2.9365 | LR: 7.23e-05 | 3441.3ms/step | 5.8 samples/s +Step 33270/44000 | Loss: 2.9170 | LR: 7.22e-05 | 1291.1ms/step | 15.5 samples/s +Step 33280/44000 | Loss: 2.9354 | LR: 7.20e-05 | 1291.0ms/step | 15.5 samples/s +Step 33290/44000 | Loss: 2.8842 | LR: 7.19e-05 | 1291.2ms/step | 15.5 samples/s +Step 33300/44000 | Loss: 2.9252 | LR: 7.18e-05 | 1292.8ms/step | 15.5 samples/s +Step 33310/44000 | Loss: 2.9415 | LR: 7.17e-05 | 1291.8ms/step | 15.5 samples/s +Step 33320/44000 | Loss: 2.9233 | LR: 7.15e-05 | 1289.8ms/step | 15.5 samples/s +Step 33330/44000 | Loss: 2.9133 | LR: 7.14e-05 | 1288.5ms/step | 15.5 samples/s +Step 33340/44000 | Loss: 2.9156 | LR: 7.13e-05 | 1290.3ms/step | 15.5 samples/s +Step 33350/44000 | Loss: 2.9248 | LR: 7.11e-05 | 1288.9ms/step | 15.5 samples/s +Step 33360/44000 | Loss: 2.9365 | LR: 7.10e-05 | 1291.0ms/step | 15.5 samples/s +Step 33370/44000 | Loss: 2.8960 | LR: 7.09e-05 | 1288.8ms/step | 15.5 samples/s +Step 33380/44000 | Loss: 2.9122 | LR: 7.08e-05 | 1287.2ms/step | 15.5 samples/s +Step 33390/44000 | Loss: 2.9439 | LR: 7.06e-05 | 1288.4ms/step | 15.5 samples/s +Step 33400/44000 | Loss: 2.8910 | LR: 7.05e-05 | 1287.3ms/step | 15.5 samples/s +Step 33410/44000 | Loss: 2.9380 | LR: 7.04e-05 | 1285.6ms/step | 15.6 samples/s +Step 33420/44000 | Loss: 2.9167 | LR: 7.03e-05 | 1286.9ms/step | 15.5 samples/s +Step 33430/44000 | Loss: 2.9139 | LR: 7.01e-05 | 1290.4ms/step | 15.5 samples/s +Step 33440/44000 | Loss: 2.9407 | LR: 7.00e-05 | 1289.7ms/step | 15.5 samples/s +Step 33450/44000 | Loss: 2.9354 | LR: 6.99e-05 | 1289.2ms/step | 15.5 samples/s +Step 33460/44000 | Loss: 2.9288 | LR: 6.98e-05 | 1290.3ms/step | 15.5 samples/s +Step 33470/44000 | Loss: 2.9211 | LR: 6.96e-05 | 1288.8ms/step | 15.5 samples/s +Step 33480/44000 | Loss: 2.9068 | LR: 6.95e-05 | 1287.0ms/step | 15.5 samples/s +Step 33490/44000 | Loss: 2.9380 | LR: 6.94e-05 | 1290.0ms/step | 15.5 samples/s +Step 33500/44000 | Loss: 2.8957 | LR: 6.93e-05 | 1288.3ms/step | 15.5 samples/s +Validation at step 33500: Loss = 2.9379, Perplexity = 18.88 +Step 33510/44000 | Loss: 2.9026 | LR: 6.91e-05 | 3443.8ms/step | 5.8 samples/s +Step 33520/44000 | Loss: 2.9407 | LR: 6.90e-05 | 1289.3ms/step | 15.5 samples/s +Step 33530/44000 | Loss: 2.9027 | LR: 6.89e-05 | 1290.6ms/step | 15.5 samples/s +Step 33540/44000 | Loss: 2.9108 | LR: 6.88e-05 | 1292.4ms/step | 15.5 samples/s +Step 33550/44000 | Loss: 2.9196 | LR: 6.86e-05 | 1292.5ms/step | 15.5 samples/s +Step 33560/44000 | Loss: 2.9001 | LR: 6.85e-05 | 1290.0ms/step | 15.5 samples/s +Step 33570/44000 | Loss: 2.8946 | LR: 6.84e-05 | 1290.9ms/step | 15.5 samples/s +Step 33580/44000 | Loss: 2.9410 | LR: 6.83e-05 | 1289.5ms/step | 15.5 samples/s +Step 33590/44000 | Loss: 2.9376 | LR: 6.81e-05 | 1288.5ms/step | 15.5 samples/s +Step 33600/44000 | Loss: 2.9432 | LR: 6.80e-05 | 1287.7ms/step | 15.5 samples/s +Step 33610/44000 | Loss: 2.8988 | LR: 6.79e-05 | 1290.9ms/step | 15.5 samples/s +Step 33620/44000 | Loss: 2.9147 | LR: 6.78e-05 | 1291.8ms/step | 15.5 samples/s +Step 33630/44000 | Loss: 2.9024 | LR: 6.76e-05 | 1289.6ms/step | 15.5 samples/s +Step 33640/44000 | Loss: 2.9168 | LR: 6.75e-05 | 1288.9ms/step | 15.5 samples/s +Step 33650/44000 | Loss: 2.9241 | LR: 6.74e-05 | 1292.2ms/step | 15.5 samples/s +Step 33660/44000 | Loss: 2.9294 | LR: 6.73e-05 | 1290.1ms/step | 15.5 samples/s +Step 33670/44000 | Loss: 2.9378 | LR: 6.71e-05 | 1291.1ms/step | 15.5 samples/s +Step 33680/44000 | Loss: 2.9228 | LR: 6.70e-05 | 1291.0ms/step | 15.5 samples/s +Step 33690/44000 | Loss: 2.9197 | LR: 6.69e-05 | 1290.4ms/step | 15.5 samples/s +Step 33700/44000 | Loss: 2.9360 | LR: 6.68e-05 | 1289.8ms/step | 15.5 samples/s +Step 33710/44000 | Loss: 2.9502 | LR: 6.66e-05 | 1289.6ms/step | 15.5 samples/s +Step 33720/44000 | Loss: 2.9272 | LR: 6.65e-05 | 1290.7ms/step | 15.5 samples/s +Step 33730/44000 | Loss: 2.9117 | LR: 6.64e-05 | 1290.6ms/step | 15.5 samples/s +Step 33740/44000 | Loss: 2.9260 | LR: 6.63e-05 | 1289.8ms/step | 15.5 samples/s +Step 33750/44000 | Loss: 2.9318 | LR: 6.61e-05 | 1289.8ms/step | 15.5 samples/s +Validation at step 33750: Loss = 2.9376, Perplexity = 18.87 +Step 33760/44000 | Loss: 2.8969 | LR: 6.60e-05 | 3443.2ms/step | 5.8 samples/s +Step 33770/44000 | Loss: 2.9442 | LR: 6.59e-05 | 1287.6ms/step | 15.5 samples/s +Step 33780/44000 | Loss: 2.9465 | LR: 6.58e-05 | 1285.7ms/step | 15.6 samples/s +Step 33790/44000 | Loss: 2.9002 | LR: 6.57e-05 | 1288.0ms/step | 15.5 samples/s +Step 33800/44000 | Loss: 2.9332 | LR: 6.55e-05 | 1296.2ms/step | 15.4 samples/s +Step 33810/44000 | Loss: 2.9507 | LR: 6.54e-05 | 1291.5ms/step | 15.5 samples/s +Step 33820/44000 | Loss: 2.9234 | LR: 6.53e-05 | 1289.2ms/step | 15.5 samples/s +Step 33830/44000 | Loss: 2.9441 | LR: 6.52e-05 | 1293.3ms/step | 15.5 samples/s +Step 33840/44000 | Loss: 2.9256 | LR: 6.50e-05 | 1293.2ms/step | 15.5 samples/s +Step 33850/44000 | Loss: 2.9351 | LR: 6.49e-05 | 1293.4ms/step | 15.5 samples/s +Step 33860/44000 | Loss: 2.9184 | LR: 6.48e-05 | 1296.0ms/step | 15.4 samples/s +Step 33870/44000 | Loss: 2.9284 | LR: 6.47e-05 | 1292.6ms/step | 15.5 samples/s +Step 33880/44000 | Loss: 2.9593 | LR: 6.46e-05 | 1293.6ms/step | 15.5 samples/s +Step 33890/44000 | Loss: 2.9155 | LR: 6.44e-05 | 1292.4ms/step | 15.5 samples/s +Step 33900/44000 | Loss: 2.9424 | LR: 6.43e-05 | 1294.1ms/step | 15.5 samples/s +Step 33910/44000 | Loss: 2.9316 | LR: 6.42e-05 | 1290.5ms/step | 15.5 samples/s +Step 33920/44000 | Loss: 2.9245 | LR: 6.41e-05 | 1291.0ms/step | 15.5 samples/s +Step 33930/44000 | Loss: 2.8832 | LR: 6.40e-05 | 1288.4ms/step | 15.5 samples/s +Step 33940/44000 | Loss: 2.9191 | LR: 6.38e-05 | 1289.5ms/step | 15.5 samples/s +Step 33950/44000 | Loss: 2.9161 | LR: 6.37e-05 | 1291.0ms/step | 15.5 samples/s +Step 33960/44000 | Loss: 2.9050 | LR: 6.36e-05 | 1290.9ms/step | 15.5 samples/s +Step 33970/44000 | Loss: 2.9132 | LR: 6.35e-05 | 1291.8ms/step | 15.5 samples/s +Step 33980/44000 | Loss: 2.9469 | LR: 6.33e-05 | 1292.9ms/step | 15.5 samples/s +Step 33990/44000 | Loss: 2.9399 | LR: 6.32e-05 | 1291.2ms/step | 15.5 samples/s +Step 34000/44000 | Loss: 2.9427 | LR: 6.31e-05 | 1291.7ms/step | 15.5 samples/s +Validation at step 34000: Loss = 2.9372, Perplexity = 18.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-34000 +Step 34010/44000 | Loss: 2.9210 | LR: 6.30e-05 | 3734.9ms/step | 5.4 samples/s +Step 34020/44000 | Loss: 2.9141 | LR: 6.29e-05 | 1291.7ms/step | 15.5 samples/s +Step 34030/44000 | Loss: 2.9648 | LR: 6.27e-05 | 1289.5ms/step | 15.5 samples/s +Step 34040/44000 | Loss: 2.9154 | LR: 6.26e-05 | 1287.8ms/step | 15.5 samples/s +Step 34050/44000 | Loss: 2.9164 | LR: 6.25e-05 | 1289.1ms/step | 15.5 samples/s +Step 34060/44000 | Loss: 2.9221 | LR: 6.24e-05 | 1288.8ms/step | 15.5 samples/s +Step 34070/44000 | Loss: 2.9460 | LR: 6.23e-05 | 1289.8ms/step | 15.5 samples/s +Step 34080/44000 | Loss: 2.9277 | LR: 6.21e-05 | 1289.1ms/step | 15.5 samples/s +Step 34090/44000 | Loss: 2.9101 | LR: 6.20e-05 | 1289.1ms/step | 15.5 samples/s +Step 34100/44000 | Loss: 2.9263 | LR: 6.19e-05 | 1290.9ms/step | 15.5 samples/s +Step 34110/44000 | Loss: 2.8940 | LR: 6.18e-05 | 1290.8ms/step | 15.5 samples/s +Step 34120/44000 | Loss: 2.9187 | LR: 6.17e-05 | 1291.9ms/step | 15.5 samples/s +Step 34130/44000 | Loss: 2.9226 | LR: 6.15e-05 | 1291.6ms/step | 15.5 samples/s +Step 34140/44000 | Loss: 2.9324 | LR: 6.14e-05 | 1289.3ms/step | 15.5 samples/s +Step 34150/44000 | Loss: 2.9268 | LR: 6.13e-05 | 1290.7ms/step | 15.5 samples/s +Step 34160/44000 | Loss: 2.9548 | LR: 6.12e-05 | 1287.9ms/step | 15.5 samples/s +Step 34170/44000 | Loss: 2.9324 | LR: 6.11e-05 | 1289.1ms/step | 15.5 samples/s +Step 34180/44000 | Loss: 2.9086 | LR: 6.09e-05 | 1288.8ms/step | 15.5 samples/s +Step 34190/44000 | Loss: 2.9312 | LR: 6.08e-05 | 1289.1ms/step | 15.5 samples/s +Step 34200/44000 | Loss: 2.9505 | LR: 6.07e-05 | 1289.9ms/step | 15.5 samples/s +Step 34210/44000 | Loss: 2.9307 | LR: 6.06e-05 | 1288.6ms/step | 15.5 samples/s +Step 34220/44000 | Loss: 2.9224 | LR: 6.05e-05 | 1288.2ms/step | 15.5 samples/s +Step 34230/44000 | Loss: 2.9393 | LR: 6.04e-05 | 1287.0ms/step | 15.5 samples/s +Step 34240/44000 | Loss: 2.9565 | LR: 6.02e-05 | 1288.6ms/step | 15.5 samples/s +Step 34250/44000 | Loss: 2.9344 | LR: 6.01e-05 | 1288.6ms/step | 15.5 samples/s +Validation at step 34250: Loss = 2.9369, Perplexity = 18.86 +Step 34260/44000 | Loss: 2.9308 | LR: 6.00e-05 | 3447.3ms/step | 5.8 samples/s +Step 34270/44000 | Loss: 2.9053 | LR: 5.99e-05 | 1290.0ms/step | 15.5 samples/s +Step 34280/44000 | Loss: 2.9476 | LR: 5.98e-05 | 1291.8ms/step | 15.5 samples/s +Step 34290/44000 | Loss: 2.9085 | LR: 5.96e-05 | 1295.4ms/step | 15.4 samples/s +Step 34300/44000 | Loss: 2.9076 | LR: 5.95e-05 | 1290.8ms/step | 15.5 samples/s +Step 34310/44000 | Loss: 2.9219 | LR: 5.94e-05 | 1292.3ms/step | 15.5 samples/s +Step 34320/44000 | Loss: 2.9709 | LR: 5.93e-05 | 1293.7ms/step | 15.5 samples/s +Step 34330/44000 | Loss: 2.8919 | LR: 5.92e-05 | 1294.3ms/step | 15.5 samples/s +Step 34340/44000 | Loss: 2.8950 | LR: 5.91e-05 | 1291.8ms/step | 15.5 samples/s +Step 34350/44000 | Loss: 2.9165 | LR: 5.89e-05 | 1291.3ms/step | 15.5 samples/s +Step 34360/44000 | Loss: 2.9276 | LR: 5.88e-05 | 1291.5ms/step | 15.5 samples/s +Step 34370/44000 | Loss: 2.9164 | LR: 5.87e-05 | 1290.3ms/step | 15.5 samples/s +Step 34380/44000 | Loss: 2.9148 | LR: 5.86e-05 | 1288.0ms/step | 15.5 samples/s +Step 34390/44000 | Loss: 2.9281 | LR: 5.85e-05 | 1288.7ms/step | 15.5 samples/s +Step 34400/44000 | Loss: 2.9261 | LR: 5.84e-05 | 1288.7ms/step | 15.5 samples/s +Step 34410/44000 | Loss: 2.9439 | LR: 5.82e-05 | 1289.2ms/step | 15.5 samples/s +Step 34420/44000 | Loss: 2.9239 | LR: 5.81e-05 | 1289.7ms/step | 15.5 samples/s +Step 34430/44000 | Loss: 2.9202 | LR: 5.80e-05 | 1288.5ms/step | 15.5 samples/s +Step 34440/44000 | Loss: 2.9318 | LR: 5.79e-05 | 1289.0ms/step | 15.5 samples/s +Step 34450/44000 | Loss: 2.9111 | LR: 5.78e-05 | 1288.0ms/step | 15.5 samples/s +Step 34460/44000 | Loss: 2.9398 | LR: 5.77e-05 | 1287.8ms/step | 15.5 samples/s +Step 34470/44000 | Loss: 2.9224 | LR: 5.75e-05 | 1289.5ms/step | 15.5 samples/s +Step 34480/44000 | Loss: 2.9291 | LR: 5.74e-05 | 1289.0ms/step | 15.5 samples/s +Step 34490/44000 | Loss: 2.8921 | LR: 5.73e-05 | 1289.9ms/step | 15.5 samples/s +Step 34500/44000 | Loss: 2.9399 | LR: 5.72e-05 | 1292.2ms/step | 15.5 samples/s +Validation at step 34500: Loss = 2.9365, Perplexity = 18.85 +Step 34510/44000 | Loss: 2.9221 | LR: 5.71e-05 | 3448.0ms/step | 5.8 samples/s +Step 34520/44000 | Loss: 2.9148 | LR: 5.70e-05 | 1290.0ms/step | 15.5 samples/s +Step 34530/44000 | Loss: 2.9059 | LR: 5.69e-05 | 1288.1ms/step | 15.5 samples/s +Step 34540/44000 | Loss: 2.9258 | LR: 5.67e-05 | 1289.0ms/step | 15.5 samples/s +Step 34550/44000 | Loss: 2.9439 | LR: 5.66e-05 | 1288.9ms/step | 15.5 samples/s +Step 34560/44000 | Loss: 2.8918 | LR: 5.65e-05 | 1286.6ms/step | 15.5 samples/s +Step 34570/44000 | Loss: 2.9097 | LR: 5.64e-05 | 1290.7ms/step | 15.5 samples/s +Step 34580/44000 | Loss: 2.9030 | LR: 5.63e-05 | 1290.1ms/step | 15.5 samples/s +Step 34590/44000 | Loss: 2.9152 | LR: 5.62e-05 | 1288.9ms/step | 15.5 samples/s +Step 34600/44000 | Loss: 2.9278 | LR: 5.60e-05 | 1293.3ms/step | 15.5 samples/s +Step 34610/44000 | Loss: 2.9288 | LR: 5.59e-05 | 1288.7ms/step | 15.5 samples/s +Step 34620/44000 | Loss: 2.9246 | LR: 5.58e-05 | 1290.1ms/step | 15.5 samples/s +Step 34630/44000 | Loss: 2.9122 | LR: 5.57e-05 | 1291.6ms/step | 15.5 samples/s +Step 34640/44000 | Loss: 2.9198 | LR: 5.56e-05 | 1289.2ms/step | 15.5 samples/s +Step 34650/44000 | Loss: 2.9339 | LR: 5.55e-05 | 1289.7ms/step | 15.5 samples/s +Step 34660/44000 | Loss: 2.9417 | LR: 5.54e-05 | 1290.5ms/step | 15.5 samples/s +Step 34670/44000 | Loss: 2.8916 | LR: 5.52e-05 | 1289.3ms/step | 15.5 samples/s +Step 34680/44000 | Loss: 2.9242 | LR: 5.51e-05 | 1289.9ms/step | 15.5 samples/s +Step 34690/44000 | Loss: 2.9083 | LR: 5.50e-05 | 1292.1ms/step | 15.5 samples/s +Step 34700/44000 | Loss: 2.8981 | LR: 5.49e-05 | 1290.2ms/step | 15.5 samples/s +Step 34710/44000 | Loss: 2.9366 | LR: 5.48e-05 | 1289.8ms/step | 15.5 samples/s +Step 34720/44000 | Loss: 2.9098 | LR: 5.47e-05 | 1290.0ms/step | 15.5 samples/s +Step 34730/44000 | Loss: 2.9361 | LR: 5.46e-05 | 1289.2ms/step | 15.5 samples/s +Step 34740/44000 | Loss: 2.9097 | LR: 5.45e-05 | 1289.1ms/step | 15.5 samples/s +Step 34750/44000 | Loss: 2.9180 | LR: 5.43e-05 | 1289.5ms/step | 15.5 samples/s +Validation at step 34750: Loss = 2.9363, Perplexity = 18.85 +Step 34760/44000 | Loss: 2.9047 | LR: 5.42e-05 | 3444.8ms/step | 5.8 samples/s +Step 34770/44000 | Loss: 2.9113 | LR: 5.41e-05 | 1291.5ms/step | 15.5 samples/s +Step 34780/44000 | Loss: 2.9512 | LR: 5.40e-05 | 1295.2ms/step | 15.4 samples/s +Step 34790/44000 | Loss: 2.9464 | LR: 5.39e-05 | 1290.6ms/step | 15.5 samples/s +Step 34800/44000 | Loss: 2.9257 | LR: 5.38e-05 | 1290.7ms/step | 15.5 samples/s +Step 34810/44000 | Loss: 2.9421 | LR: 5.37e-05 | 1289.1ms/step | 15.5 samples/s +Step 34820/44000 | Loss: 2.9224 | LR: 5.36e-05 | 1290.3ms/step | 15.5 samples/s +Step 34830/44000 | Loss: 2.9210 | LR: 5.34e-05 | 1291.6ms/step | 15.5 samples/s +Step 34840/44000 | Loss: 2.9305 | LR: 5.33e-05 | 1290.8ms/step | 15.5 samples/s +Step 34850/44000 | Loss: 2.9405 | LR: 5.32e-05 | 1291.4ms/step | 15.5 samples/s +Step 34860/44000 | Loss: 2.9315 | LR: 5.31e-05 | 1289.2ms/step | 15.5 samples/s +Step 34870/44000 | Loss: 2.9112 | LR: 5.30e-05 | 1288.8ms/step | 15.5 samples/s +Step 34880/44000 | Loss: 2.9194 | LR: 5.29e-05 | 1287.9ms/step | 15.5 samples/s +Step 34890/44000 | Loss: 2.9156 | LR: 5.28e-05 | 1286.5ms/step | 15.5 samples/s +Step 34900/44000 | Loss: 2.9319 | LR: 5.27e-05 | 1292.4ms/step | 15.5 samples/s +Step 34910/44000 | Loss: 2.9290 | LR: 5.25e-05 | 1291.1ms/step | 15.5 samples/s +Step 34920/44000 | Loss: 2.9157 | LR: 5.24e-05 | 1290.8ms/step | 15.5 samples/s +Step 34930/44000 | Loss: 2.9027 | LR: 5.23e-05 | 1290.2ms/step | 15.5 samples/s +Step 34940/44000 | Loss: 2.8776 | LR: 5.22e-05 | 1291.6ms/step | 15.5 samples/s +Step 34950/44000 | Loss: 2.9344 | LR: 5.21e-05 | 1293.3ms/step | 15.5 samples/s +Step 34960/44000 | Loss: 2.9240 | LR: 5.20e-05 | 1293.2ms/step | 15.5 samples/s +Step 34970/44000 | Loss: 2.9050 | LR: 5.19e-05 | 1288.6ms/step | 15.5 samples/s +Step 34980/44000 | Loss: 2.9291 | LR: 5.18e-05 | 1289.8ms/step | 15.5 samples/s +Step 34990/44000 | Loss: 2.9316 | LR: 5.17e-05 | 1291.0ms/step | 15.5 samples/s +Step 35000/44000 | Loss: 2.8996 | LR: 5.15e-05 | 1291.1ms/step | 15.5 samples/s +Validation at step 35000: Loss = 2.9360, Perplexity = 18.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-35000 +Step 35010/44000 | Loss: 2.9250 | LR: 5.14e-05 | 3733.7ms/step | 5.4 samples/s +Step 35020/44000 | Loss: 2.8929 | LR: 5.13e-05 | 1290.0ms/step | 15.5 samples/s +Step 35030/44000 | Loss: 2.9020 | LR: 5.12e-05 | 1288.7ms/step | 15.5 samples/s +Step 35040/44000 | Loss: 2.9133 | LR: 5.11e-05 | 1289.8ms/step | 15.5 samples/s +Step 35050/44000 | Loss: 2.9042 | LR: 5.10e-05 | 1290.5ms/step | 15.5 samples/s +Step 35060/44000 | Loss: 2.9138 | LR: 5.09e-05 | 1290.3ms/step | 15.5 samples/s +Step 35070/44000 | Loss: 2.8825 | LR: 5.08e-05 | 1290.8ms/step | 15.5 samples/s +Step 35080/44000 | Loss: 2.9178 | LR: 5.07e-05 | 1292.2ms/step | 15.5 samples/s +Step 35090/44000 | Loss: 2.9059 | LR: 5.06e-05 | 1290.6ms/step | 15.5 samples/s +Step 35100/44000 | Loss: 2.9476 | LR: 5.04e-05 | 1290.4ms/step | 15.5 samples/s +Step 35110/44000 | Loss: 2.9105 | LR: 5.03e-05 | 1289.7ms/step | 15.5 samples/s +Step 35120/44000 | Loss: 2.9282 | LR: 5.02e-05 | 1289.9ms/step | 15.5 samples/s +Step 35130/44000 | Loss: 2.9253 | LR: 5.01e-05 | 1290.7ms/step | 15.5 samples/s +Step 35140/44000 | Loss: 2.8933 | LR: 5.00e-05 | 1290.5ms/step | 15.5 samples/s +Step 35150/44000 | Loss: 2.9323 | LR: 4.99e-05 | 1291.6ms/step | 15.5 samples/s +Step 35160/44000 | Loss: 2.9540 | LR: 4.98e-05 | 1293.7ms/step | 15.5 samples/s +Step 35170/44000 | Loss: 2.9314 | LR: 4.97e-05 | 1292.1ms/step | 15.5 samples/s +Step 35180/44000 | Loss: 2.9379 | LR: 4.96e-05 | 1294.4ms/step | 15.5 samples/s +Step 35190/44000 | Loss: 2.9068 | LR: 4.95e-05 | 1296.3ms/step | 15.4 samples/s +Step 35200/44000 | Loss: 2.9366 | LR: 4.94e-05 | 1293.8ms/step | 15.5 samples/s +Step 35210/44000 | Loss: 2.9339 | LR: 4.93e-05 | 1296.4ms/step | 15.4 samples/s +Step 35220/44000 | Loss: 2.9111 | LR: 4.91e-05 | 1293.7ms/step | 15.5 samples/s +Step 35230/44000 | Loss: 2.9192 | LR: 4.90e-05 | 1292.1ms/step | 15.5 samples/s +Step 35240/44000 | Loss: 2.9079 | LR: 4.89e-05 | 1288.4ms/step | 15.5 samples/s +Step 35250/44000 | Loss: 2.9553 | LR: 4.88e-05 | 1288.7ms/step | 15.5 samples/s +Validation at step 35250: Loss = 2.9358, Perplexity = 18.84 +Step 35260/44000 | Loss: 2.9127 | LR: 4.87e-05 | 3453.0ms/step | 5.8 samples/s +Step 35270/44000 | Loss: 2.9436 | LR: 4.86e-05 | 1292.4ms/step | 15.5 samples/s +Step 35280/44000 | Loss: 2.8985 | LR: 4.85e-05 | 1290.9ms/step | 15.5 samples/s +Step 35290/44000 | Loss: 2.9077 | LR: 4.84e-05 | 1292.3ms/step | 15.5 samples/s +Step 35300/44000 | Loss: 2.9098 | LR: 4.83e-05 | 1291.3ms/step | 15.5 samples/s +Step 35310/44000 | Loss: 2.9295 | LR: 4.82e-05 | 1289.2ms/step | 15.5 samples/s +Step 35320/44000 | Loss: 2.8929 | LR: 4.81e-05 | 1288.6ms/step | 15.5 samples/s +Step 35330/44000 | Loss: 2.9271 | LR: 4.80e-05 | 1290.3ms/step | 15.5 samples/s +Step 35340/44000 | Loss: 2.9232 | LR: 4.79e-05 | 1291.0ms/step | 15.5 samples/s +Step 35350/44000 | Loss: 2.9484 | LR: 4.77e-05 | 1289.2ms/step | 15.5 samples/s +Step 35360/44000 | Loss: 2.9044 | LR: 4.76e-05 | 1288.6ms/step | 15.5 samples/s +Step 35370/44000 | Loss: 2.9157 | LR: 4.75e-05 | 1289.1ms/step | 15.5 samples/s +Step 35380/44000 | Loss: 2.9044 | LR: 4.74e-05 | 1290.3ms/step | 15.5 samples/s +Step 35390/44000 | Loss: 2.9097 | LR: 4.73e-05 | 1289.2ms/step | 15.5 samples/s +Step 35400/44000 | Loss: 2.8985 | LR: 4.72e-05 | 1289.5ms/step | 15.5 samples/s +Step 35410/44000 | Loss: 2.9320 | LR: 4.71e-05 | 1289.4ms/step | 15.5 samples/s +Step 35420/44000 | Loss: 2.9029 | LR: 4.70e-05 | 1287.5ms/step | 15.5 samples/s +Step 35430/44000 | Loss: 2.9621 | LR: 4.69e-05 | 1290.9ms/step | 15.5 samples/s +Step 35440/44000 | Loss: 2.9423 | LR: 4.68e-05 | 1294.2ms/step | 15.5 samples/s +Step 35450/44000 | Loss: 2.9254 | LR: 4.67e-05 | 1290.7ms/step | 15.5 samples/s +Step 35460/44000 | Loss: 2.9434 | LR: 4.66e-05 | 1291.4ms/step | 15.5 samples/s +Step 35470/44000 | Loss: 2.9158 | LR: 4.65e-05 | 1292.6ms/step | 15.5 samples/s +Step 35480/44000 | Loss: 2.9252 | LR: 4.64e-05 | 1293.2ms/step | 15.5 samples/s +Step 35490/44000 | Loss: 2.9303 | LR: 4.63e-05 | 1291.8ms/step | 15.5 samples/s +Step 35500/44000 | Loss: 2.9040 | LR: 4.62e-05 | 1291.9ms/step | 15.5 samples/s +Validation at step 35500: Loss = 2.9356, Perplexity = 18.83 +Step 35510/44000 | Loss: 2.9145 | LR: 4.61e-05 | 3450.1ms/step | 5.8 samples/s +Step 35520/44000 | Loss: 2.9235 | LR: 4.59e-05 | 1290.6ms/step | 15.5 samples/s +Step 35530/44000 | Loss: 2.9227 | LR: 4.58e-05 | 1291.7ms/step | 15.5 samples/s +Step 35540/44000 | Loss: 2.9405 | LR: 4.57e-05 | 1291.6ms/step | 15.5 samples/s +Step 35550/44000 | Loss: 2.9425 | LR: 4.56e-05 | 1290.7ms/step | 15.5 samples/s +Step 35560/44000 | Loss: 2.9333 | LR: 4.55e-05 | 1288.9ms/step | 15.5 samples/s +Step 35570/44000 | Loss: 2.9419 | LR: 4.54e-05 | 1289.9ms/step | 15.5 samples/s +Step 35580/44000 | Loss: 2.9250 | LR: 4.53e-05 | 1293.4ms/step | 15.5 samples/s +Step 35590/44000 | Loss: 2.9360 | LR: 4.52e-05 | 1295.7ms/step | 15.4 samples/s +Step 35600/44000 | Loss: 2.8996 | LR: 4.51e-05 | 1299.0ms/step | 15.4 samples/s +Step 35610/44000 | Loss: 2.9382 | LR: 4.50e-05 | 1291.4ms/step | 15.5 samples/s +Step 35620/44000 | Loss: 2.9160 | LR: 4.49e-05 | 1291.8ms/step | 15.5 samples/s +Step 35630/44000 | Loss: 2.9279 | LR: 4.48e-05 | 1293.4ms/step | 15.5 samples/s +Step 35640/44000 | Loss: 2.9045 | LR: 4.47e-05 | 1293.7ms/step | 15.5 samples/s +Step 35650/44000 | Loss: 2.9126 | LR: 4.46e-05 | 1293.9ms/step | 15.5 samples/s +Step 35660/44000 | Loss: 2.9277 | LR: 4.45e-05 | 1294.4ms/step | 15.5 samples/s +Step 35670/44000 | Loss: 2.9164 | LR: 4.44e-05 | 1292.5ms/step | 15.5 samples/s +Step 35680/44000 | Loss: 2.9202 | LR: 4.43e-05 | 1291.7ms/step | 15.5 samples/s +Step 35690/44000 | Loss: 2.9314 | LR: 4.42e-05 | 1291.8ms/step | 15.5 samples/s +Step 35700/44000 | Loss: 2.9086 | LR: 4.41e-05 | 1291.6ms/step | 15.5 samples/s +Step 35710/44000 | Loss: 2.9239 | LR: 4.40e-05 | 1289.8ms/step | 15.5 samples/s +Step 35720/44000 | Loss: 2.9058 | LR: 4.39e-05 | 1291.2ms/step | 15.5 samples/s +Step 35730/44000 | Loss: 2.8977 | LR: 4.38e-05 | 1289.5ms/step | 15.5 samples/s +Step 35740/44000 | Loss: 2.9282 | LR: 4.37e-05 | 1289.2ms/step | 15.5 samples/s +Step 35750/44000 | Loss: 2.9093 | LR: 4.36e-05 | 1291.5ms/step | 15.5 samples/s +Validation at step 35750: Loss = 2.9354, Perplexity = 18.83 +Step 35760/44000 | Loss: 2.9388 | LR: 4.35e-05 | 3447.1ms/step | 5.8 samples/s +Step 35770/44000 | Loss: 2.9130 | LR: 4.34e-05 | 1290.9ms/step | 15.5 samples/s +Step 35780/44000 | Loss: 2.9028 | LR: 4.33e-05 | 1290.3ms/step | 15.5 samples/s +Step 35790/44000 | Loss: 2.9300 | LR: 4.32e-05 | 1290.4ms/step | 15.5 samples/s +Step 35800/44000 | Loss: 2.9374 | LR: 4.31e-05 | 1290.5ms/step | 15.5 samples/s +Step 35810/44000 | Loss: 2.9336 | LR: 4.29e-05 | 1290.6ms/step | 15.5 samples/s +Step 35820/44000 | Loss: 2.9367 | LR: 4.28e-05 | 1290.3ms/step | 15.5 samples/s +Step 35830/44000 | Loss: 2.9543 | LR: 4.27e-05 | 1289.4ms/step | 15.5 samples/s +Step 35840/44000 | Loss: 2.9140 | LR: 4.26e-05 | 1293.6ms/step | 15.5 samples/s +Step 35850/44000 | Loss: 2.9230 | LR: 4.25e-05 | 1292.0ms/step | 15.5 samples/s +Step 35860/44000 | Loss: 2.9463 | LR: 4.24e-05 | 1290.6ms/step | 15.5 samples/s +Step 35870/44000 | Loss: 2.9230 | LR: 4.23e-05 | 1289.0ms/step | 15.5 samples/s +Step 35880/44000 | Loss: 2.9397 | LR: 4.22e-05 | 1288.7ms/step | 15.5 samples/s +Step 35890/44000 | Loss: 2.9052 | LR: 4.21e-05 | 1290.9ms/step | 15.5 samples/s +Step 35900/44000 | Loss: 2.9261 | LR: 4.20e-05 | 1289.4ms/step | 15.5 samples/s +Step 35910/44000 | Loss: 2.9096 | LR: 4.19e-05 | 1291.8ms/step | 15.5 samples/s +Step 35920/44000 | Loss: 2.9223 | LR: 4.18e-05 | 1291.4ms/step | 15.5 samples/s +Step 35930/44000 | Loss: 2.9109 | LR: 4.17e-05 | 1291.6ms/step | 15.5 samples/s +Step 35940/44000 | Loss: 2.9120 | LR: 4.16e-05 | 1290.5ms/step | 15.5 samples/s +Step 35950/44000 | Loss: 2.8989 | LR: 4.15e-05 | 1292.4ms/step | 15.5 samples/s +Step 35960/44000 | Loss: 2.9035 | LR: 4.14e-05 | 1289.3ms/step | 15.5 samples/s +Step 35970/44000 | Loss: 2.8960 | LR: 4.13e-05 | 1289.4ms/step | 15.5 samples/s +Step 35980/44000 | Loss: 2.9482 | LR: 4.12e-05 | 1291.2ms/step | 15.5 samples/s +Step 35990/44000 | Loss: 2.9190 | LR: 4.11e-05 | 1289.3ms/step | 15.5 samples/s +Step 36000/44000 | Loss: 2.9182 | LR: 4.10e-05 | 1286.9ms/step | 15.5 samples/s +Validation at step 36000: Loss = 2.9352, Perplexity = 18.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-36000 +Step 36010/44000 | Loss: 2.9344 | LR: 4.09e-05 | 3735.0ms/step | 5.4 samples/s +Step 36020/44000 | Loss: 2.9127 | LR: 4.08e-05 | 1290.7ms/step | 15.5 samples/s +Step 36030/44000 | Loss: 2.9024 | LR: 4.07e-05 | 1290.2ms/step | 15.5 samples/s +Step 36040/44000 | Loss: 2.9078 | LR: 4.06e-05 | 1289.8ms/step | 15.5 samples/s +Step 36050/44000 | Loss: 2.9055 | LR: 4.05e-05 | 1290.3ms/step | 15.5 samples/s +Step 36060/44000 | Loss: 2.9332 | LR: 4.04e-05 | 1289.9ms/step | 15.5 samples/s +Step 36070/44000 | Loss: 2.9230 | LR: 4.03e-05 | 1290.7ms/step | 15.5 samples/s +Step 36080/44000 | Loss: 2.9390 | LR: 4.02e-05 | 1291.0ms/step | 15.5 samples/s +Step 36090/44000 | Loss: 2.9459 | LR: 4.01e-05 | 1291.2ms/step | 15.5 samples/s +Step 36100/44000 | Loss: 2.9234 | LR: 4.00e-05 | 1290.1ms/step | 15.5 samples/s +Step 36110/44000 | Loss: 2.9388 | LR: 3.99e-05 | 1290.7ms/step | 15.5 samples/s +Step 36120/44000 | Loss: 2.9255 | LR: 3.98e-05 | 1290.0ms/step | 15.5 samples/s +Step 36130/44000 | Loss: 2.9282 | LR: 3.97e-05 | 1290.0ms/step | 15.5 samples/s +Step 36140/44000 | Loss: 2.9202 | LR: 3.97e-05 | 1287.3ms/step | 15.5 samples/s +Step 36150/44000 | Loss: 2.9314 | LR: 3.96e-05 | 1288.4ms/step | 15.5 samples/s +Step 36160/44000 | Loss: 2.9225 | LR: 3.95e-05 | 1290.5ms/step | 15.5 samples/s +Step 36170/44000 | Loss: 2.8796 | LR: 3.94e-05 | 1291.8ms/step | 15.5 samples/s +Step 36180/44000 | Loss: 2.9216 | LR: 3.93e-05 | 1293.3ms/step | 15.5 samples/s +Step 36190/44000 | Loss: 2.9181 | LR: 3.92e-05 | 1292.4ms/step | 15.5 samples/s +Step 36200/44000 | Loss: 2.9120 | LR: 3.91e-05 | 1290.8ms/step | 15.5 samples/s +Step 36210/44000 | Loss: 2.9211 | LR: 3.90e-05 | 1291.1ms/step | 15.5 samples/s +Step 36220/44000 | Loss: 2.9146 | LR: 3.89e-05 | 1291.2ms/step | 15.5 samples/s +Step 36230/44000 | Loss: 2.9158 | LR: 3.88e-05 | 1294.1ms/step | 15.5 samples/s +Step 36240/44000 | Loss: 2.9260 | LR: 3.87e-05 | 1292.4ms/step | 15.5 samples/s +Step 36250/44000 | Loss: 2.9411 | LR: 3.86e-05 | 1290.2ms/step | 15.5 samples/s +Validation at step 36250: Loss = 2.9350, Perplexity = 18.82 +Step 36260/44000 | Loss: 2.9318 | LR: 3.85e-05 | 3446.1ms/step | 5.8 samples/s +Step 36270/44000 | Loss: 2.9431 | LR: 3.84e-05 | 1289.5ms/step | 15.5 samples/s +Step 36280/44000 | Loss: 2.9156 | LR: 3.83e-05 | 1290.8ms/step | 15.5 samples/s +Step 36290/44000 | Loss: 2.9290 | LR: 3.82e-05 | 1288.8ms/step | 15.5 samples/s +Step 36300/44000 | Loss: 2.9138 | LR: 3.81e-05 | 1290.3ms/step | 15.5 samples/s +Step 36310/44000 | Loss: 2.9151 | LR: 3.80e-05 | 1291.2ms/step | 15.5 samples/s +Step 36320/44000 | Loss: 2.8922 | LR: 3.79e-05 | 1290.9ms/step | 15.5 samples/s +Step 36330/44000 | Loss: 2.9077 | LR: 3.78e-05 | 1290.5ms/step | 15.5 samples/s +Step 36340/44000 | Loss: 2.9269 | LR: 3.77e-05 | 1292.0ms/step | 15.5 samples/s +Step 36350/44000 | Loss: 2.8971 | LR: 3.76e-05 | 1293.5ms/step | 15.5 samples/s +Step 36360/44000 | Loss: 2.9060 | LR: 3.75e-05 | 1290.4ms/step | 15.5 samples/s +Step 36370/44000 | Loss: 2.9163 | LR: 3.74e-05 | 1285.3ms/step | 15.6 samples/s +Step 36380/44000 | Loss: 2.9143 | LR: 3.73e-05 | 1287.8ms/step | 15.5 samples/s +Step 36390/44000 | Loss: 2.9372 | LR: 3.72e-05 | 1288.1ms/step | 15.5 samples/s +Step 36400/44000 | Loss: 2.9140 | LR: 3.71e-05 | 1290.7ms/step | 15.5 samples/s +Step 36410/44000 | Loss: 2.9234 | LR: 3.70e-05 | 1292.4ms/step | 15.5 samples/s +Step 36420/44000 | Loss: 2.8926 | LR: 3.69e-05 | 1291.2ms/step | 15.5 samples/s +Step 36430/44000 | Loss: 2.9002 | LR: 3.69e-05 | 1288.1ms/step | 15.5 samples/s +Step 36440/44000 | Loss: 2.9098 | LR: 3.68e-05 | 1287.9ms/step | 15.5 samples/s +Step 36450/44000 | Loss: 2.9499 | LR: 3.67e-05 | 1289.9ms/step | 15.5 samples/s +Step 36460/44000 | Loss: 2.9256 | LR: 3.66e-05 | 1290.9ms/step | 15.5 samples/s +Step 36470/44000 | Loss: 2.8998 | LR: 3.65e-05 | 1291.5ms/step | 15.5 samples/s +Step 36480/44000 | Loss: 2.9238 | LR: 3.64e-05 | 1290.2ms/step | 15.5 samples/s +Step 36490/44000 | Loss: 2.9448 | LR: 3.63e-05 | 1290.9ms/step | 15.5 samples/s +Step 36500/44000 | Loss: 2.8934 | LR: 3.62e-05 | 1290.8ms/step | 15.5 samples/s +Validation at step 36500: Loss = 2.9349, Perplexity = 18.82 +Step 36510/44000 | Loss: 2.9342 | LR: 3.61e-05 | 3466.9ms/step | 5.8 samples/s +Step 36520/44000 | Loss: 2.9080 | LR: 3.60e-05 | 1291.8ms/step | 15.5 samples/s +Step 36530/44000 | Loss: 2.9155 | LR: 3.59e-05 | 1294.5ms/step | 15.5 samples/s +Step 36540/44000 | Loss: 2.9040 | LR: 3.58e-05 | 1293.0ms/step | 15.5 samples/s +Step 36550/44000 | Loss: 2.9123 | LR: 3.57e-05 | 1290.6ms/step | 15.5 samples/s +Step 36560/44000 | Loss: 2.9322 | LR: 3.56e-05 | 1294.1ms/step | 15.5 samples/s +Step 36570/44000 | Loss: 2.8916 | LR: 3.55e-05 | 1289.8ms/step | 15.5 samples/s +Step 36580/44000 | Loss: 2.9108 | LR: 3.54e-05 | 1298.5ms/step | 15.4 samples/s +Step 36590/44000 | Loss: 2.9205 | LR: 3.53e-05 | 1290.8ms/step | 15.5 samples/s +Step 36600/44000 | Loss: 2.8929 | LR: 3.53e-05 | 1287.8ms/step | 15.5 samples/s +Step 36610/44000 | Loss: 2.9158 | LR: 3.52e-05 | 1286.1ms/step | 15.6 samples/s +Step 36620/44000 | Loss: 2.9062 | LR: 3.51e-05 | 1290.1ms/step | 15.5 samples/s +Step 36630/44000 | Loss: 2.9198 | LR: 3.50e-05 | 1292.0ms/step | 15.5 samples/s +Step 36640/44000 | Loss: 2.9181 | LR: 3.49e-05 | 1290.5ms/step | 15.5 samples/s +Step 36650/44000 | Loss: 2.9152 | LR: 3.48e-05 | 1289.3ms/step | 15.5 samples/s +Step 36660/44000 | Loss: 2.9077 | LR: 3.47e-05 | 1290.9ms/step | 15.5 samples/s +Step 36670/44000 | Loss: 2.9469 | LR: 3.46e-05 | 1289.9ms/step | 15.5 samples/s +Step 36680/44000 | Loss: 2.9344 | LR: 3.45e-05 | 1291.4ms/step | 15.5 samples/s +Step 36690/44000 | Loss: 2.9106 | LR: 3.44e-05 | 1289.3ms/step | 15.5 samples/s +Step 36700/44000 | Loss: 2.9126 | LR: 3.43e-05 | 1289.2ms/step | 15.5 samples/s +Step 36710/44000 | Loss: 2.9048 | LR: 3.42e-05 | 1290.8ms/step | 15.5 samples/s +Step 36720/44000 | Loss: 2.9171 | LR: 3.41e-05 | 1288.2ms/step | 15.5 samples/s +Step 36730/44000 | Loss: 2.9083 | LR: 3.41e-05 | 1291.4ms/step | 15.5 samples/s +Step 36740/44000 | Loss: 2.9318 | LR: 3.40e-05 | 1290.6ms/step | 15.5 samples/s +Step 36750/44000 | Loss: 2.9004 | LR: 3.39e-05 | 1290.1ms/step | 15.5 samples/s +Validation at step 36750: Loss = 2.9348, Perplexity = 18.82 +Step 36760/44000 | Loss: 2.9086 | LR: 3.38e-05 | 3444.4ms/step | 5.8 samples/s +Step 36770/44000 | Loss: 2.9209 | LR: 3.37e-05 | 1288.6ms/step | 15.5 samples/s +Step 36780/44000 | Loss: 2.9303 | LR: 3.36e-05 | 1288.7ms/step | 15.5 samples/s +Step 36790/44000 | Loss: 2.9269 | LR: 3.35e-05 | 1290.0ms/step | 15.5 samples/s +Step 36800/44000 | Loss: 2.9191 | LR: 3.34e-05 | 1289.3ms/step | 15.5 samples/s +Step 36810/44000 | Loss: 2.9143 | LR: 3.33e-05 | 1289.1ms/step | 15.5 samples/s +Step 36820/44000 | Loss: 2.9137 | LR: 3.32e-05 | 1291.0ms/step | 15.5 samples/s +Step 36830/44000 | Loss: 2.9160 | LR: 3.31e-05 | 1291.7ms/step | 15.5 samples/s +Step 36840/44000 | Loss: 2.9199 | LR: 3.31e-05 | 1290.7ms/step | 15.5 samples/s +Step 36850/44000 | Loss: 2.9208 | LR: 3.30e-05 | 1288.8ms/step | 15.5 samples/s +Step 36860/44000 | Loss: 2.9017 | LR: 3.29e-05 | 1289.9ms/step | 15.5 samples/s +Step 36870/44000 | Loss: 2.9220 | LR: 3.28e-05 | 1290.3ms/step | 15.5 samples/s +Step 36880/44000 | Loss: 2.8944 | LR: 3.27e-05 | 1292.6ms/step | 15.5 samples/s +Step 36890/44000 | Loss: 2.8921 | LR: 3.26e-05 | 1289.9ms/step | 15.5 samples/s +Step 36900/44000 | Loss: 2.8966 | LR: 3.25e-05 | 1286.0ms/step | 15.6 samples/s +Step 36910/44000 | Loss: 2.9126 | LR: 3.24e-05 | 1291.5ms/step | 15.5 samples/s +Step 36920/44000 | Loss: 2.9118 | LR: 3.23e-05 | 1290.8ms/step | 15.5 samples/s +Step 36930/44000 | Loss: 2.8903 | LR: 3.22e-05 | 1291.3ms/step | 15.5 samples/s +Step 36940/44000 | Loss: 2.9291 | LR: 3.22e-05 | 1291.5ms/step | 15.5 samples/s +Step 36950/44000 | Loss: 2.9255 | LR: 3.21e-05 | 1291.4ms/step | 15.5 samples/s +Step 36960/44000 | Loss: 2.9080 | LR: 3.20e-05 | 1292.2ms/step | 15.5 samples/s +Step 36970/44000 | Loss: 2.8891 | LR: 3.19e-05 | 1293.5ms/step | 15.5 samples/s +Step 36980/44000 | Loss: 2.9120 | LR: 3.18e-05 | 1292.8ms/step | 15.5 samples/s +Step 36990/44000 | Loss: 2.9226 | LR: 3.17e-05 | 1293.1ms/step | 15.5 samples/s +Step 37000/44000 | Loss: 2.9332 | LR: 3.16e-05 | 1297.5ms/step | 15.4 samples/s +Validation at step 37000: Loss = 2.9347, Perplexity = 18.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-37000 +Step 37010/44000 | Loss: 2.9302 | LR: 3.15e-05 | 3739.6ms/step | 5.3 samples/s +Step 37020/44000 | Loss: 2.9693 | LR: 3.15e-05 | 1288.7ms/step | 15.5 samples/s +Step 37030/44000 | Loss: 2.9351 | LR: 3.14e-05 | 1287.9ms/step | 15.5 samples/s +Step 37040/44000 | Loss: 2.9314 | LR: 3.13e-05 | 1290.2ms/step | 15.5 samples/s +Step 37050/44000 | Loss: 2.9231 | LR: 3.12e-05 | 1293.5ms/step | 15.5 samples/s +Step 37060/44000 | Loss: 2.8915 | LR: 3.11e-05 | 1289.4ms/step | 15.5 samples/s +Step 37070/44000 | Loss: 2.9224 | LR: 3.10e-05 | 1293.1ms/step | 15.5 samples/s +Step 37080/44000 | Loss: 2.9526 | LR: 3.09e-05 | 1290.5ms/step | 15.5 samples/s +Step 37090/44000 | Loss: 2.9312 | LR: 3.08e-05 | 1290.5ms/step | 15.5 samples/s +Step 37100/44000 | Loss: 2.9141 | LR: 3.07e-05 | 1288.9ms/step | 15.5 samples/s +Step 37110/44000 | Loss: 2.9084 | LR: 3.07e-05 | 1287.6ms/step | 15.5 samples/s +Step 37120/44000 | Loss: 2.9322 | LR: 3.06e-05 | 1287.9ms/step | 15.5 samples/s +Step 37130/44000 | Loss: 2.9224 | LR: 3.05e-05 | 1288.6ms/step | 15.5 samples/s +Step 37140/44000 | Loss: 2.9159 | LR: 3.04e-05 | 1287.9ms/step | 15.5 samples/s +Step 37150/44000 | Loss: 2.8779 | LR: 3.03e-05 | 1288.5ms/step | 15.5 samples/s +Step 37160/44000 | Loss: 2.9279 | LR: 3.02e-05 | 1288.2ms/step | 15.5 samples/s +Step 37170/44000 | Loss: 2.9595 | LR: 3.01e-05 | 1289.0ms/step | 15.5 samples/s +Step 37180/44000 | Loss: 2.9123 | LR: 3.01e-05 | 1289.2ms/step | 15.5 samples/s +Step 37190/44000 | Loss: 2.9166 | LR: 3.00e-05 | 1291.0ms/step | 15.5 samples/s +Step 37200/44000 | Loss: 2.9214 | LR: 2.99e-05 | 1290.1ms/step | 15.5 samples/s +Step 37210/44000 | Loss: 2.9355 | LR: 2.98e-05 | 1291.7ms/step | 15.5 samples/s +Step 37220/44000 | Loss: 2.9152 | LR: 2.97e-05 | 1290.4ms/step | 15.5 samples/s +Step 37230/44000 | Loss: 2.9443 | LR: 2.96e-05 | 1290.4ms/step | 15.5 samples/s +Step 37240/44000 | Loss: 2.9214 | LR: 2.95e-05 | 1290.8ms/step | 15.5 samples/s +Step 37250/44000 | Loss: 2.9132 | LR: 2.95e-05 | 1290.9ms/step | 15.5 samples/s +Validation at step 37250: Loss = 2.9346, Perplexity = 18.81 +Step 37260/44000 | Loss: 2.9260 | LR: 2.94e-05 | 3446.8ms/step | 5.8 samples/s +Step 37270/44000 | Loss: 2.9378 | LR: 2.93e-05 | 1288.8ms/step | 15.5 samples/s +Step 37280/44000 | Loss: 2.9179 | LR: 2.92e-05 | 1289.9ms/step | 15.5 samples/s +Step 37290/44000 | Loss: 2.9271 | LR: 2.91e-05 | 1290.6ms/step | 15.5 samples/s +Step 37300/44000 | Loss: 2.9131 | LR: 2.90e-05 | 1290.7ms/step | 15.5 samples/s +Step 37310/44000 | Loss: 2.9161 | LR: 2.89e-05 | 1291.4ms/step | 15.5 samples/s +Step 37320/44000 | Loss: 2.9205 | LR: 2.89e-05 | 1291.7ms/step | 15.5 samples/s +Step 37330/44000 | Loss: 2.9447 | LR: 2.88e-05 | 1288.0ms/step | 15.5 samples/s +Step 37340/44000 | Loss: 2.9164 | LR: 2.87e-05 | 1288.1ms/step | 15.5 samples/s +Step 37350/44000 | Loss: 2.9362 | LR: 2.86e-05 | 1287.3ms/step | 15.5 samples/s +Step 37360/44000 | Loss: 2.9217 | LR: 2.85e-05 | 1288.1ms/step | 15.5 samples/s +Step 37370/44000 | Loss: 2.9237 | LR: 2.84e-05 | 1290.5ms/step | 15.5 samples/s +Step 37380/44000 | Loss: 2.9059 | LR: 2.84e-05 | 1291.8ms/step | 15.5 samples/s +Step 37390/44000 | Loss: 2.9040 | LR: 2.83e-05 | 1289.9ms/step | 15.5 samples/s +Step 37400/44000 | Loss: 2.9088 | LR: 2.82e-05 | 1289.6ms/step | 15.5 samples/s +Step 37410/44000 | Loss: 2.9025 | LR: 2.81e-05 | 1287.6ms/step | 15.5 samples/s +Step 37420/44000 | Loss: 2.9327 | LR: 2.80e-05 | 1291.3ms/step | 15.5 samples/s +Step 37430/44000 | Loss: 2.8937 | LR: 2.79e-05 | 1292.9ms/step | 15.5 samples/s +Step 37440/44000 | Loss: 2.9055 | LR: 2.78e-05 | 1290.8ms/step | 15.5 samples/s +Step 37450/44000 | Loss: 2.9205 | LR: 2.78e-05 | 1290.4ms/step | 15.5 samples/s +Step 37460/44000 | Loss: 2.9353 | LR: 2.77e-05 | 1290.0ms/step | 15.5 samples/s +Step 37470/44000 | Loss: 2.9190 | LR: 2.76e-05 | 1291.8ms/step | 15.5 samples/s +Step 37480/44000 | Loss: 2.9288 | LR: 2.75e-05 | 1290.8ms/step | 15.5 samples/s +Step 37490/44000 | Loss: 2.9144 | LR: 2.74e-05 | 1290.5ms/step | 15.5 samples/s +Step 37500/44000 | Loss: 2.9188 | LR: 2.74e-05 | 1290.4ms/step | 15.5 samples/s +Validation at step 37500: Loss = 2.9345, Perplexity = 18.81 +Step 37510/44000 | Loss: 2.9390 | LR: 2.73e-05 | 3440.3ms/step | 5.8 samples/s +Step 37520/44000 | Loss: 2.8936 | LR: 2.72e-05 | 1287.5ms/step | 15.5 samples/s +Step 37530/44000 | Loss: 2.9580 | LR: 2.71e-05 | 1289.5ms/step | 15.5 samples/s +Step 37540/44000 | Loss: 2.9269 | LR: 2.70e-05 | 1288.6ms/step | 15.5 samples/s +Step 37550/44000 | Loss: 2.9202 | LR: 2.69e-05 | 1289.7ms/step | 15.5 samples/s +Step 37560/44000 | Loss: 2.9381 | LR: 2.69e-05 | 1286.2ms/step | 15.5 samples/s +Step 37570/44000 | Loss: 2.9172 | LR: 2.68e-05 | 1289.4ms/step | 15.5 samples/s +Step 37580/44000 | Loss: 2.9101 | LR: 2.67e-05 | 1289.0ms/step | 15.5 samples/s +Step 37590/44000 | Loss: 2.9116 | LR: 2.66e-05 | 1289.7ms/step | 15.5 samples/s +Step 37600/44000 | Loss: 2.9165 | LR: 2.65e-05 | 1292.1ms/step | 15.5 samples/s +Step 37610/44000 | Loss: 2.9077 | LR: 2.65e-05 | 1290.1ms/step | 15.5 samples/s +Step 37620/44000 | Loss: 2.9310 | LR: 2.64e-05 | 1291.2ms/step | 15.5 samples/s +Step 37630/44000 | Loss: 2.8849 | LR: 2.63e-05 | 1291.1ms/step | 15.5 samples/s +Step 37640/44000 | Loss: 2.9340 | LR: 2.62e-05 | 1289.6ms/step | 15.5 samples/s +Step 37650/44000 | Loss: 2.9061 | LR: 2.61e-05 | 1289.6ms/step | 15.5 samples/s +Step 37660/44000 | Loss: 2.9437 | LR: 2.60e-05 | 1291.5ms/step | 15.5 samples/s +Step 37670/44000 | Loss: 2.9248 | LR: 2.60e-05 | 1290.9ms/step | 15.5 samples/s +Step 37680/44000 | Loss: 2.9360 | LR: 2.59e-05 | 1292.9ms/step | 15.5 samples/s +Step 37690/44000 | Loss: 2.9113 | LR: 2.58e-05 | 1291.3ms/step | 15.5 samples/s +Step 37700/44000 | Loss: 2.9100 | LR: 2.57e-05 | 1289.0ms/step | 15.5 samples/s +Step 37710/44000 | Loss: 2.9001 | LR: 2.56e-05 | 1287.3ms/step | 15.5 samples/s +Step 37720/44000 | Loss: 2.9366 | LR: 2.56e-05 | 1290.4ms/step | 15.5 samples/s +Step 37730/44000 | Loss: 2.9076 | LR: 2.55e-05 | 1291.9ms/step | 15.5 samples/s +Step 37740/44000 | Loss: 2.9187 | LR: 2.54e-05 | 1290.2ms/step | 15.5 samples/s +Step 37750/44000 | Loss: 2.9036 | LR: 2.53e-05 | 1289.8ms/step | 15.5 samples/s +Validation at step 37750: Loss = 2.9344, Perplexity = 18.81 +Step 37760/44000 | Loss: 2.9247 | LR: 2.52e-05 | 3440.2ms/step | 5.8 samples/s +Step 37770/44000 | Loss: 2.8991 | LR: 2.52e-05 | 1288.0ms/step | 15.5 samples/s +Step 37780/44000 | Loss: 2.9051 | LR: 2.51e-05 | 1288.9ms/step | 15.5 samples/s +Step 37790/44000 | Loss: 2.9394 | LR: 2.50e-05 | 1289.0ms/step | 15.5 samples/s +Step 37800/44000 | Loss: 2.9105 | LR: 2.49e-05 | 1288.1ms/step | 15.5 samples/s +Step 37810/44000 | Loss: 2.8835 | LR: 2.48e-05 | 1290.0ms/step | 15.5 samples/s +Step 37820/44000 | Loss: 2.9420 | LR: 2.48e-05 | 1290.4ms/step | 15.5 samples/s +Step 37830/44000 | Loss: 2.9422 | LR: 2.47e-05 | 1289.9ms/step | 15.5 samples/s +Step 37840/44000 | Loss: 2.9270 | LR: 2.46e-05 | 1292.2ms/step | 15.5 samples/s +Step 37850/44000 | Loss: 2.9064 | LR: 2.45e-05 | 1293.8ms/step | 15.5 samples/s +Step 37860/44000 | Loss: 2.8863 | LR: 2.45e-05 | 1292.1ms/step | 15.5 samples/s +Step 37870/44000 | Loss: 2.9108 | LR: 2.44e-05 | 1292.7ms/step | 15.5 samples/s +Step 37880/44000 | Loss: 2.9078 | LR: 2.43e-05 | 1293.5ms/step | 15.5 samples/s +Step 37890/44000 | Loss: 2.9318 | LR: 2.42e-05 | 1293.3ms/step | 15.5 samples/s +Step 37900/44000 | Loss: 2.9346 | LR: 2.41e-05 | 1290.7ms/step | 15.5 samples/s +Step 37910/44000 | Loss: 2.9179 | LR: 2.41e-05 | 1287.2ms/step | 15.5 samples/s +Step 37920/44000 | Loss: 2.9378 | LR: 2.40e-05 | 1292.3ms/step | 15.5 samples/s +Step 37930/44000 | Loss: 2.9144 | LR: 2.39e-05 | 1289.6ms/step | 15.5 samples/s +Step 37940/44000 | Loss: 2.9276 | LR: 2.38e-05 | 1292.3ms/step | 15.5 samples/s +Step 37950/44000 | Loss: 2.9410 | LR: 2.38e-05 | 1293.0ms/step | 15.5 samples/s +Step 37960/44000 | Loss: 2.9583 | LR: 2.37e-05 | 1293.6ms/step | 15.5 samples/s +Step 37970/44000 | Loss: 2.9157 | LR: 2.36e-05 | 1292.2ms/step | 15.5 samples/s +Step 37980/44000 | Loss: 2.9474 | LR: 2.35e-05 | 1291.4ms/step | 15.5 samples/s +Step 37990/44000 | Loss: 2.9156 | LR: 2.34e-05 | 1291.7ms/step | 15.5 samples/s +Step 38000/44000 | Loss: 2.9199 | LR: 2.34e-05 | 1288.7ms/step | 15.5 samples/s +Validation at step 38000: Loss = 2.9344, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-38000 +Step 38010/44000 | Loss: 2.8707 | LR: 2.33e-05 | 3735.6ms/step | 5.4 samples/s +Step 38020/44000 | Loss: 2.9059 | LR: 2.32e-05 | 1289.6ms/step | 15.5 samples/s +Step 38030/44000 | Loss: 2.8730 | LR: 2.31e-05 | 1290.1ms/step | 15.5 samples/s +Step 38040/44000 | Loss: 2.9429 | LR: 2.31e-05 | 1293.0ms/step | 15.5 samples/s +Step 38050/44000 | Loss: 2.9266 | LR: 2.30e-05 | 1291.8ms/step | 15.5 samples/s +Step 38060/44000 | Loss: 2.9320 | LR: 2.29e-05 | 1292.2ms/step | 15.5 samples/s +Step 38070/44000 | Loss: 2.9045 | LR: 2.28e-05 | 1292.2ms/step | 15.5 samples/s +Step 38080/44000 | Loss: 2.9180 | LR: 2.28e-05 | 1291.2ms/step | 15.5 samples/s +Step 38090/44000 | Loss: 2.9312 | LR: 2.27e-05 | 1291.3ms/step | 15.5 samples/s +Step 38100/44000 | Loss: 2.9128 | LR: 2.26e-05 | 1291.8ms/step | 15.5 samples/s +Step 38110/44000 | Loss: 2.9243 | LR: 2.25e-05 | 1289.5ms/step | 15.5 samples/s +Step 38120/44000 | Loss: 2.9363 | LR: 2.25e-05 | 1290.8ms/step | 15.5 samples/s +Step 38130/44000 | Loss: 2.9117 | LR: 2.24e-05 | 1290.7ms/step | 15.5 samples/s +Step 38140/44000 | Loss: 2.9374 | LR: 2.23e-05 | 1291.8ms/step | 15.5 samples/s +Step 38150/44000 | Loss: 2.9560 | LR: 2.22e-05 | 1290.2ms/step | 15.5 samples/s +Step 38160/44000 | Loss: 2.9601 | LR: 2.22e-05 | 1290.3ms/step | 15.5 samples/s +Step 38170/44000 | Loss: 2.9239 | LR: 2.21e-05 | 1288.6ms/step | 15.5 samples/s +Step 38180/44000 | Loss: 2.9521 | LR: 2.20e-05 | 1289.2ms/step | 15.5 samples/s +Step 38190/44000 | Loss: 2.8945 | LR: 2.19e-05 | 1288.6ms/step | 15.5 samples/s +Step 38200/44000 | Loss: 2.9248 | LR: 2.19e-05 | 1288.5ms/step | 15.5 samples/s +Step 38210/44000 | Loss: 2.9233 | LR: 2.18e-05 | 1287.8ms/step | 15.5 samples/s +Step 38220/44000 | Loss: 2.9420 | LR: 2.17e-05 | 1292.3ms/step | 15.5 samples/s +Step 38230/44000 | Loss: 2.9268 | LR: 2.16e-05 | 1289.2ms/step | 15.5 samples/s +Step 38240/44000 | Loss: 2.9280 | LR: 2.16e-05 | 1290.4ms/step | 15.5 samples/s +Step 38250/44000 | Loss: 2.9276 | LR: 2.15e-05 | 1289.3ms/step | 15.5 samples/s +Validation at step 38250: Loss = 2.9343, Perplexity = 18.81 +Step 38260/44000 | Loss: 2.9023 | LR: 2.14e-05 | 3444.5ms/step | 5.8 samples/s +Step 38270/44000 | Loss: 2.9322 | LR: 2.13e-05 | 1290.7ms/step | 15.5 samples/s +Step 38280/44000 | Loss: 2.8908 | LR: 2.13e-05 | 1291.9ms/step | 15.5 samples/s +Step 38290/44000 | Loss: 2.8990 | LR: 2.12e-05 | 1292.0ms/step | 15.5 samples/s +Step 38300/44000 | Loss: 2.9327 | LR: 2.11e-05 | 1291.6ms/step | 15.5 samples/s +Step 38310/44000 | Loss: 2.9037 | LR: 2.11e-05 | 1296.9ms/step | 15.4 samples/s +Step 38320/44000 | Loss: 2.9196 | LR: 2.10e-05 | 1290.2ms/step | 15.5 samples/s +Step 38330/44000 | Loss: 2.9386 | LR: 2.09e-05 | 1292.9ms/step | 15.5 samples/s +Step 38340/44000 | Loss: 2.8879 | LR: 2.08e-05 | 1293.4ms/step | 15.5 samples/s +Step 38350/44000 | Loss: 2.9168 | LR: 2.08e-05 | 1291.8ms/step | 15.5 samples/s +Step 38360/44000 | Loss: 2.9243 | LR: 2.07e-05 | 1289.5ms/step | 15.5 samples/s +Step 38370/44000 | Loss: 2.9320 | LR: 2.06e-05 | 1290.4ms/step | 15.5 samples/s +Step 38380/44000 | Loss: 2.9355 | LR: 2.05e-05 | 1291.9ms/step | 15.5 samples/s +Step 38390/44000 | Loss: 2.9151 | LR: 2.05e-05 | 1291.5ms/step | 15.5 samples/s +Step 38400/44000 | Loss: 2.9352 | LR: 2.04e-05 | 1291.5ms/step | 15.5 samples/s +Step 38410/44000 | Loss: 2.9179 | LR: 2.03e-05 | 1290.9ms/step | 15.5 samples/s +Step 38420/44000 | Loss: 2.9276 | LR: 2.03e-05 | 1291.1ms/step | 15.5 samples/s +Step 38430/44000 | Loss: 2.9326 | LR: 2.02e-05 | 1289.0ms/step | 15.5 samples/s +Step 38440/44000 | Loss: 2.8855 | LR: 2.01e-05 | 1291.0ms/step | 15.5 samples/s +Step 38450/44000 | Loss: 2.9220 | LR: 2.00e-05 | 1289.5ms/step | 15.5 samples/s +Step 38460/44000 | Loss: 2.9213 | LR: 2.00e-05 | 1289.4ms/step | 15.5 samples/s +Step 38470/44000 | Loss: 2.9166 | LR: 1.99e-05 | 1289.9ms/step | 15.5 samples/s +Step 38480/44000 | Loss: 2.9400 | LR: 1.98e-05 | 1289.4ms/step | 15.5 samples/s +Step 38490/44000 | Loss: 2.9275 | LR: 1.98e-05 | 1287.2ms/step | 15.5 samples/s +Step 38500/44000 | Loss: 2.9291 | LR: 1.97e-05 | 1286.1ms/step | 15.6 samples/s +Validation at step 38500: Loss = 2.9342, Perplexity = 18.81 +Step 38510/44000 | Loss: 2.9201 | LR: 1.96e-05 | 3445.4ms/step | 5.8 samples/s +Step 38520/44000 | Loss: 2.8978 | LR: 1.95e-05 | 1289.7ms/step | 15.5 samples/s +Step 38530/44000 | Loss: 2.9190 | LR: 1.95e-05 | 1293.8ms/step | 15.5 samples/s +Step 38540/44000 | Loss: 2.9312 | LR: 1.94e-05 | 1289.7ms/step | 15.5 samples/s +Step 38550/44000 | Loss: 2.9404 | LR: 1.93e-05 | 1290.3ms/step | 15.5 samples/s +Step 38560/44000 | Loss: 2.9237 | LR: 1.93e-05 | 1291.6ms/step | 15.5 samples/s +Step 38570/44000 | Loss: 2.9159 | LR: 1.92e-05 | 1293.6ms/step | 15.5 samples/s +Step 38580/44000 | Loss: 2.8977 | LR: 1.91e-05 | 1291.6ms/step | 15.5 samples/s +Step 38590/44000 | Loss: 2.9263 | LR: 1.91e-05 | 1291.9ms/step | 15.5 samples/s +Step 38600/44000 | Loss: 2.9034 | LR: 1.90e-05 | 1291.8ms/step | 15.5 samples/s +Step 38610/44000 | Loss: 2.9088 | LR: 1.89e-05 | 1291.1ms/step | 15.5 samples/s +Step 38620/44000 | Loss: 2.9290 | LR: 1.88e-05 | 1291.7ms/step | 15.5 samples/s +Step 38630/44000 | Loss: 2.9038 | LR: 1.88e-05 | 1291.2ms/step | 15.5 samples/s +Step 38640/44000 | Loss: 2.9434 | LR: 1.87e-05 | 1289.8ms/step | 15.5 samples/s +Step 38650/44000 | Loss: 2.8990 | LR: 1.86e-05 | 1290.2ms/step | 15.5 samples/s +Step 38660/44000 | Loss: 2.9093 | LR: 1.86e-05 | 1288.5ms/step | 15.5 samples/s +Step 38670/44000 | Loss: 2.9232 | LR: 1.85e-05 | 1289.9ms/step | 15.5 samples/s +Step 38680/44000 | Loss: 2.9466 | LR: 1.84e-05 | 1290.6ms/step | 15.5 samples/s +Step 38690/44000 | Loss: 2.9217 | LR: 1.84e-05 | 1291.2ms/step | 15.5 samples/s +Step 38700/44000 | Loss: 2.9399 | LR: 1.83e-05 | 1292.2ms/step | 15.5 samples/s +Step 38710/44000 | Loss: 2.9250 | LR: 1.82e-05 | 1289.0ms/step | 15.5 samples/s +Step 38720/44000 | Loss: 2.8979 | LR: 1.82e-05 | 1287.6ms/step | 15.5 samples/s +Step 38730/44000 | Loss: 2.9142 | LR: 1.81e-05 | 1292.0ms/step | 15.5 samples/s +Step 38740/44000 | Loss: 2.9343 | LR: 1.80e-05 | 1291.9ms/step | 15.5 samples/s +Step 38750/44000 | Loss: 2.9165 | LR: 1.80e-05 | 1289.4ms/step | 15.5 samples/s +Validation at step 38750: Loss = 2.9342, Perplexity = 18.81 +Step 38760/44000 | Loss: 2.9270 | LR: 1.79e-05 | 3443.4ms/step | 5.8 samples/s +Step 38770/44000 | Loss: 2.9160 | LR: 1.78e-05 | 1287.5ms/step | 15.5 samples/s +Step 38780/44000 | Loss: 2.9072 | LR: 1.78e-05 | 1291.6ms/step | 15.5 samples/s +Step 38790/44000 | Loss: 2.9395 | LR: 1.77e-05 | 1293.1ms/step | 15.5 samples/s +Step 38800/44000 | Loss: 2.9059 | LR: 1.76e-05 | 1292.1ms/step | 15.5 samples/s +Step 38810/44000 | Loss: 2.8891 | LR: 1.76e-05 | 1290.6ms/step | 15.5 samples/s +Step 38820/44000 | Loss: 2.9259 | LR: 1.75e-05 | 1291.4ms/step | 15.5 samples/s +Step 38830/44000 | Loss: 2.9261 | LR: 1.74e-05 | 1289.5ms/step | 15.5 samples/s +Step 38840/44000 | Loss: 2.8900 | LR: 1.74e-05 | 1288.5ms/step | 15.5 samples/s +Step 38850/44000 | Loss: 2.9275 | LR: 1.73e-05 | 1291.9ms/step | 15.5 samples/s +Step 38860/44000 | Loss: 2.9160 | LR: 1.72e-05 | 1291.8ms/step | 15.5 samples/s +Step 38870/44000 | Loss: 2.9334 | LR: 1.72e-05 | 1293.0ms/step | 15.5 samples/s +Step 38880/44000 | Loss: 2.9319 | LR: 1.71e-05 | 1290.8ms/step | 15.5 samples/s +Step 38890/44000 | Loss: 2.9204 | LR: 1.70e-05 | 1293.2ms/step | 15.5 samples/s +Step 38900/44000 | Loss: 2.9168 | LR: 1.70e-05 | 1291.5ms/step | 15.5 samples/s +Step 38910/44000 | Loss: 2.9393 | LR: 1.69e-05 | 1292.6ms/step | 15.5 samples/s +Step 38920/44000 | Loss: 2.9213 | LR: 1.68e-05 | 1290.0ms/step | 15.5 samples/s +Step 38930/44000 | Loss: 2.8993 | LR: 1.68e-05 | 1289.0ms/step | 15.5 samples/s +Step 38940/44000 | Loss: 2.9063 | LR: 1.67e-05 | 1289.7ms/step | 15.5 samples/s +Step 38950/44000 | Loss: 2.8932 | LR: 1.66e-05 | 1287.8ms/step | 15.5 samples/s +Step 38960/44000 | Loss: 2.9114 | LR: 1.66e-05 | 1290.7ms/step | 15.5 samples/s +Step 38970/44000 | Loss: 2.9266 | LR: 1.65e-05 | 1290.9ms/step | 15.5 samples/s +Step 38980/44000 | Loss: 2.8999 | LR: 1.64e-05 | 1288.5ms/step | 15.5 samples/s +Step 38990/44000 | Loss: 2.9312 | LR: 1.64e-05 | 1288.3ms/step | 15.5 samples/s +Step 39000/44000 | Loss: 2.9193 | LR: 1.63e-05 | 1291.2ms/step | 15.5 samples/s +Validation at step 39000: Loss = 2.9342, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-39000 +Step 39010/44000 | Loss: 2.9553 | LR: 1.62e-05 | 3752.7ms/step | 5.3 samples/s +Step 39020/44000 | Loss: 2.9086 | LR: 1.62e-05 | 1285.4ms/step | 15.6 samples/s +Step 39030/44000 | Loss: 2.9467 | LR: 1.61e-05 | 1284.2ms/step | 15.6 samples/s +Step 39040/44000 | Loss: 2.9006 | LR: 1.61e-05 | 1289.3ms/step | 15.5 samples/s +Step 39050/44000 | Loss: 2.9076 | LR: 1.60e-05 | 1289.3ms/step | 15.5 samples/s +Step 39060/44000 | Loss: 2.9199 | LR: 1.59e-05 | 1288.8ms/step | 15.5 samples/s +Step 39070/44000 | Loss: 2.8978 | LR: 1.59e-05 | 1289.1ms/step | 15.5 samples/s +Step 39080/44000 | Loss: 2.9282 | LR: 1.58e-05 | 1289.6ms/step | 15.5 samples/s +Step 39090/44000 | Loss: 2.8888 | LR: 1.57e-05 | 1291.5ms/step | 15.5 samples/s +Step 39100/44000 | Loss: 2.8956 | LR: 1.57e-05 | 1291.1ms/step | 15.5 samples/s +Step 39110/44000 | Loss: 2.9318 | LR: 1.56e-05 | 1290.4ms/step | 15.5 samples/s +Step 39120/44000 | Loss: 2.9230 | LR: 1.55e-05 | 1292.0ms/step | 15.5 samples/s +Step 39130/44000 | Loss: 2.8934 | LR: 1.55e-05 | 1289.8ms/step | 15.5 samples/s +Step 39140/44000 | Loss: 2.9125 | LR: 1.54e-05 | 1288.7ms/step | 15.5 samples/s +Step 39150/44000 | Loss: 2.9255 | LR: 1.54e-05 | 1291.8ms/step | 15.5 samples/s +Step 39160/44000 | Loss: 2.9528 | LR: 1.53e-05 | 1291.5ms/step | 15.5 samples/s +Step 39170/44000 | Loss: 2.9289 | LR: 1.52e-05 | 1291.0ms/step | 15.5 samples/s +Step 39180/44000 | Loss: 2.9325 | LR: 1.52e-05 | 1291.1ms/step | 15.5 samples/s +Step 39190/44000 | Loss: 2.8748 | LR: 1.51e-05 | 1291.1ms/step | 15.5 samples/s +Step 39200/44000 | Loss: 2.8959 | LR: 1.50e-05 | 1295.1ms/step | 15.4 samples/s +Step 39210/44000 | Loss: 2.8858 | LR: 1.50e-05 | 1295.4ms/step | 15.4 samples/s +Step 39220/44000 | Loss: 2.9242 | LR: 1.49e-05 | 1295.4ms/step | 15.4 samples/s +Step 39230/44000 | Loss: 2.9103 | LR: 1.49e-05 | 1289.9ms/step | 15.5 samples/s +Step 39240/44000 | Loss: 2.9125 | LR: 1.48e-05 | 1292.3ms/step | 15.5 samples/s +Step 39250/44000 | Loss: 2.9114 | LR: 1.47e-05 | 1292.4ms/step | 15.5 samples/s +Validation at step 39250: Loss = 2.9341, Perplexity = 18.81 +Step 39260/44000 | Loss: 2.9096 | LR: 1.47e-05 | 3465.1ms/step | 5.8 samples/s +Step 39270/44000 | Loss: 2.9019 | LR: 1.46e-05 | 1294.3ms/step | 15.5 samples/s +Step 39280/44000 | Loss: 2.9154 | LR: 1.46e-05 | 1301.3ms/step | 15.4 samples/s +Step 39290/44000 | Loss: 2.9132 | LR: 1.45e-05 | 1293.8ms/step | 15.5 samples/s +Step 39300/44000 | Loss: 2.9122 | LR: 1.44e-05 | 1289.2ms/step | 15.5 samples/s +Step 39310/44000 | Loss: 2.9035 | LR: 1.44e-05 | 1288.4ms/step | 15.5 samples/s +Step 39320/44000 | Loss: 2.8818 | LR: 1.43e-05 | 1291.6ms/step | 15.5 samples/s +Step 39330/44000 | Loss: 2.9252 | LR: 1.42e-05 | 1290.9ms/step | 15.5 samples/s +Step 39340/44000 | Loss: 2.9218 | LR: 1.42e-05 | 1290.7ms/step | 15.5 samples/s +Step 39350/44000 | Loss: 2.9140 | LR: 1.41e-05 | 1295.0ms/step | 15.4 samples/s +Step 39360/44000 | Loss: 2.9540 | LR: 1.41e-05 | 1290.2ms/step | 15.5 samples/s +Step 39370/44000 | Loss: 2.8983 | LR: 1.40e-05 | 1289.9ms/step | 15.5 samples/s +Step 39380/44000 | Loss: 2.9364 | LR: 1.39e-05 | 1289.3ms/step | 15.5 samples/s +Step 39390/44000 | Loss: 2.9206 | LR: 1.39e-05 | 1286.5ms/step | 15.5 samples/s +Step 39400/44000 | Loss: 2.9094 | LR: 1.38e-05 | 1290.0ms/step | 15.5 samples/s +Step 39410/44000 | Loss: 2.9187 | LR: 1.38e-05 | 1290.9ms/step | 15.5 samples/s +Step 39420/44000 | Loss: 2.9424 | LR: 1.37e-05 | 1288.1ms/step | 15.5 samples/s +Step 39430/44000 | Loss: 2.9390 | LR: 1.36e-05 | 1288.1ms/step | 15.5 samples/s +Step 39440/44000 | Loss: 2.9048 | LR: 1.36e-05 | 1288.1ms/step | 15.5 samples/s +Step 39450/44000 | Loss: 2.9386 | LR: 1.35e-05 | 1288.9ms/step | 15.5 samples/s +Step 39460/44000 | Loss: 2.9014 | LR: 1.35e-05 | 1290.3ms/step | 15.5 samples/s +Step 39470/44000 | Loss: 2.9063 | LR: 1.34e-05 | 1289.7ms/step | 15.5 samples/s +Step 39480/44000 | Loss: 2.9292 | LR: 1.34e-05 | 1289.9ms/step | 15.5 samples/s +Step 39490/44000 | Loss: 2.9310 | LR: 1.33e-05 | 1291.3ms/step | 15.5 samples/s +Step 39500/44000 | Loss: 2.9127 | LR: 1.32e-05 | 1287.1ms/step | 15.5 samples/s +Validation at step 39500: Loss = 2.9341, Perplexity = 18.80 +Step 39510/44000 | Loss: 2.8968 | LR: 1.32e-05 | 3451.1ms/step | 5.8 samples/s +Step 39520/44000 | Loss: 2.8976 | LR: 1.31e-05 | 1293.6ms/step | 15.5 samples/s +Step 39530/44000 | Loss: 2.9157 | LR: 1.31e-05 | 1290.8ms/step | 15.5 samples/s +Step 39540/44000 | Loss: 2.9207 | LR: 1.30e-05 | 1289.0ms/step | 15.5 samples/s +Step 39550/44000 | Loss: 2.9136 | LR: 1.29e-05 | 1289.0ms/step | 15.5 samples/s +Step 39560/44000 | Loss: 2.9272 | LR: 1.29e-05 | 1289.8ms/step | 15.5 samples/s +Step 39570/44000 | Loss: 2.8726 | LR: 1.28e-05 | 1289.6ms/step | 15.5 samples/s +Step 39580/44000 | Loss: 2.9144 | LR: 1.28e-05 | 1289.4ms/step | 15.5 samples/s +Step 39590/44000 | Loss: 2.9023 | LR: 1.27e-05 | 1289.7ms/step | 15.5 samples/s +Step 39600/44000 | Loss: 2.9106 | LR: 1.27e-05 | 1289.3ms/step | 15.5 samples/s +Step 39610/44000 | Loss: 2.9141 | LR: 1.26e-05 | 1289.6ms/step | 15.5 samples/s +Step 39620/44000 | Loss: 2.9210 | LR: 1.25e-05 | 1289.7ms/step | 15.5 samples/s +Step 39630/44000 | Loss: 2.9213 | LR: 1.25e-05 | 1292.0ms/step | 15.5 samples/s +Step 39640/44000 | Loss: 2.9233 | LR: 1.24e-05 | 1295.0ms/step | 15.4 samples/s +Step 39650/44000 | Loss: 2.9226 | LR: 1.24e-05 | 1291.7ms/step | 15.5 samples/s +Step 39660/44000 | Loss: 2.9093 | LR: 1.23e-05 | 1292.0ms/step | 15.5 samples/s +Step 39670/44000 | Loss: 2.9008 | LR: 1.23e-05 | 1291.6ms/step | 15.5 samples/s +Step 39680/44000 | Loss: 2.9330 | LR: 1.22e-05 | 1294.3ms/step | 15.5 samples/s +Step 39690/44000 | Loss: 2.9316 | LR: 1.22e-05 | 1289.9ms/step | 15.5 samples/s +Step 39700/44000 | Loss: 2.9226 | LR: 1.21e-05 | 1288.5ms/step | 15.5 samples/s +Step 39710/44000 | Loss: 2.9604 | LR: 1.20e-05 | 1290.6ms/step | 15.5 samples/s +Step 39720/44000 | Loss: 2.9294 | LR: 1.20e-05 | 1289.1ms/step | 15.5 samples/s +Step 39730/44000 | Loss: 2.9265 | LR: 1.19e-05 | 1289.5ms/step | 15.5 samples/s +Step 39740/44000 | Loss: 2.9301 | LR: 1.19e-05 | 1292.9ms/step | 15.5 samples/s +Step 39750/44000 | Loss: 2.9449 | LR: 1.18e-05 | 1292.7ms/step | 15.5 samples/s +Validation at step 39750: Loss = 2.9341, Perplexity = 18.80 +Step 39760/44000 | Loss: 2.9496 | LR: 1.18e-05 | 3446.2ms/step | 5.8 samples/s +Step 39770/44000 | Loss: 2.9134 | LR: 1.17e-05 | 1291.0ms/step | 15.5 samples/s +Step 39780/44000 | Loss: 2.9163 | LR: 1.17e-05 | 1289.5ms/step | 15.5 samples/s +Step 39790/44000 | Loss: 2.9019 | LR: 1.16e-05 | 1291.4ms/step | 15.5 samples/s +Step 39800/44000 | Loss: 2.9184 | LR: 1.15e-05 | 1289.6ms/step | 15.5 samples/s +Step 39810/44000 | Loss: 2.9028 | LR: 1.15e-05 | 1291.2ms/step | 15.5 samples/s +Step 39820/44000 | Loss: 2.9139 | LR: 1.14e-05 | 1291.9ms/step | 15.5 samples/s +Step 39830/44000 | Loss: 2.9249 | LR: 1.14e-05 | 1291.7ms/step | 15.5 samples/s +Step 39840/44000 | Loss: 2.9390 | LR: 1.13e-05 | 1291.8ms/step | 15.5 samples/s +Step 39850/44000 | Loss: 2.9300 | LR: 1.13e-05 | 1290.1ms/step | 15.5 samples/s +Step 39860/44000 | Loss: 2.9213 | LR: 1.12e-05 | 1289.8ms/step | 15.5 samples/s +Step 39870/44000 | Loss: 2.9133 | LR: 1.12e-05 | 1291.0ms/step | 15.5 samples/s +Step 39880/44000 | Loss: 2.8917 | LR: 1.11e-05 | 1290.6ms/step | 15.5 samples/s +Step 39890/44000 | Loss: 2.9074 | LR: 1.11e-05 | 1289.3ms/step | 15.5 samples/s +Step 39900/44000 | Loss: 2.9216 | LR: 1.10e-05 | 1287.3ms/step | 15.5 samples/s +Step 39910/44000 | Loss: 2.8905 | LR: 1.10e-05 | 1289.0ms/step | 15.5 samples/s +Step 39920/44000 | Loss: 2.9470 | LR: 1.09e-05 | 1290.7ms/step | 15.5 samples/s +Step 39930/44000 | Loss: 2.9199 | LR: 1.08e-05 | 1290.2ms/step | 15.5 samples/s +Step 39940/44000 | Loss: 2.9419 | LR: 1.08e-05 | 1291.0ms/step | 15.5 samples/s +Step 39950/44000 | Loss: 2.9042 | LR: 1.07e-05 | 1293.0ms/step | 15.5 samples/s +Step 39960/44000 | Loss: 2.9226 | LR: 1.07e-05 | 1291.8ms/step | 15.5 samples/s +Step 39970/44000 | Loss: 2.9188 | LR: 1.06e-05 | 1289.9ms/step | 15.5 samples/s +Step 39980/44000 | Loss: 2.9077 | LR: 1.06e-05 | 1290.6ms/step | 15.5 samples/s +Step 39990/44000 | Loss: 2.9330 | LR: 1.05e-05 | 1292.8ms/step | 15.5 samples/s +Step 40000/44000 | Loss: 2.9019 | LR: 1.05e-05 | 1291.3ms/step | 15.5 samples/s +Validation at step 40000: Loss = 2.9341, Perplexity = 18.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-40000 +Step 40010/44000 | Loss: 2.9211 | LR: 1.04e-05 | 3743.3ms/step | 5.3 samples/s +Step 40020/44000 | Loss: 2.9205 | LR: 1.04e-05 | 1290.2ms/step | 15.5 samples/s +Step 40030/44000 | Loss: 2.9314 | LR: 1.03e-05 | 1290.9ms/step | 15.5 samples/s +Step 40040/44000 | Loss: 2.9129 | LR: 1.03e-05 | 1289.7ms/step | 15.5 samples/s +Step 40050/44000 | Loss: 2.8951 | LR: 1.02e-05 | 1290.2ms/step | 15.5 samples/s +Step 40060/44000 | Loss: 2.9028 | LR: 1.02e-05 | 1289.3ms/step | 15.5 samples/s +Step 40070/44000 | Loss: 2.9267 | LR: 1.01e-05 | 1290.1ms/step | 15.5 samples/s +Step 40080/44000 | Loss: 2.9296 | LR: 1.01e-05 | 1287.8ms/step | 15.5 samples/s +Step 40090/44000 | Loss: 2.9093 | LR: 1.00e-05 | 1285.2ms/step | 15.6 samples/s +Step 40100/44000 | Loss: 2.9184 | LR: 9.96e-06 | 1286.4ms/step | 15.5 samples/s +Step 40110/44000 | Loss: 2.9174 | LR: 9.91e-06 | 1287.2ms/step | 15.5 samples/s +Step 40120/44000 | Loss: 2.9327 | LR: 9.86e-06 | 1286.3ms/step | 15.5 samples/s +Step 40130/44000 | Loss: 2.9266 | LR: 9.81e-06 | 1286.2ms/step | 15.6 samples/s +Step 40140/44000 | Loss: 2.9312 | LR: 9.76e-06 | 1287.3ms/step | 15.5 samples/s +Step 40150/44000 | Loss: 2.8904 | LR: 9.71e-06 | 1287.4ms/step | 15.5 samples/s +Step 40160/44000 | Loss: 2.8976 | LR: 9.66e-06 | 1286.9ms/step | 15.5 samples/s +Step 40170/44000 | Loss: 2.9025 | LR: 9.61e-06 | 1292.3ms/step | 15.5 samples/s +Step 40180/44000 | Loss: 2.9428 | LR: 9.56e-06 | 1289.2ms/step | 15.5 samples/s +Step 40190/44000 | Loss: 2.9181 | LR: 9.51e-06 | 1286.4ms/step | 15.5 samples/s +Step 40200/44000 | Loss: 2.9495 | LR: 9.46e-06 | 1288.2ms/step | 15.5 samples/s +Step 40210/44000 | Loss: 2.9161 | LR: 9.41e-06 | 1288.7ms/step | 15.5 samples/s +Step 40220/44000 | Loss: 2.9354 | LR: 9.36e-06 | 1290.6ms/step | 15.5 samples/s +Step 40230/44000 | Loss: 2.9074 | LR: 9.32e-06 | 1290.3ms/step | 15.5 samples/s +Step 40240/44000 | Loss: 2.9173 | LR: 9.27e-06 | 1291.1ms/step | 15.5 samples/s +Step 40250/44000 | Loss: 2.8804 | LR: 9.22e-06 | 1291.2ms/step | 15.5 samples/s +Validation at step 40250: Loss = 2.9341, Perplexity = 18.80 +Step 40260/44000 | Loss: 2.9097 | LR: 9.17e-06 | 3446.1ms/step | 5.8 samples/s +Step 40270/44000 | Loss: 2.9266 | LR: 9.12e-06 | 1289.3ms/step | 15.5 samples/s +Step 40280/44000 | Loss: 2.9232 | LR: 9.07e-06 | 1289.7ms/step | 15.5 samples/s +Step 40290/44000 | Loss: 2.8980 | LR: 9.02e-06 | 1289.5ms/step | 15.5 samples/s +Step 40300/44000 | Loss: 2.9241 | LR: 8.97e-06 | 1287.8ms/step | 15.5 samples/s +Step 40310/44000 | Loss: 2.9120 | LR: 8.93e-06 | 1287.7ms/step | 15.5 samples/s +Step 40320/44000 | Loss: 2.9398 | LR: 8.88e-06 | 1289.4ms/step | 15.5 samples/s +Step 40330/44000 | Loss: 2.9205 | LR: 8.83e-06 | 1291.5ms/step | 15.5 samples/s +Step 40340/44000 | Loss: 2.9181 | LR: 8.78e-06 | 1290.6ms/step | 15.5 samples/s +Step 40350/44000 | Loss: 2.9097 | LR: 8.74e-06 | 1288.7ms/step | 15.5 samples/s +Step 40360/44000 | Loss: 2.9230 | LR: 8.69e-06 | 1289.5ms/step | 15.5 samples/s +Step 40370/44000 | Loss: 2.8996 | LR: 8.64e-06 | 1289.2ms/step | 15.5 samples/s +Step 40380/44000 | Loss: 2.9200 | LR: 8.59e-06 | 1288.9ms/step | 15.5 samples/s +Step 40390/44000 | Loss: 2.9368 | LR: 8.55e-06 | 1289.4ms/step | 15.5 samples/s +Step 40400/44000 | Loss: 2.9056 | LR: 8.50e-06 | 1289.7ms/step | 15.5 samples/s +Step 40410/44000 | Loss: 2.8915 | LR: 8.45e-06 | 1289.7ms/step | 15.5 samples/s +Step 40420/44000 | Loss: 2.9148 | LR: 8.41e-06 | 1291.6ms/step | 15.5 samples/s +Step 40430/44000 | Loss: 2.9238 | LR: 8.36e-06 | 1289.4ms/step | 15.5 samples/s +Step 40440/44000 | Loss: 2.9000 | LR: 8.31e-06 | 1290.9ms/step | 15.5 samples/s +Step 40450/44000 | Loss: 2.9121 | LR: 8.27e-06 | 1291.2ms/step | 15.5 samples/s +Step 40460/44000 | Loss: 2.8810 | LR: 8.22e-06 | 1289.5ms/step | 15.5 samples/s +Step 40470/44000 | Loss: 2.9363 | LR: 8.17e-06 | 1290.7ms/step | 15.5 samples/s +Step 40480/44000 | Loss: 2.9241 | LR: 8.13e-06 | 1291.4ms/step | 15.5 samples/s +Step 40490/44000 | Loss: 2.9249 | LR: 8.08e-06 | 1291.1ms/step | 15.5 samples/s +Step 40500/44000 | Loss: 2.9187 | LR: 8.04e-06 | 1291.6ms/step | 15.5 samples/s +Validation at step 40500: Loss = 2.9340, Perplexity = 18.80 +Step 40510/44000 | Loss: 2.9058 | LR: 7.99e-06 | 3442.9ms/step | 5.8 samples/s +Step 40520/44000 | Loss: 2.9531 | LR: 7.94e-06 | 1284.6ms/step | 15.6 samples/s +Step 40530/44000 | Loss: 2.9416 | LR: 7.90e-06 | 1287.3ms/step | 15.5 samples/s +Step 40540/44000 | Loss: 2.9336 | LR: 7.85e-06 | 1290.2ms/step | 15.5 samples/s +Step 40550/44000 | Loss: 2.8907 | LR: 7.81e-06 | 1290.6ms/step | 15.5 samples/s +Step 40560/44000 | Loss: 2.8906 | LR: 7.76e-06 | 1290.8ms/step | 15.5 samples/s +Step 40570/44000 | Loss: 2.9264 | LR: 7.72e-06 | 1295.4ms/step | 15.4 samples/s +Step 40580/44000 | Loss: 2.9178 | LR: 7.67e-06 | 1294.9ms/step | 15.4 samples/s +Step 40590/44000 | Loss: 2.8942 | LR: 7.63e-06 | 1294.8ms/step | 15.4 samples/s +Step 40600/44000 | Loss: 2.9224 | LR: 7.59e-06 | 1303.4ms/step | 15.3 samples/s +Step 40610/44000 | Loss: 2.9522 | LR: 7.54e-06 | 1297.0ms/step | 15.4 samples/s +Step 40620/44000 | Loss: 2.9179 | LR: 7.50e-06 | 1294.9ms/step | 15.4 samples/s +Step 40630/44000 | Loss: 2.9324 | LR: 7.45e-06 | 1296.4ms/step | 15.4 samples/s +Step 40640/44000 | Loss: 2.9379 | LR: 7.41e-06 | 1291.1ms/step | 15.5 samples/s +Step 40650/44000 | Loss: 2.9367 | LR: 7.37e-06 | 1291.6ms/step | 15.5 samples/s +Step 40660/44000 | Loss: 2.9262 | LR: 7.32e-06 | 1291.6ms/step | 15.5 samples/s +Step 40670/44000 | Loss: 2.9352 | LR: 7.28e-06 | 1290.0ms/step | 15.5 samples/s +Step 40680/44000 | Loss: 2.9200 | LR: 7.23e-06 | 1289.9ms/step | 15.5 samples/s +Step 40690/44000 | Loss: 2.9309 | LR: 7.19e-06 | 1290.6ms/step | 15.5 samples/s +Step 40700/44000 | Loss: 2.9088 | LR: 7.15e-06 | 1290.0ms/step | 15.5 samples/s +Step 40710/44000 | Loss: 2.9243 | LR: 7.10e-06 | 1291.9ms/step | 15.5 samples/s +Step 40720/44000 | Loss: 2.9323 | LR: 7.06e-06 | 1289.8ms/step | 15.5 samples/s +Step 40730/44000 | Loss: 2.9393 | LR: 7.02e-06 | 1292.3ms/step | 15.5 samples/s +Step 40740/44000 | Loss: 2.9121 | LR: 6.98e-06 | 1291.1ms/step | 15.5 samples/s +Step 40750/44000 | Loss: 2.9071 | LR: 6.93e-06 | 1291.0ms/step | 15.5 samples/s +Validation at step 40750: Loss = 2.9340, Perplexity = 18.80 +Step 40760/44000 | Loss: 2.9365 | LR: 6.89e-06 | 3448.9ms/step | 5.8 samples/s +Step 40770/44000 | Loss: 2.9150 | LR: 6.85e-06 | 1289.1ms/step | 15.5 samples/s +Step 40780/44000 | Loss: 2.9361 | LR: 6.81e-06 | 1288.8ms/step | 15.5 samples/s +Step 40790/44000 | Loss: 2.9396 | LR: 6.77e-06 | 1289.9ms/step | 15.5 samples/s +Step 40800/44000 | Loss: 2.9420 | LR: 6.72e-06 | 1289.6ms/step | 15.5 samples/s +Step 40810/44000 | Loss: 2.9218 | LR: 6.68e-06 | 1288.7ms/step | 15.5 samples/s +Step 40820/44000 | Loss: 2.8989 | LR: 6.64e-06 | 1290.7ms/step | 15.5 samples/s +Step 40830/44000 | Loss: 2.8961 | LR: 6.60e-06 | 1289.1ms/step | 15.5 samples/s +Step 40840/44000 | Loss: 2.9400 | LR: 6.56e-06 | 1286.8ms/step | 15.5 samples/s +Step 40850/44000 | Loss: 2.9270 | LR: 6.52e-06 | 1288.1ms/step | 15.5 samples/s +Step 40860/44000 | Loss: 2.9388 | LR: 6.47e-06 | 1288.6ms/step | 15.5 samples/s +Step 40870/44000 | Loss: 2.9420 | LR: 6.43e-06 | 1288.1ms/step | 15.5 samples/s +Step 40880/44000 | Loss: 2.9140 | LR: 6.39e-06 | 1290.3ms/step | 15.5 samples/s +Step 40890/44000 | Loss: 2.9317 | LR: 6.35e-06 | 1291.4ms/step | 15.5 samples/s +Step 40900/44000 | Loss: 2.9073 | LR: 6.31e-06 | 1288.8ms/step | 15.5 samples/s +Step 40910/44000 | Loss: 2.9774 | LR: 6.27e-06 | 1287.5ms/step | 15.5 samples/s +Step 40920/44000 | Loss: 2.9257 | LR: 6.23e-06 | 1288.8ms/step | 15.5 samples/s +Step 40930/44000 | Loss: 2.9334 | LR: 6.19e-06 | 1289.7ms/step | 15.5 samples/s +Step 40940/44000 | Loss: 2.9369 | LR: 6.15e-06 | 1291.5ms/step | 15.5 samples/s +Step 40950/44000 | Loss: 2.8967 | LR: 6.11e-06 | 1288.6ms/step | 15.5 samples/s +Step 40960/44000 | Loss: 2.8854 | LR: 6.07e-06 | 1289.2ms/step | 15.5 samples/s +Step 40970/44000 | Loss: 2.9143 | LR: 6.03e-06 | 1289.2ms/step | 15.5 samples/s +Step 40980/44000 | Loss: 2.9028 | LR: 5.99e-06 | 1288.4ms/step | 15.5 samples/s +Step 40990/44000 | Loss: 2.9244 | LR: 5.95e-06 | 1289.6ms/step | 15.5 samples/s +Step 41000/44000 | Loss: 2.9287 | LR: 5.91e-06 | 1290.4ms/step | 15.5 samples/s +Validation at step 41000: Loss = 2.9340, Perplexity = 18.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-41000 +Step 41010/44000 | Loss: 2.9359 | LR: 5.87e-06 | 3739.6ms/step | 5.3 samples/s +Step 41020/44000 | Loss: 2.9436 | LR: 5.83e-06 | 1288.9ms/step | 15.5 samples/s +Step 41030/44000 | Loss: 2.9182 | LR: 5.80e-06 | 1288.2ms/step | 15.5 samples/s +Step 41040/44000 | Loss: 2.9192 | LR: 5.76e-06 | 1295.1ms/step | 15.4 samples/s +Step 41050/44000 | Loss: 2.9172 | LR: 5.72e-06 | 1289.6ms/step | 15.5 samples/s +Step 41060/44000 | Loss: 2.8882 | LR: 5.68e-06 | 1288.8ms/step | 15.5 samples/s +Step 41070/44000 | Loss: 2.9148 | LR: 5.64e-06 | 1293.8ms/step | 15.5 samples/s +Step 41080/44000 | Loss: 2.9468 | LR: 5.60e-06 | 1289.8ms/step | 15.5 samples/s +Step 41090/44000 | Loss: 2.9626 | LR: 5.56e-06 | 1291.3ms/step | 15.5 samples/s +Step 41100/44000 | Loss: 2.9416 | LR: 5.53e-06 | 1290.5ms/step | 15.5 samples/s +Step 41110/44000 | Loss: 2.9630 | LR: 5.49e-06 | 1289.7ms/step | 15.5 samples/s +Step 41120/44000 | Loss: 2.9498 | LR: 5.45e-06 | 1288.9ms/step | 15.5 samples/s +Step 41130/44000 | Loss: 2.9203 | LR: 5.41e-06 | 1289.4ms/step | 15.5 samples/s +Step 41140/44000 | Loss: 2.8995 | LR: 5.38e-06 | 1287.1ms/step | 15.5 samples/s +Step 41150/44000 | Loss: 2.9400 | LR: 5.34e-06 | 1292.7ms/step | 15.5 samples/s +Step 41160/44000 | Loss: 2.9139 | LR: 5.30e-06 | 1291.4ms/step | 15.5 samples/s +Step 41170/44000 | Loss: 2.9254 | LR: 5.26e-06 | 1287.7ms/step | 15.5 samples/s +Step 41180/44000 | Loss: 2.9004 | LR: 5.23e-06 | 1289.5ms/step | 15.5 samples/s +Step 41190/44000 | Loss: 2.8836 | LR: 5.19e-06 | 1291.3ms/step | 15.5 samples/s +Step 41200/44000 | Loss: 2.9236 | LR: 5.15e-06 | 1292.5ms/step | 15.5 samples/s +Step 41210/44000 | Loss: 2.9321 | LR: 5.12e-06 | 1288.4ms/step | 15.5 samples/s +Step 41220/44000 | Loss: 2.9396 | LR: 5.08e-06 | 1285.7ms/step | 15.6 samples/s +Step 41230/44000 | Loss: 2.9137 | LR: 5.04e-06 | 1287.1ms/step | 15.5 samples/s +Step 41240/44000 | Loss: 2.9020 | LR: 5.01e-06 | 1288.2ms/step | 15.5 samples/s +Step 41250/44000 | Loss: 2.9195 | LR: 4.97e-06 | 1287.5ms/step | 15.5 samples/s +Validation at step 41250: Loss = 2.9340, Perplexity = 18.80 +Step 41260/44000 | Loss: 2.9463 | LR: 4.94e-06 | 3441.9ms/step | 5.8 samples/s +Step 41270/44000 | Loss: 2.9260 | LR: 4.90e-06 | 1287.3ms/step | 15.5 samples/s +Step 41280/44000 | Loss: 2.9225 | LR: 4.86e-06 | 1287.8ms/step | 15.5 samples/s +Step 41290/44000 | Loss: 2.9104 | LR: 4.83e-06 | 1289.7ms/step | 15.5 samples/s +Step 41300/44000 | Loss: 2.9156 | LR: 4.79e-06 | 1289.3ms/step | 15.5 samples/s +Step 41310/44000 | Loss: 2.9032 | LR: 4.76e-06 | 1290.3ms/step | 15.5 samples/s +Step 41320/44000 | Loss: 2.9285 | LR: 4.72e-06 | 1289.7ms/step | 15.5 samples/s +Step 41330/44000 | Loss: 2.9131 | LR: 4.69e-06 | 1288.5ms/step | 15.5 samples/s +Step 41340/44000 | Loss: 2.9368 | LR: 4.65e-06 | 1288.9ms/step | 15.5 samples/s +Step 41350/44000 | Loss: 2.9002 | LR: 4.62e-06 | 1291.0ms/step | 15.5 samples/s +Step 41360/44000 | Loss: 2.9486 | LR: 4.58e-06 | 1291.2ms/step | 15.5 samples/s +Step 41370/44000 | Loss: 2.9304 | LR: 4.55e-06 | 1290.7ms/step | 15.5 samples/s +Step 41380/44000 | Loss: 2.9242 | LR: 4.51e-06 | 1289.9ms/step | 15.5 samples/s +Step 41390/44000 | Loss: 2.9035 | LR: 4.48e-06 | 1288.0ms/step | 15.5 samples/s +Step 41400/44000 | Loss: 2.8907 | LR: 4.45e-06 | 1289.3ms/step | 15.5 samples/s +Step 41410/44000 | Loss: 2.8968 | LR: 4.41e-06 | 1288.4ms/step | 15.5 samples/s +Step 41420/44000 | Loss: 2.8896 | LR: 4.38e-06 | 1289.1ms/step | 15.5 samples/s +Step 41430/44000 | Loss: 2.9153 | LR: 4.34e-06 | 1290.6ms/step | 15.5 samples/s +Step 41440/44000 | Loss: 2.9550 | LR: 4.31e-06 | 1289.7ms/step | 15.5 samples/s +Step 41450/44000 | Loss: 2.9198 | LR: 4.28e-06 | 1292.0ms/step | 15.5 samples/s +Step 41460/44000 | Loss: 2.9205 | LR: 4.24e-06 | 1291.2ms/step | 15.5 samples/s +Step 41470/44000 | Loss: 2.9075 | LR: 4.21e-06 | 1289.7ms/step | 15.5 samples/s +Step 41480/44000 | Loss: 2.9372 | LR: 4.18e-06 | 1294.5ms/step | 15.5 samples/s +Step 41490/44000 | Loss: 2.9055 | LR: 4.14e-06 | 1289.3ms/step | 15.5 samples/s +Step 41500/44000 | Loss: 2.9224 | LR: 4.11e-06 | 1287.7ms/step | 15.5 samples/s +Validation at step 41500: Loss = 2.9340, Perplexity = 18.80 +Step 41510/44000 | Loss: 2.9178 | LR: 4.08e-06 | 3447.0ms/step | 5.8 samples/s +Step 41520/44000 | Loss: 2.9184 | LR: 4.05e-06 | 1292.8ms/step | 15.5 samples/s +Step 41530/44000 | Loss: 2.9228 | LR: 4.01e-06 | 1291.0ms/step | 15.5 samples/s +Step 41540/44000 | Loss: 2.9457 | LR: 3.98e-06 | 1290.6ms/step | 15.5 samples/s +Step 41550/44000 | Loss: 2.9195 | LR: 3.95e-06 | 1292.1ms/step | 15.5 samples/s +Step 41560/44000 | Loss: 2.9315 | LR: 3.92e-06 | 1290.9ms/step | 15.5 samples/s +Step 41570/44000 | Loss: 2.9472 | LR: 3.88e-06 | 1290.2ms/step | 15.5 samples/s +Step 41580/44000 | Loss: 2.9052 | LR: 3.85e-06 | 1292.4ms/step | 15.5 samples/s +Step 41590/44000 | Loss: 2.9184 | LR: 3.82e-06 | 1293.5ms/step | 15.5 samples/s +Step 41600/44000 | Loss: 2.9313 | LR: 3.79e-06 | 1292.1ms/step | 15.5 samples/s +Step 41610/44000 | Loss: 2.9094 | LR: 3.76e-06 | 1291.2ms/step | 15.5 samples/s +Step 41620/44000 | Loss: 2.8852 | LR: 3.73e-06 | 1289.6ms/step | 15.5 samples/s +Step 41630/44000 | Loss: 2.9441 | LR: 3.70e-06 | 1289.1ms/step | 15.5 samples/s +Step 41640/44000 | Loss: 2.9427 | LR: 3.66e-06 | 1290.3ms/step | 15.5 samples/s +Step 41650/44000 | Loss: 2.9250 | LR: 3.63e-06 | 1285.8ms/step | 15.6 samples/s +Step 41660/44000 | Loss: 2.9437 | LR: 3.60e-06 | 1288.5ms/step | 15.5 samples/s +Step 41670/44000 | Loss: 2.9191 | LR: 3.57e-06 | 1291.4ms/step | 15.5 samples/s +Step 41680/44000 | Loss: 2.8912 | LR: 3.54e-06 | 1290.5ms/step | 15.5 samples/s +Step 41690/44000 | Loss: 2.9043 | LR: 3.51e-06 | 1291.5ms/step | 15.5 samples/s +Step 41700/44000 | Loss: 2.9386 | LR: 3.48e-06 | 1291.0ms/step | 15.5 samples/s +Step 41710/44000 | Loss: 2.9101 | LR: 3.45e-06 | 1290.6ms/step | 15.5 samples/s +Step 41720/44000 | Loss: 2.9153 | LR: 3.42e-06 | 1291.1ms/step | 15.5 samples/s +Step 41730/44000 | Loss: 2.9270 | LR: 3.39e-06 | 1289.3ms/step | 15.5 samples/s +Step 41740/44000 | Loss: 2.9380 | LR: 3.36e-06 | 1288.9ms/step | 15.5 samples/s +Step 41750/44000 | Loss: 2.9130 | LR: 3.33e-06 | 1289.1ms/step | 15.5 samples/s +Validation at step 41750: Loss = 2.9340, Perplexity = 18.80 +Step 41760/44000 | Loss: 2.9445 | LR: 3.30e-06 | 3440.6ms/step | 5.8 samples/s +Step 41770/44000 | Loss: 2.9202 | LR: 3.27e-06 | 1287.5ms/step | 15.5 samples/s +Step 41780/44000 | Loss: 2.9121 | LR: 3.24e-06 | 1286.9ms/step | 15.5 samples/s +Step 41790/44000 | Loss: 2.9550 | LR: 3.21e-06 | 1288.0ms/step | 15.5 samples/s +Step 41800/44000 | Loss: 2.9662 | LR: 3.19e-06 | 1290.4ms/step | 15.5 samples/s +Step 41810/44000 | Loss: 2.9307 | LR: 3.16e-06 | 1289.5ms/step | 15.5 samples/s +Step 41820/44000 | Loss: 2.9056 | LR: 3.13e-06 | 1291.0ms/step | 15.5 samples/s +Step 41830/44000 | Loss: 2.9076 | LR: 3.10e-06 | 1290.7ms/step | 15.5 samples/s +Step 41840/44000 | Loss: 2.8899 | LR: 3.07e-06 | 1290.5ms/step | 15.5 samples/s +Step 41850/44000 | Loss: 2.8951 | LR: 3.04e-06 | 1291.6ms/step | 15.5 samples/s +Step 41860/44000 | Loss: 2.9441 | LR: 3.01e-06 | 1290.0ms/step | 15.5 samples/s +Step 41870/44000 | Loss: 2.9538 | LR: 2.99e-06 | 1291.0ms/step | 15.5 samples/s +Step 41880/44000 | Loss: 2.9310 | LR: 2.96e-06 | 1291.7ms/step | 15.5 samples/s +Step 41890/44000 | Loss: 2.9137 | LR: 2.93e-06 | 1286.8ms/step | 15.5 samples/s +Step 41900/44000 | Loss: 2.9154 | LR: 2.90e-06 | 1290.2ms/step | 15.5 samples/s +Step 41910/44000 | Loss: 2.9090 | LR: 2.88e-06 | 1290.2ms/step | 15.5 samples/s +Step 41920/44000 | Loss: 2.9610 | LR: 2.85e-06 | 1289.0ms/step | 15.5 samples/s +Step 41930/44000 | Loss: 2.9123 | LR: 2.82e-06 | 1292.9ms/step | 15.5 samples/s +Step 41940/44000 | Loss: 2.9211 | LR: 2.79e-06 | 1291.9ms/step | 15.5 samples/s +Step 41950/44000 | Loss: 2.8820 | LR: 2.77e-06 | 1292.4ms/step | 15.5 samples/s +Step 41960/44000 | Loss: 2.9301 | LR: 2.74e-06 | 1293.7ms/step | 15.5 samples/s +Step 41970/44000 | Loss: 2.9244 | LR: 2.71e-06 | 1294.2ms/step | 15.5 samples/s +Step 41980/44000 | Loss: 2.9193 | LR: 2.69e-06 | 1288.3ms/step | 15.5 samples/s +Step 41990/44000 | Loss: 2.9220 | LR: 2.66e-06 | 1290.8ms/step | 15.5 samples/s +Step 42000/44000 | Loss: 2.9208 | LR: 2.63e-06 | 1290.8ms/step | 15.5 samples/s +Validation at step 42000: Loss = 2.9340, Perplexity = 18.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-42000 +Step 42010/44000 | Loss: 2.9301 | LR: 2.61e-06 | 3735.8ms/step | 5.4 samples/s +Step 42020/44000 | Loss: 2.9380 | LR: 2.58e-06 | 1288.1ms/step | 15.5 samples/s +Step 42030/44000 | Loss: 2.9447 | LR: 2.56e-06 | 1289.0ms/step | 15.5 samples/s +Step 42040/44000 | Loss: 2.9413 | LR: 2.53e-06 | 1290.3ms/step | 15.5 samples/s +Step 42050/44000 | Loss: 2.9004 | LR: 2.50e-06 | 1288.9ms/step | 15.5 samples/s +Step 42060/44000 | Loss: 2.9366 | LR: 2.48e-06 | 1288.0ms/step | 15.5 samples/s +Step 42070/44000 | Loss: 2.9376 | LR: 2.45e-06 | 1288.8ms/step | 15.5 samples/s +Step 42080/44000 | Loss: 2.9176 | LR: 2.43e-06 | 1289.4ms/step | 15.5 samples/s +Step 42090/44000 | Loss: 2.9028 | LR: 2.40e-06 | 1287.3ms/step | 15.5 samples/s +Step 42100/44000 | Loss: 2.9239 | LR: 2.38e-06 | 1291.2ms/step | 15.5 samples/s +Step 42110/44000 | Loss: 2.9378 | LR: 2.35e-06 | 1287.4ms/step | 15.5 samples/s +Step 42120/44000 | Loss: 2.9242 | LR: 2.33e-06 | 1289.9ms/step | 15.5 samples/s +Step 42130/44000 | Loss: 2.9368 | LR: 2.30e-06 | 1291.1ms/step | 15.5 samples/s +Step 42140/44000 | Loss: 2.9323 | LR: 2.28e-06 | 1289.0ms/step | 15.5 samples/s +Step 42150/44000 | Loss: 2.9537 | LR: 2.25e-06 | 1288.7ms/step | 15.5 samples/s +Step 42160/44000 | Loss: 2.9189 | LR: 2.23e-06 | 1288.4ms/step | 15.5 samples/s +Step 42170/44000 | Loss: 2.9068 | LR: 2.21e-06 | 1290.0ms/step | 15.5 samples/s +Step 42180/44000 | Loss: 2.9339 | LR: 2.18e-06 | 1289.2ms/step | 15.5 samples/s +Step 42190/44000 | Loss: 2.9184 | LR: 2.16e-06 | 1288.6ms/step | 15.5 samples/s +Step 42200/44000 | Loss: 2.9035 | LR: 2.13e-06 | 1288.0ms/step | 15.5 samples/s +Step 42210/44000 | Loss: 2.9487 | LR: 2.11e-06 | 1287.9ms/step | 15.5 samples/s +Step 42220/44000 | Loss: 2.8891 | LR: 2.09e-06 | 1289.8ms/step | 15.5 samples/s +Step 42230/44000 | Loss: 2.9430 | LR: 2.06e-06 | 1290.7ms/step | 15.5 samples/s +Step 42240/44000 | Loss: 2.9208 | LR: 2.04e-06 | 1293.0ms/step | 15.5 samples/s +Step 42250/44000 | Loss: 2.9366 | LR: 2.02e-06 | 1288.5ms/step | 15.5 samples/s +Validation at step 42250: Loss = 2.9340, Perplexity = 18.80 +Step 42260/44000 | Loss: 2.9257 | LR: 1.99e-06 | 3446.6ms/step | 5.8 samples/s +Step 42270/44000 | Loss: 2.9210 | LR: 1.97e-06 | 1289.9ms/step | 15.5 samples/s +Step 42280/44000 | Loss: 2.9213 | LR: 1.95e-06 | 1291.6ms/step | 15.5 samples/s +Step 42290/44000 | Loss: 2.8919 | LR: 1.93e-06 | 1289.9ms/step | 15.5 samples/s +Step 42300/44000 | Loss: 2.9183 | LR: 1.90e-06 | 1291.3ms/step | 15.5 samples/s +Step 42310/44000 | Loss: 2.8922 | LR: 1.88e-06 | 1292.0ms/step | 15.5 samples/s +Step 42320/44000 | Loss: 2.9391 | LR: 1.86e-06 | 1293.0ms/step | 15.5 samples/s +Step 42330/44000 | Loss: 2.9047 | LR: 1.84e-06 | 1292.9ms/step | 15.5 samples/s +Step 42340/44000 | Loss: 2.9487 | LR: 1.82e-06 | 1293.4ms/step | 15.5 samples/s +Step 42350/44000 | Loss: 2.8800 | LR: 1.79e-06 | 1292.3ms/step | 15.5 samples/s +Step 42360/44000 | Loss: 2.9216 | LR: 1.77e-06 | 1291.1ms/step | 15.5 samples/s +Step 42370/44000 | Loss: 2.9215 | LR: 1.75e-06 | 1292.1ms/step | 15.5 samples/s +Step 42380/44000 | Loss: 2.9543 | LR: 1.73e-06 | 1292.6ms/step | 15.5 samples/s +Step 42390/44000 | Loss: 2.8874 | LR: 1.71e-06 | 1292.3ms/step | 15.5 samples/s +Step 42400/44000 | Loss: 2.9145 | LR: 1.69e-06 | 1291.6ms/step | 15.5 samples/s +Step 42410/44000 | Loss: 2.9189 | LR: 1.67e-06 | 1293.3ms/step | 15.5 samples/s +Step 42420/44000 | Loss: 2.9468 | LR: 1.64e-06 | 1292.5ms/step | 15.5 samples/s +Step 42430/44000 | Loss: 2.9495 | LR: 1.62e-06 | 1292.4ms/step | 15.5 samples/s +Step 42440/44000 | Loss: 2.9402 | LR: 1.60e-06 | 1290.7ms/step | 15.5 samples/s +Step 42450/44000 | Loss: 2.9096 | LR: 1.58e-06 | 1290.4ms/step | 15.5 samples/s +Step 42460/44000 | Loss: 2.9192 | LR: 1.56e-06 | 1288.6ms/step | 15.5 samples/s +Step 42470/44000 | Loss: 2.9527 | LR: 1.54e-06 | 1287.8ms/step | 15.5 samples/s +Step 42480/44000 | Loss: 2.9213 | LR: 1.52e-06 | 1288.6ms/step | 15.5 samples/s +Step 42490/44000 | Loss: 2.9287 | LR: 1.50e-06 | 1292.1ms/step | 15.5 samples/s +Step 42500/44000 | Loss: 2.8968 | LR: 1.48e-06 | 1291.5ms/step | 15.5 samples/s +Validation at step 42500: Loss = 2.9340, Perplexity = 18.80 +Step 42510/44000 | Loss: 2.9388 | LR: 1.46e-06 | 3444.8ms/step | 5.8 samples/s +Step 42520/44000 | Loss: 2.8991 | LR: 1.44e-06 | 1286.5ms/step | 15.5 samples/s +Step 42530/44000 | Loss: 2.9274 | LR: 1.42e-06 | 1286.0ms/step | 15.6 samples/s +Step 42540/44000 | Loss: 2.9150 | LR: 1.40e-06 | 1289.7ms/step | 15.5 samples/s +Step 42550/44000 | Loss: 2.9315 | LR: 1.39e-06 | 1286.7ms/step | 15.5 samples/s +Step 42560/44000 | Loss: 2.9037 | LR: 1.37e-06 | 1287.3ms/step | 15.5 samples/s +Step 42570/44000 | Loss: 2.9211 | LR: 1.35e-06 | 1290.3ms/step | 15.5 samples/s +Step 42580/44000 | Loss: 2.9215 | LR: 1.33e-06 | 1288.7ms/step | 15.5 samples/s +Step 42590/44000 | Loss: 2.9058 | LR: 1.31e-06 | 1289.6ms/step | 15.5 samples/s +Step 42600/44000 | Loss: 2.9014 | LR: 1.29e-06 | 1289.4ms/step | 15.5 samples/s +Step 42610/44000 | Loss: 2.9278 | LR: 1.27e-06 | 1289.6ms/step | 15.5 samples/s +Step 42620/44000 | Loss: 2.9370 | LR: 1.25e-06 | 1290.5ms/step | 15.5 samples/s +Step 42630/44000 | Loss: 2.9067 | LR: 1.24e-06 | 1288.3ms/step | 15.5 samples/s +Step 42640/44000 | Loss: 2.9158 | LR: 1.22e-06 | 1289.6ms/step | 15.5 samples/s +Step 42650/44000 | Loss: 2.9367 | LR: 1.20e-06 | 1292.0ms/step | 15.5 samples/s +Step 42660/44000 | Loss: 2.9082 | LR: 1.18e-06 | 1291.2ms/step | 15.5 samples/s +Step 42670/44000 | Loss: 2.9057 | LR: 1.17e-06 | 1289.7ms/step | 15.5 samples/s +Step 42680/44000 | Loss: 2.8999 | LR: 1.15e-06 | 1291.3ms/step | 15.5 samples/s +Step 42690/44000 | Loss: 2.8938 | LR: 1.13e-06 | 1293.3ms/step | 15.5 samples/s +Step 42700/44000 | Loss: 2.8960 | LR: 1.11e-06 | 1291.8ms/step | 15.5 samples/s +Step 42710/44000 | Loss: 2.9344 | LR: 1.10e-06 | 1290.1ms/step | 15.5 samples/s +Step 42720/44000 | Loss: 2.9262 | LR: 1.08e-06 | 1289.3ms/step | 15.5 samples/s +Step 42730/44000 | Loss: 2.9189 | LR: 1.06e-06 | 1290.7ms/step | 15.5 samples/s +Step 42740/44000 | Loss: 2.8941 | LR: 1.05e-06 | 1292.5ms/step | 15.5 samples/s +Step 42750/44000 | Loss: 2.9033 | LR: 1.03e-06 | 1292.0ms/step | 15.5 samples/s +Validation at step 42750: Loss = 2.9340, Perplexity = 18.80 +Step 42760/44000 | Loss: 2.9196 | LR: 1.01e-06 | 3449.9ms/step | 5.8 samples/s +Step 42770/44000 | Loss: 2.9344 | LR: 9.97e-07 | 1292.4ms/step | 15.5 samples/s +Step 42780/44000 | Loss: 2.9394 | LR: 9.81e-07 | 1291.4ms/step | 15.5 samples/s +Step 42790/44000 | Loss: 2.9113 | LR: 9.65e-07 | 1290.8ms/step | 15.5 samples/s +Step 42800/44000 | Loss: 2.9043 | LR: 9.49e-07 | 1292.8ms/step | 15.5 samples/s +Step 42810/44000 | Loss: 2.9014 | LR: 9.33e-07 | 1289.0ms/step | 15.5 samples/s +Step 42820/44000 | Loss: 2.9572 | LR: 9.18e-07 | 1289.0ms/step | 15.5 samples/s +Step 42830/44000 | Loss: 2.9195 | LR: 9.02e-07 | 1287.4ms/step | 15.5 samples/s +Step 42840/44000 | Loss: 2.9301 | LR: 8.87e-07 | 1289.5ms/step | 15.5 samples/s +Step 42850/44000 | Loss: 2.9290 | LR: 8.72e-07 | 1287.5ms/step | 15.5 samples/s +Step 42860/44000 | Loss: 2.9378 | LR: 8.57e-07 | 1287.3ms/step | 15.5 samples/s +Step 42870/44000 | Loss: 2.9350 | LR: 8.42e-07 | 1287.2ms/step | 15.5 samples/s +Step 42880/44000 | Loss: 2.9226 | LR: 8.27e-07 | 1286.9ms/step | 15.5 samples/s +Step 42890/44000 | Loss: 2.8762 | LR: 8.12e-07 | 1290.4ms/step | 15.5 samples/s +Step 42900/44000 | Loss: 2.9326 | LR: 7.98e-07 | 1291.4ms/step | 15.5 samples/s +Step 42910/44000 | Loss: 2.9253 | LR: 7.83e-07 | 1289.8ms/step | 15.5 samples/s +Step 42920/44000 | Loss: 2.9054 | LR: 7.69e-07 | 1289.3ms/step | 15.5 samples/s +Step 42930/44000 | Loss: 2.9014 | LR: 7.55e-07 | 1290.8ms/step | 15.5 samples/s +Step 42940/44000 | Loss: 2.9129 | LR: 7.41e-07 | 1290.6ms/step | 15.5 samples/s +Step 42950/44000 | Loss: 2.9275 | LR: 7.27e-07 | 1291.6ms/step | 15.5 samples/s +Step 42960/44000 | Loss: 2.9197 | LR: 7.13e-07 | 1292.1ms/step | 15.5 samples/s +Step 42970/44000 | Loss: 2.9135 | LR: 6.99e-07 | 1291.3ms/step | 15.5 samples/s +Step 42980/44000 | Loss: 2.9239 | LR: 6.86e-07 | 1291.4ms/step | 15.5 samples/s +Step 42990/44000 | Loss: 2.9196 | LR: 6.72e-07 | 1291.8ms/step | 15.5 samples/s +Step 43000/44000 | Loss: 2.9110 | LR: 6.59e-07 | 1291.8ms/step | 15.5 samples/s +Validation at step 43000: Loss = 2.9340, Perplexity = 18.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-43000 +Step 43010/44000 | Loss: 2.9191 | LR: 6.46e-07 | 3729.5ms/step | 5.4 samples/s +Step 43020/44000 | Loss: 2.9039 | LR: 6.33e-07 | 1290.1ms/step | 15.5 samples/s +Step 43030/44000 | Loss: 2.9247 | LR: 6.20e-07 | 1289.9ms/step | 15.5 samples/s +Step 43040/44000 | Loss: 2.8890 | LR: 6.08e-07 | 1290.9ms/step | 15.5 samples/s +Step 43050/44000 | Loss: 2.9145 | LR: 5.95e-07 | 1286.5ms/step | 15.5 samples/s +Step 43060/44000 | Loss: 2.9239 | LR: 5.83e-07 | 1287.8ms/step | 15.5 samples/s +Step 43070/44000 | Loss: 2.9398 | LR: 5.70e-07 | 1291.5ms/step | 15.5 samples/s +Step 43080/44000 | Loss: 2.9132 | LR: 5.58e-07 | 1287.9ms/step | 15.5 samples/s +Step 43090/44000 | Loss: 2.8965 | LR: 5.46e-07 | 1288.9ms/step | 15.5 samples/s +Step 43100/44000 | Loss: 2.9091 | LR: 5.34e-07 | 1290.3ms/step | 15.5 samples/s +Step 43110/44000 | Loss: 2.9217 | LR: 5.22e-07 | 1291.1ms/step | 15.5 samples/s +Step 43120/44000 | Loss: 2.9339 | LR: 5.11e-07 | 1289.2ms/step | 15.5 samples/s +Step 43130/44000 | Loss: 2.9362 | LR: 4.99e-07 | 1292.2ms/step | 15.5 samples/s +Step 43140/44000 | Loss: 2.9227 | LR: 4.88e-07 | 1286.6ms/step | 15.5 samples/s +Step 43150/44000 | Loss: 2.9079 | LR: 4.76e-07 | 1287.1ms/step | 15.5 samples/s +Step 43160/44000 | Loss: 2.9064 | LR: 4.65e-07 | 1286.2ms/step | 15.6 samples/s +Step 43170/44000 | Loss: 2.8932 | LR: 4.54e-07 | 1287.8ms/step | 15.5 samples/s +Step 43180/44000 | Loss: 2.9056 | LR: 4.43e-07 | 1290.3ms/step | 15.5 samples/s +Step 43190/44000 | Loss: 2.8999 | LR: 4.33e-07 | 1290.2ms/step | 15.5 samples/s +Step 43200/44000 | Loss: 2.8900 | LR: 4.22e-07 | 1289.9ms/step | 15.5 samples/s +Step 43210/44000 | Loss: 2.9241 | LR: 4.12e-07 | 1290.1ms/step | 15.5 samples/s +Step 43220/44000 | Loss: 2.9122 | LR: 4.01e-07 | 1290.7ms/step | 15.5 samples/s +Step 43230/44000 | Loss: 2.9345 | LR: 3.91e-07 | 1288.1ms/step | 15.5 samples/s +Step 43240/44000 | Loss: 2.9314 | LR: 3.81e-07 | 1289.5ms/step | 15.5 samples/s +Step 43250/44000 | Loss: 2.9339 | LR: 3.71e-07 | 1291.7ms/step | 15.5 samples/s +Validation at step 43250: Loss = 2.9340, Perplexity = 18.80 +Step 43260/44000 | Loss: 2.9268 | LR: 3.61e-07 | 3452.5ms/step | 5.8 samples/s +Step 43270/44000 | Loss: 2.9147 | LR: 3.51e-07 | 1292.8ms/step | 15.5 samples/s +Step 43280/44000 | Loss: 2.9218 | LR: 3.42e-07 | 1290.8ms/step | 15.5 samples/s +Step 43290/44000 | Loss: 2.9052 | LR: 3.32e-07 | 1292.9ms/step | 15.5 samples/s +Step 43300/44000 | Loss: 2.9373 | LR: 3.23e-07 | 1293.4ms/step | 15.5 samples/s +Step 43310/44000 | Loss: 2.9410 | LR: 3.14e-07 | 1292.1ms/step | 15.5 samples/s +Step 43320/44000 | Loss: 2.9466 | LR: 3.05e-07 | 1291.7ms/step | 15.5 samples/s +Step 43330/44000 | Loss: 2.9088 | LR: 2.96e-07 | 1290.9ms/step | 15.5 samples/s +Step 43340/44000 | Loss: 2.9293 | LR: 2.87e-07 | 1292.4ms/step | 15.5 samples/s +Step 43350/44000 | Loss: 2.9054 | LR: 2.79e-07 | 1291.3ms/step | 15.5 samples/s +Step 43360/44000 | Loss: 2.9261 | LR: 2.70e-07 | 1290.4ms/step | 15.5 samples/s +Step 43370/44000 | Loss: 2.9299 | LR: 2.62e-07 | 1291.2ms/step | 15.5 samples/s +Step 43380/44000 | Loss: 2.9232 | LR: 2.53e-07 | 1291.3ms/step | 15.5 samples/s +Step 43390/44000 | Loss: 2.8979 | LR: 2.45e-07 | 1290.2ms/step | 15.5 samples/s +Step 43400/44000 | Loss: 2.9143 | LR: 2.37e-07 | 1288.6ms/step | 15.5 samples/s +Step 43410/44000 | Loss: 2.9139 | LR: 2.30e-07 | 1289.1ms/step | 15.5 samples/s +Step 43420/44000 | Loss: 2.8931 | LR: 2.22e-07 | 1288.7ms/step | 15.5 samples/s +Step 43430/44000 | Loss: 2.9082 | LR: 2.14e-07 | 1290.0ms/step | 15.5 samples/s +Step 43440/44000 | Loss: 2.9556 | LR: 2.07e-07 | 1292.9ms/step | 15.5 samples/s +Step 43450/44000 | Loss: 2.9358 | LR: 1.99e-07 | 1291.9ms/step | 15.5 samples/s +Step 43460/44000 | Loss: 2.9271 | LR: 1.92e-07 | 1292.4ms/step | 15.5 samples/s +Step 43470/44000 | Loss: 2.9277 | LR: 1.85e-07 | 1293.1ms/step | 15.5 samples/s +Step 43480/44000 | Loss: 2.8964 | LR: 1.78e-07 | 1290.9ms/step | 15.5 samples/s +Step 43490/44000 | Loss: 2.9156 | LR: 1.72e-07 | 1289.3ms/step | 15.5 samples/s +Step 43500/44000 | Loss: 2.9535 | LR: 1.65e-07 | 1288.3ms/step | 15.5 samples/s +Validation at step 43500: Loss = 2.9340, Perplexity = 18.80 +Step 43510/44000 | Loss: 2.9195 | LR: 1.58e-07 | 3442.7ms/step | 5.8 samples/s +Step 43520/44000 | Loss: 2.9394 | LR: 1.52e-07 | 1290.2ms/step | 15.5 samples/s +Step 43530/44000 | Loss: 2.9034 | LR: 1.46e-07 | 1288.7ms/step | 15.5 samples/s +Step 43540/44000 | Loss: 2.8917 | LR: 1.40e-07 | 1290.1ms/step | 15.5 samples/s +Step 43550/44000 | Loss: 2.9138 | LR: 1.34e-07 | 1288.5ms/step | 15.5 samples/s +Step 43560/44000 | Loss: 2.9143 | LR: 1.28e-07 | 1288.9ms/step | 15.5 samples/s +Step 43570/44000 | Loss: 2.9130 | LR: 1.22e-07 | 1289.1ms/step | 15.5 samples/s +Step 43580/44000 | Loss: 2.9387 | LR: 1.16e-07 | 1289.1ms/step | 15.5 samples/s +Step 43590/44000 | Loss: 2.9145 | LR: 1.11e-07 | 1288.7ms/step | 15.5 samples/s +Step 43600/44000 | Loss: 2.9256 | LR: 1.06e-07 | 1292.1ms/step | 15.5 samples/s +Step 43610/44000 | Loss: 2.8729 | LR: 1.00e-07 | 1290.5ms/step | 15.5 samples/s +Step 43620/44000 | Loss: 2.9115 | LR: 9.52e-08 | 1287.2ms/step | 15.5 samples/s +Step 43630/44000 | Loss: 2.9033 | LR: 9.03e-08 | 1286.8ms/step | 15.5 samples/s +Step 43640/44000 | Loss: 2.9193 | LR: 8.55e-08 | 1287.0ms/step | 15.5 samples/s +Step 43650/44000 | Loss: 2.9221 | LR: 8.08e-08 | 1288.6ms/step | 15.5 samples/s +Step 43660/44000 | Loss: 2.9231 | LR: 7.62e-08 | 1289.5ms/step | 15.5 samples/s +Step 43670/44000 | Loss: 2.9298 | LR: 7.18e-08 | 1289.7ms/step | 15.5 samples/s +Step 43680/44000 | Loss: 2.9385 | LR: 6.75e-08 | 1288.3ms/step | 15.5 samples/s +Step 43690/44000 | Loss: 2.8989 | LR: 6.34e-08 | 1290.2ms/step | 15.5 samples/s +Step 43700/44000 | Loss: 2.9188 | LR: 5.94e-08 | 1289.5ms/step | 15.5 samples/s +Step 43710/44000 | Loss: 2.9369 | LR: 5.55e-08 | 1290.7ms/step | 15.5 samples/s +Step 43720/44000 | Loss: 2.9064 | LR: 5.17e-08 | 1289.4ms/step | 15.5 samples/s +Step 43730/44000 | Loss: 2.9448 | LR: 4.81e-08 | 1287.6ms/step | 15.5 samples/s +Step 43740/44000 | Loss: 2.9507 | LR: 4.46e-08 | 1289.1ms/step | 15.5 samples/s +Step 43750/44000 | Loss: 2.9224 | LR: 4.12e-08 | 1292.1ms/step | 15.5 samples/s +Validation at step 43750: Loss = 2.9340, Perplexity = 18.80 +Step 43760/44000 | Loss: 2.9403 | LR: 3.80e-08 | 3454.3ms/step | 5.8 samples/s +Step 43770/44000 | Loss: 2.9386 | LR: 3.49e-08 | 1290.5ms/step | 15.5 samples/s +Step 43780/44000 | Loss: 2.9210 | LR: 3.19e-08 | 1292.4ms/step | 15.5 samples/s +Step 43790/44000 | Loss: 2.9080 | LR: 2.91e-08 | 1298.3ms/step | 15.4 samples/s +Step 43800/44000 | Loss: 2.9110 | LR: 2.64e-08 | 1300.0ms/step | 15.4 samples/s +Step 43810/44000 | Loss: 2.9191 | LR: 2.38e-08 | 1293.4ms/step | 15.5 samples/s +Step 43820/44000 | Loss: 2.9114 | LR: 2.14e-08 | 1293.7ms/step | 15.5 samples/s +Step 43830/44000 | Loss: 2.9351 | LR: 1.91e-08 | 1291.9ms/step | 15.5 samples/s +Step 43840/44000 | Loss: 2.9189 | LR: 1.69e-08 | 1291.3ms/step | 15.5 samples/s +Step 43850/44000 | Loss: 2.9113 | LR: 1.48e-08 | 1290.2ms/step | 15.5 samples/s +Step 43860/44000 | Loss: 2.9228 | LR: 1.29e-08 | 1293.5ms/step | 15.5 samples/s +Step 43870/44000 | Loss: 2.9375 | LR: 1.11e-08 | 1291.3ms/step | 15.5 samples/s +Step 43880/44000 | Loss: 2.9536 | LR: 9.50e-09 | 1289.9ms/step | 15.5 samples/s +Step 43890/44000 | Loss: 2.9102 | LR: 7.98e-09 | 1291.5ms/step | 15.5 samples/s +Step 43900/44000 | Loss: 2.9192 | LR: 6.60e-09 | 1288.7ms/step | 15.5 samples/s +Step 43910/44000 | Loss: 2.9239 | LR: 5.34e-09 | 1293.0ms/step | 15.5 samples/s +Step 43920/44000 | Loss: 2.9083 | LR: 4.22e-09 | 1291.6ms/step | 15.5 samples/s +Step 43930/44000 | Loss: 2.9005 | LR: 3.23e-09 | 1289.6ms/step | 15.5 samples/s +Step 43940/44000 | Loss: 2.9086 | LR: 2.37e-09 | 1291.2ms/step | 15.5 samples/s +Step 43950/44000 | Loss: 2.9223 | LR: 1.65e-09 | 1287.8ms/step | 15.5 samples/s +Step 43960/44000 | Loss: 2.9007 | LR: 1.06e-09 | 1287.2ms/step | 15.5 samples/s +Step 43970/44000 | Loss: 2.9312 | LR: 5.94e-10 | 1285.3ms/step | 15.6 samples/s +Step 43980/44000 | Loss: 2.9336 | LR: 2.64e-10 | 1289.4ms/step | 15.5 samples/s +Step 43990/44000 | Loss: 2.9253 | LR: 6.60e-11 | 1291.9ms/step | 15.5 samples/s +Step 44000/44000 | Loss: 2.9673 | LR: 0.00e+00 | 1291.2ms/step | 15.5 samples/s +Validation at step 44000: Loss = 2.9340, Perplexity = 18.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_fixed_s2/run_20260707_034600/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss ███▇▇▇▆▅▅▅▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ████▇▅▅▅▄▄▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ███▁████████████████████████▁███████████ +wandb: speed/steps_per_sec ███████████████████▁█████████▁█████▁████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.93402 +wandb: eval/perplexity 18.8031 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1291.24215 +wandb: speed/samples_per_sec 15.48896 +wandb: speed/steps_per_sec 0.77445 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m fixed s2 8k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/lgjoqt2r +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260707_170941-lgjoqt2r/logs +Training complete! diff --git a/epfl_final/logs/l8k365m_pool10.log b/epfl_final/logs/l8k365m_pool10.log new file mode 100644 index 0000000000000000000000000000000000000000..2700eea61023998aa7b2774aa48736170ec42931 --- /dev/null +++ b/epfl_final/logs/l8k365m_pool10.log @@ -0,0 +1,1072 @@ +W0708 20:48:06.039000 254 site-packages/torch/distributed/run.py:851] +W0708 20:48:06.039000 254 site-packages/torch/distributed/run.py:851] ***************************************** +W0708 20:48:06.039000 254 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0708 20:48:06.039000 254 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m pool10 8k nrb31 freqabl", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "resampled", + "freqablation", + "pool10" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32", + "sparse_mask_seed": 0, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ti3a1x4c +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_205109-ti3a1x4c +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m pool10 8k nrb31 freqabl +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ti3a1x4c +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0382 | LR: 6.67e-06 | 6783.2ms/step | 2.9 samples/s +Step 20/44000 | Loss: 11.9252 | LR: 1.33e-05 | 671.2ms/step | 29.8 samples/s +Step 30/44000 | Loss: 11.5508 | LR: 2.00e-05 | 670.7ms/step | 29.8 samples/s +Step 40/44000 | Loss: 11.0227 | LR: 2.67e-05 | 668.7ms/step | 29.9 samples/s +Step 50/44000 | Loss: 10.7142 | LR: 3.33e-05 | 667.4ms/step | 30.0 samples/s +Step 60/44000 | Loss: 10.4834 | LR: 4.00e-05 | 666.6ms/step | 30.0 samples/s +Step 70/44000 | Loss: 10.2641 | LR: 4.67e-05 | 667.3ms/step | 30.0 samples/s +Step 80/44000 | Loss: 9.9467 | LR: 5.33e-05 | 665.5ms/step | 30.1 samples/s +Step 90/44000 | Loss: 9.5576 | LR: 6.00e-05 | 670.1ms/step | 29.8 samples/s +Step 100/44000 | Loss: 9.1273 | LR: 6.67e-05 | 669.9ms/step | 29.9 samples/s +Step 110/44000 | Loss: 8.6009 | LR: 7.33e-05 | 669.8ms/step | 29.9 samples/s +Step 120/44000 | Loss: 8.1706 | LR: 8.00e-05 | 670.7ms/step | 29.8 samples/s +Step 130/44000 | Loss: 7.8000 | LR: 8.67e-05 | 669.7ms/step | 29.9 samples/s +Step 140/44000 | Loss: 7.5213 | LR: 9.33e-05 | 671.1ms/step | 29.8 samples/s +Step 150/44000 | Loss: 7.3526 | LR: 1.00e-04 | 671.4ms/step | 29.8 samples/s +Step 160/44000 | Loss: 7.1745 | LR: 1.07e-04 | 672.9ms/step | 29.7 samples/s +Step 170/44000 | Loss: 7.1121 | LR: 1.13e-04 | 672.2ms/step | 29.8 samples/s +Step 180/44000 | Loss: 7.0473 | LR: 1.20e-04 | 673.6ms/step | 29.7 samples/s +Step 190/44000 | Loss: 6.9656 | LR: 1.27e-04 | 674.7ms/step | 29.6 samples/s +Step 200/44000 | Loss: 6.8609 | LR: 1.33e-04 | 674.4ms/step | 29.7 samples/s +Step 210/44000 | Loss: 6.7756 | LR: 1.40e-04 | 675.2ms/step | 29.6 samples/s +Step 220/44000 | Loss: 6.6990 | LR: 1.47e-04 | 675.0ms/step | 29.6 samples/s +Step 230/44000 | Loss: 6.5972 | LR: 1.53e-04 | 676.2ms/step | 29.6 samples/s +Step 240/44000 | Loss: 6.5657 | LR: 1.60e-04 | 676.3ms/step | 29.6 samples/s +Step 250/44000 | Loss: 6.4418 | LR: 1.67e-04 | 675.6ms/step | 29.6 samples/s +Validation at step 250: Loss = 6.4439, Perplexity = 628.86 +Step 260/44000 | Loss: 6.4414 | LR: 1.73e-04 | 3935.5ms/step | 5.1 samples/s +Step 270/44000 | Loss: 6.3787 | LR: 1.80e-04 | 676.7ms/step | 29.6 samples/s +Step 280/44000 | Loss: 6.2945 | LR: 1.87e-04 | 676.6ms/step | 29.6 samples/s +Step 290/44000 | Loss: 6.2513 | LR: 1.93e-04 | 678.2ms/step | 29.5 samples/s +Step 300/44000 | Loss: 6.2507 | LR: 2.00e-04 | 676.8ms/step | 29.6 samples/s +Step 310/44000 | Loss: 6.2086 | LR: 2.07e-04 | 679.3ms/step | 29.4 samples/s +Step 320/44000 | Loss: 6.1304 | LR: 2.13e-04 | 677.8ms/step | 29.5 samples/s +Step 330/44000 | Loss: 6.0840 | LR: 2.20e-04 | 678.7ms/step | 29.5 samples/s +Step 340/44000 | Loss: 6.0627 | LR: 2.27e-04 | 678.2ms/step | 29.5 samples/s +Step 350/44000 | Loss: 5.9852 | LR: 2.33e-04 | 679.5ms/step | 29.4 samples/s +Step 360/44000 | Loss: 5.9827 | LR: 2.40e-04 | 679.9ms/step | 29.4 samples/s +Step 370/44000 | Loss: 5.8705 | LR: 2.47e-04 | 678.0ms/step | 29.5 samples/s +Step 380/44000 | Loss: 5.8938 | LR: 2.53e-04 | 677.8ms/step | 29.5 samples/s +Step 390/44000 | Loss: 5.8599 | LR: 2.60e-04 | 678.1ms/step | 29.5 samples/s +Step 400/44000 | Loss: 5.8131 | LR: 2.67e-04 | 678.1ms/step | 29.5 samples/s +Step 410/44000 | Loss: 5.7854 | LR: 2.73e-04 | 678.6ms/step | 29.5 samples/s +Step 420/44000 | Loss: 5.7321 | LR: 2.80e-04 | 678.1ms/step | 29.5 samples/s +Step 430/44000 | Loss: 5.7388 | LR: 2.87e-04 | 678.6ms/step | 29.5 samples/s +Step 440/44000 | Loss: 5.7220 | LR: 2.93e-04 | 678.3ms/step | 29.5 samples/s +Step 450/44000 | Loss: 5.6070 | LR: 3.00e-04 | 680.3ms/step | 29.4 samples/s +Step 460/44000 | Loss: 5.6095 | LR: 3.07e-04 | 677.9ms/step | 29.5 samples/s +Step 470/44000 | Loss: 5.5903 | LR: 3.13e-04 | 678.9ms/step | 29.5 samples/s +Step 480/44000 | Loss: 5.5445 | LR: 3.20e-04 | 679.7ms/step | 29.4 samples/s +Step 490/44000 | Loss: 5.5292 | LR: 3.27e-04 | 679.7ms/step | 29.4 samples/s +Step 500/44000 | Loss: 5.4936 | LR: 3.33e-04 | 678.5ms/step | 29.5 samples/s +Validation at step 500: Loss = 5.4865, Perplexity = 241.40 +Step 510/44000 | Loss: 5.3963 | LR: 3.40e-04 | 1233.9ms/step | 16.2 samples/s +Step 520/44000 | Loss: 5.3932 | LR: 3.47e-04 | 680.1ms/step | 29.4 samples/s +Step 530/44000 | Loss: 5.3959 | LR: 3.53e-04 | 679.0ms/step | 29.5 samples/s +Step 540/44000 | Loss: 5.3567 | LR: 3.60e-04 | 678.1ms/step | 29.5 samples/s +Step 550/44000 | Loss: 5.3682 | LR: 3.67e-04 | 679.1ms/step | 29.5 samples/s +Step 560/44000 | Loss: 5.3081 | LR: 3.73e-04 | 679.1ms/step | 29.4 samples/s +Step 570/44000 | Loss: 5.3110 | LR: 3.80e-04 | 678.3ms/step | 29.5 samples/s +Step 580/44000 | Loss: 5.2360 | LR: 3.87e-04 | 678.2ms/step | 29.5 samples/s +Step 590/44000 | Loss: 5.2242 | LR: 3.93e-04 | 680.0ms/step | 29.4 samples/s +Step 600/44000 | Loss: 5.2097 | LR: 4.00e-04 | 678.8ms/step | 29.5 samples/s +Step 610/44000 | Loss: 5.1316 | LR: 4.07e-04 | 679.2ms/step | 29.4 samples/s +Step 620/44000 | Loss: 5.1077 | LR: 4.13e-04 | 679.4ms/step | 29.4 samples/s +Step 630/44000 | Loss: 5.1647 | LR: 4.20e-04 | 678.6ms/step | 29.5 samples/s +Step 640/44000 | Loss: 5.1062 | LR: 4.27e-04 | 678.7ms/step | 29.5 samples/s +Step 650/44000 | Loss: 5.0631 | LR: 4.33e-04 | 677.8ms/step | 29.5 samples/s +Step 660/44000 | Loss: 5.0732 | LR: 4.40e-04 | 678.6ms/step | 29.5 samples/s +Step 670/44000 | Loss: 5.0441 | LR: 4.47e-04 | 679.0ms/step | 29.5 samples/s +Step 680/44000 | Loss: 5.0482 | LR: 4.53e-04 | 680.4ms/step | 29.4 samples/s +Step 690/44000 | Loss: 4.9983 | LR: 4.60e-04 | 681.3ms/step | 29.4 samples/s +Step 700/44000 | Loss: 4.9409 | LR: 4.67e-04 | 679.7ms/step | 29.4 samples/s +Step 710/44000 | Loss: 4.9727 | LR: 4.73e-04 | 680.2ms/step | 29.4 samples/s +Step 720/44000 | Loss: 4.8840 | LR: 4.80e-04 | 680.4ms/step | 29.4 samples/s +Step 730/44000 | Loss: 4.9085 | LR: 4.87e-04 | 679.2ms/step | 29.4 samples/s +Step 740/44000 | Loss: 4.8286 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s +Step 750/44000 | Loss: 4.7973 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s +Validation at step 750: Loss = 4.8014, Perplexity = 121.68 +Step 760/44000 | Loss: 4.7792 | LR: 5.00e-04 | 1233.5ms/step | 16.2 samples/s +Step 770/44000 | Loss: 4.8051 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s +Step 780/44000 | Loss: 4.7135 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s +Step 790/44000 | Loss: 4.7176 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 800/44000 | Loss: 4.7575 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s +Step 810/44000 | Loss: 4.6824 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s +Step 820/44000 | Loss: 4.6420 | LR: 5.00e-04 | 678.4ms/step | 29.5 samples/s +Step 830/44000 | Loss: 4.6077 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s +Step 840/44000 | Loss: 4.5562 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s +Step 850/44000 | Loss: 4.5649 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 860/44000 | Loss: 4.5487 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 870/44000 | Loss: 4.5591 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s +Step 880/44000 | Loss: 4.4944 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s +Step 890/44000 | Loss: 4.4827 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s +Step 900/44000 | Loss: 4.4674 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s +Step 910/44000 | Loss: 4.4617 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s +Step 920/44000 | Loss: 4.4134 | LR: 5.00e-04 | 679.3ms/step | 29.4 samples/s +Step 930/44000 | Loss: 4.4133 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s +Step 940/44000 | Loss: 4.3863 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s +Step 950/44000 | Loss: 4.4044 | LR: 5.00e-04 | 677.9ms/step | 29.5 samples/s +Step 960/44000 | Loss: 4.3391 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s +Step 970/44000 | Loss: 4.3446 | LR: 5.00e-04 | 678.9ms/step | 29.5 samples/s +Step 980/44000 | Loss: 4.3478 | LR: 5.00e-04 | 677.2ms/step | 29.5 samples/s +Step 990/44000 | Loss: 4.3144 | LR: 5.00e-04 | 678.0ms/step | 29.5 samples/s +Step 1000/44000 | Loss: 4.3165 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s +Validation at step 1000: Loss = 4.3113, Perplexity = 74.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-1000 +Step 1010/44000 | Loss: 4.3391 | LR: 5.00e-04 | 1524.7ms/step | 13.1 samples/s +Step 1020/44000 | Loss: 4.2807 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s +Step 1030/44000 | Loss: 4.2830 | LR: 5.00e-04 | 678.1ms/step | 29.5 samples/s +Step 1040/44000 | Loss: 4.2130 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s +Step 1050/44000 | Loss: 4.2590 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s +Step 1060/44000 | Loss: 4.2902 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s +Step 1070/44000 | Loss: 4.2019 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 1080/44000 | Loss: 4.2037 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s +Step 1090/44000 | Loss: 4.2202 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s +Step 1100/44000 | Loss: 4.1972 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 1110/44000 | Loss: 4.2082 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s +Step 1120/44000 | Loss: 4.1097 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 1130/44000 | Loss: 4.1362 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s +Step 1140/44000 | Loss: 4.1240 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s +Step 1150/44000 | Loss: 4.1284 | LR: 5.00e-04 | 678.9ms/step | 29.5 samples/s +Step 1160/44000 | Loss: 4.1355 | LR: 5.00e-04 | 679.7ms/step | 29.4 samples/s +Step 1170/44000 | Loss: 4.1230 | LR: 5.00e-04 | 679.1ms/step | 29.5 samples/s +Step 1180/44000 | Loss: 4.1392 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s +Step 1190/44000 | Loss: 4.0935 | LR: 5.00e-04 | 679.1ms/step | 29.4 samples/s +Step 1200/44000 | Loss: 4.0989 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s +Step 1210/44000 | Loss: 4.1222 | LR: 5.00e-04 | 679.0ms/step | 29.5 samples/s +Step 1220/44000 | Loss: 4.1045 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s +Step 1230/44000 | Loss: 4.0698 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s +Step 1240/44000 | Loss: 4.0740 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s +Step 1250/44000 | Loss: 4.0511 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s +Validation at step 1250: Loss = 4.0719, Perplexity = 58.67 +Step 1260/44000 | Loss: 4.1055 | LR: 5.00e-04 | 1235.6ms/step | 16.2 samples/s +Step 1270/44000 | Loss: 4.0299 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s +Step 1280/44000 | Loss: 4.0458 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s +Step 1290/44000 | Loss: 4.0374 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s +Step 1300/44000 | Loss: 4.0425 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s +Step 1310/44000 | Loss: 3.9859 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s +Step 1320/44000 | Loss: 3.9726 | LR: 5.00e-04 | 679.5ms/step | 29.4 samples/s +Step 1330/44000 | Loss: 3.9942 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s +Step 1340/44000 | Loss: 4.0381 | LR: 5.00e-04 | 680.3ms/step | 29.4 samples/s +Step 1350/44000 | Loss: 3.9667 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s +Step 1360/44000 | Loss: 3.9881 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s +Step 1370/44000 | Loss: 3.9616 | LR: 5.00e-04 | 679.1ms/step | 29.4 samples/s +Step 1380/44000 | Loss: 3.9808 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s +Step 1390/44000 | Loss: 3.9663 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 1400/44000 | Loss: 3.9699 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s +Step 1410/44000 | Loss: 3.9459 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s +Step 1420/44000 | Loss: 3.9498 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s +Step 1430/44000 | Loss: 3.9607 | LR: 5.00e-04 | 677.7ms/step | 29.5 samples/s +Step 1440/44000 | Loss: 3.8915 | LR: 5.00e-04 | 678.4ms/step | 29.5 samples/s +Step 1450/44000 | Loss: 3.9141 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s +Step 1460/44000 | Loss: 3.9595 | LR: 5.00e-04 | 678.8ms/step | 29.5 samples/s +Step 1470/44000 | Loss: 3.9038 | LR: 5.00e-04 | 679.2ms/step | 29.4 samples/s +Step 1480/44000 | Loss: 3.8795 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s +Step 1490/44000 | Loss: 3.8771 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s +Step 1500/44000 | Loss: 3.9326 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s +Validation at step 1500: Loss = 3.9134, Perplexity = 50.07 +Step 1510/44000 | Loss: 3.9028 | LR: 5.00e-04 | 1233.6ms/step | 16.2 samples/s +Step 1520/44000 | Loss: 3.8939 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s +Step 1530/44000 | Loss: 3.9195 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s +Step 1540/44000 | Loss: 3.8397 | LR: 5.00e-04 | 680.2ms/step | 29.4 samples/s +Step 1550/44000 | Loss: 3.9178 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 1560/44000 | Loss: 3.8780 | LR: 5.00e-04 | 678.8ms/step | 29.5 samples/s +Step 1570/44000 | Loss: 3.8966 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 1580/44000 | Loss: 3.8516 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s +Step 1590/44000 | Loss: 3.8534 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s +Step 1600/44000 | Loss: 3.8467 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s +Step 1610/44000 | Loss: 3.8594 | LR: 5.00e-04 | 679.7ms/step | 29.4 samples/s +Step 1620/44000 | Loss: 3.8355 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s +Step 1630/44000 | Loss: 3.8507 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s +Step 1640/44000 | Loss: 3.8121 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s +Step 1650/44000 | Loss: 3.8513 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s +Step 1660/44000 | Loss: 3.8623 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s +Step 1670/44000 | Loss: 3.8700 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s +Step 1680/44000 | Loss: 3.8120 | LR: 4.99e-04 | 678.2ms/step | 29.5 samples/s +Step 1690/44000 | Loss: 3.7932 | LR: 4.99e-04 | 678.1ms/step | 29.5 samples/s +Step 1700/44000 | Loss: 3.8350 | LR: 4.99e-04 | 679.8ms/step | 29.4 samples/s +Step 1710/44000 | Loss: 3.7861 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s +Step 1720/44000 | Loss: 3.8389 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s +Step 1730/44000 | Loss: 3.8071 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s +Step 1740/44000 | Loss: 3.7915 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s +Step 1750/44000 | Loss: 3.7694 | LR: 4.99e-04 | 678.9ms/step | 29.5 samples/s +Validation at step 1750: Loss = 3.8036, Perplexity = 44.86 +Step 1760/44000 | Loss: 3.8084 | LR: 4.99e-04 | 1234.0ms/step | 16.2 samples/s +Step 1770/44000 | Loss: 3.8108 | LR: 4.99e-04 | 679.8ms/step | 29.4 samples/s +Step 1780/44000 | Loss: 3.8069 | LR: 4.99e-04 | 679.3ms/step | 29.4 samples/s +Step 1790/44000 | Loss: 3.7994 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s +Step 1800/44000 | Loss: 3.7784 | LR: 4.99e-04 | 680.0ms/step | 29.4 samples/s +Step 1810/44000 | Loss: 3.7567 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s +Step 1820/44000 | Loss: 3.7668 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s +Step 1830/44000 | Loss: 3.7511 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s +Step 1840/44000 | Loss: 3.7417 | LR: 4.99e-04 | 678.8ms/step | 29.5 samples/s +Step 1850/44000 | Loss: 3.7737 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s +Step 1860/44000 | Loss: 3.8001 | LR: 4.99e-04 | 678.0ms/step | 29.5 samples/s +Step 1870/44000 | Loss: 3.7855 | LR: 4.99e-04 | 678.1ms/step | 29.5 samples/s +Step 1880/44000 | Loss: 3.7448 | LR: 4.99e-04 | 680.2ms/step | 29.4 samples/s +Step 1890/44000 | Loss: 3.7230 | LR: 4.99e-04 | 678.4ms/step | 29.5 samples/s +Step 1900/44000 | Loss: 3.7565 | LR: 4.99e-04 | 679.6ms/step | 29.4 samples/s +Step 1910/44000 | Loss: 3.7632 | LR: 4.99e-04 | 680.1ms/step | 29.4 samples/s +Step 1920/44000 | Loss: 3.7395 | LR: 4.99e-04 | 679.9ms/step | 29.4 samples/s +Step 1930/44000 | Loss: 3.7510 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s +Step 1940/44000 | Loss: 3.7140 | LR: 4.99e-04 | 679.0ms/step | 29.5 samples/s +Step 1950/44000 | Loss: 3.6843 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s +Step 1960/44000 | Loss: 3.7313 | LR: 4.99e-04 | 680.3ms/step | 29.4 samples/s +Step 1970/44000 | Loss: 3.6992 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s +Step 1980/44000 | Loss: 3.6899 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s +Step 1990/44000 | Loss: 3.7579 | LR: 4.99e-04 | 679.2ms/step | 29.4 samples/s +Step 2000/44000 | Loss: 3.7184 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s +Validation at step 2000: Loss = 3.7220, Perplexity = 41.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-2000 +Step 2010/44000 | Loss: 3.7271 | LR: 4.99e-04 | 1526.4ms/step | 13.1 samples/s +Step 2020/44000 | Loss: 3.7128 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s +Step 2030/44000 | Loss: 3.6913 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s +Step 2040/44000 | Loss: 3.7242 | LR: 4.99e-04 | 676.4ms/step | 29.6 samples/s +Step 2050/44000 | Loss: 3.7057 | LR: 4.99e-04 | 677.0ms/step | 29.5 samples/s +Step 2060/44000 | Loss: 3.7248 | LR: 4.99e-04 | 676.5ms/step | 29.6 samples/s +Step 2070/44000 | Loss: 3.7039 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s +Step 2080/44000 | Loss: 3.7238 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s +Step 2090/44000 | Loss: 3.7226 | LR: 4.99e-04 | 677.6ms/step | 29.5 samples/s +Step 2100/44000 | Loss: 3.6758 | LR: 4.99e-04 | 678.8ms/step | 29.5 samples/s +Step 2110/44000 | Loss: 3.7050 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s +Step 2120/44000 | Loss: 3.6907 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s +Step 2130/44000 | Loss: 3.6842 | LR: 4.99e-04 | 679.1ms/step | 29.4 samples/s +Step 2140/44000 | Loss: 3.6678 | LR: 4.99e-04 | 680.2ms/step | 29.4 samples/s +Step 2150/44000 | Loss: 3.6774 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s +Step 2160/44000 | Loss: 3.6910 | LR: 4.99e-04 | 678.2ms/step | 29.5 samples/s +Step 2170/44000 | Loss: 3.6656 | LR: 4.99e-04 | 677.4ms/step | 29.5 samples/s +Step 2180/44000 | Loss: 3.6832 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s +Step 2190/44000 | Loss: 3.6646 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s +Step 2200/44000 | Loss: 3.6518 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s +Step 2210/44000 | Loss: 3.6337 | LR: 4.99e-04 | 681.1ms/step | 29.4 samples/s +Step 2220/44000 | Loss: 3.6290 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s +Step 2230/44000 | Loss: 3.6832 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s +Step 2240/44000 | Loss: 3.6783 | LR: 4.99e-04 | 678.9ms/step | 29.5 samples/s +Step 2250/44000 | Loss: 3.6675 | LR: 4.99e-04 | 679.2ms/step | 29.4 samples/s +Validation at step 2250: Loss = 3.6566, Perplexity = 38.73 +Step 2260/44000 | Loss: 3.6536 | LR: 4.98e-04 | 1235.3ms/step | 16.2 samples/s +Step 2270/44000 | Loss: 3.6374 | LR: 4.98e-04 | 678.3ms/step | 29.5 samples/s +Step 2280/44000 | Loss: 3.6526 | LR: 4.98e-04 | 684.8ms/step | 29.2 samples/s +Step 2290/44000 | Loss: 3.6724 | LR: 4.98e-04 | 679.6ms/step | 29.4 samples/s +Step 2300/44000 | Loss: 3.6020 | LR: 4.98e-04 | 678.9ms/step | 29.5 samples/s +Step 2310/44000 | Loss: 3.6434 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s +Step 2320/44000 | Loss: 3.6527 | LR: 4.98e-04 | 680.1ms/step | 29.4 samples/s +Step 2330/44000 | Loss: 3.6444 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s +Step 2340/44000 | Loss: 3.6432 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s +Step 2350/44000 | Loss: 3.6070 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s +Step 2360/44000 | Loss: 3.6312 | LR: 4.98e-04 | 678.6ms/step | 29.5 samples/s +Step 2370/44000 | Loss: 3.6329 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s +Step 2380/44000 | Loss: 3.5914 | LR: 4.98e-04 | 679.4ms/step | 29.4 samples/s +Step 2390/44000 | Loss: 3.5777 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s +Step 2400/44000 | Loss: 3.6346 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s +Step 2410/44000 | Loss: 3.6268 | LR: 4.98e-04 | 680.8ms/step | 29.4 samples/s +Step 2420/44000 | Loss: 3.6248 | LR: 4.98e-04 | 678.7ms/step | 29.5 samples/s +Step 2430/44000 | Loss: 3.6167 | LR: 4.98e-04 | 680.8ms/step | 29.4 samples/s +Step 2440/44000 | Loss: 3.6231 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s +Step 2450/44000 | Loss: 3.5986 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s +Step 2460/44000 | Loss: 3.6018 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s +Step 2470/44000 | Loss: 3.6084 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s +Step 2480/44000 | Loss: 3.6210 | LR: 4.98e-04 | 680.5ms/step | 29.4 samples/s +Step 2490/44000 | Loss: 3.5856 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s +Step 2500/44000 | Loss: 3.5987 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s +Validation at step 2500: Loss = 3.6030, Perplexity = 36.71 +Step 2510/44000 | Loss: 3.6005 | LR: 4.98e-04 | 1235.9ms/step | 16.2 samples/s +Step 2520/44000 | Loss: 3.5890 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s +Step 2530/44000 | Loss: 3.6016 | LR: 4.98e-04 | 680.3ms/step | 29.4 samples/s +Step 2540/44000 | Loss: 3.5777 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s +Step 2550/44000 | Loss: 3.5435 | LR: 4.98e-04 | 677.7ms/step | 29.5 samples/s +Step 2560/44000 | Loss: 3.5676 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s +Step 2570/44000 | Loss: 3.5774 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s +Step 2580/44000 | Loss: 3.5951 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s +Step 2590/44000 | Loss: 3.5799 | LR: 4.98e-04 | 679.3ms/step | 29.4 samples/s +Step 2600/44000 | Loss: 3.5484 | LR: 4.98e-04 | 678.9ms/step | 29.5 samples/s +Step 2610/44000 | Loss: 3.5159 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s +Step 2620/44000 | Loss: 3.5791 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s +Step 2630/44000 | Loss: 3.5372 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s +Step 2640/44000 | Loss: 3.5582 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s +Step 2650/44000 | Loss: 3.5712 | LR: 4.98e-04 | 679.3ms/step | 29.4 samples/s +Step 2660/44000 | Loss: 3.5420 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s +Step 2670/44000 | Loss: 3.5468 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s +Step 2680/44000 | Loss: 3.5787 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s +Step 2690/44000 | Loss: 3.5915 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s +Step 2700/44000 | Loss: 3.5287 | LR: 4.97e-04 | 680.3ms/step | 29.4 samples/s +Step 2710/44000 | Loss: 3.5807 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s +Step 2720/44000 | Loss: 3.5960 | LR: 4.97e-04 | 680.0ms/step | 29.4 samples/s +Step 2730/44000 | Loss: 3.5774 | LR: 4.97e-04 | 680.2ms/step | 29.4 samples/s +Step 2740/44000 | Loss: 3.5472 | LR: 4.97e-04 | 680.0ms/step | 29.4 samples/s +Step 2750/44000 | Loss: 3.5272 | LR: 4.97e-04 | 678.7ms/step | 29.5 samples/s +Validation at step 2750: Loss = 3.5600, Perplexity = 35.16 +Step 2760/44000 | Loss: 3.5219 | LR: 4.97e-04 | 1235.3ms/step | 16.2 samples/s +Step 2770/44000 | Loss: 3.5977 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s +Step 2780/44000 | Loss: 3.5897 | LR: 4.97e-04 | 679.2ms/step | 29.4 samples/s +Step 2790/44000 | Loss: 3.5675 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s +Step 2800/44000 | Loss: 3.5683 | LR: 4.97e-04 | 679.1ms/step | 29.4 samples/s +Step 2810/44000 | Loss: 3.5227 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s +Step 2820/44000 | Loss: 3.5505 | LR: 4.97e-04 | 678.3ms/step | 29.5 samples/s +Step 2830/44000 | Loss: 3.5359 | LR: 4.97e-04 | 678.1ms/step | 29.5 samples/s +Step 2840/44000 | Loss: 3.5107 | LR: 4.97e-04 | 680.1ms/step | 29.4 samples/s +Step 2850/44000 | Loss: 3.5808 | LR: 4.97e-04 | 679.7ms/step | 29.4 samples/s +Step 2860/44000 | Loss: 3.5556 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s +Step 2870/44000 | Loss: 3.5299 | LR: 4.97e-04 | 681.9ms/step | 29.3 samples/s +Step 2880/44000 | Loss: 3.4852 | LR: 4.97e-04 | 678.8ms/step | 29.5 samples/s +Step 2890/44000 | Loss: 3.5212 | LR: 4.97e-04 | 679.0ms/step | 29.5 samples/s +Step 2900/44000 | Loss: 3.5748 | LR: 4.97e-04 | 679.8ms/step | 29.4 samples/s +Step 2910/44000 | Loss: 3.5205 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s +Step 2920/44000 | Loss: 3.5393 | LR: 4.97e-04 | 681.4ms/step | 29.4 samples/s +Step 2930/44000 | Loss: 3.5234 | LR: 4.97e-04 | 681.2ms/step | 29.4 samples/s +Step 2940/44000 | Loss: 3.5169 | LR: 4.97e-04 | 679.8ms/step | 29.4 samples/s +Step 2950/44000 | Loss: 3.5483 | LR: 4.97e-04 | 679.1ms/step | 29.5 samples/s +Step 2960/44000 | Loss: 3.5279 | LR: 4.97e-04 | 679.4ms/step | 29.4 samples/s +Step 2970/44000 | Loss: 3.5482 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s +Step 2980/44000 | Loss: 3.6133 | LR: 4.97e-04 | 679.4ms/step | 29.4 samples/s +Step 2990/44000 | Loss: 3.5098 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s +Step 3000/44000 | Loss: 3.5279 | LR: 4.97e-04 | 678.3ms/step | 29.5 samples/s +Validation at step 3000: Loss = 3.5247, Perplexity = 33.94 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-3000 +Step 3010/44000 | Loss: 3.5169 | LR: 4.97e-04 | 1523.9ms/step | 13.1 samples/s +Step 3020/44000 | Loss: 3.5089 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s +Step 3030/44000 | Loss: 3.5554 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s +Step 3040/44000 | Loss: 3.5123 | LR: 4.97e-04 | 679.2ms/step | 29.4 samples/s +Step 3050/44000 | Loss: 3.5534 | LR: 4.97e-04 | 678.6ms/step | 29.5 samples/s +Step 3060/44000 | Loss: 3.4931 | LR: 4.96e-04 | 678.3ms/step | 29.5 samples/s +Step 3070/44000 | Loss: 3.5219 | LR: 4.96e-04 | 679.7ms/step | 29.4 samples/s +Step 3080/44000 | Loss: 3.5287 | LR: 4.96e-04 | 680.1ms/step | 29.4 samples/s +Step 3090/44000 | Loss: 3.4914 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s +Step 3100/44000 | Loss: 3.5144 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s +Step 3110/44000 | Loss: 3.4814 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s +Step 3120/44000 | Loss: 3.4979 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s +Step 3130/44000 | Loss: 3.4916 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s +Step 3140/44000 | Loss: 3.4857 | LR: 4.96e-04 | 678.6ms/step | 29.5 samples/s +Step 3150/44000 | Loss: 3.4805 | LR: 4.96e-04 | 680.9ms/step | 29.4 samples/s +Step 3160/44000 | Loss: 3.5001 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s +Step 3170/44000 | Loss: 3.4908 | LR: 4.96e-04 | 679.1ms/step | 29.4 samples/s +Step 3180/44000 | Loss: 3.5278 | LR: 4.96e-04 | 678.6ms/step | 29.5 samples/s +Step 3190/44000 | Loss: 3.5230 | LR: 4.96e-04 | 679.3ms/step | 29.4 samples/s +Step 3200/44000 | Loss: 3.4853 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s +Step 3210/44000 | Loss: 3.4717 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s +Step 3220/44000 | Loss: 3.5071 | LR: 4.96e-04 | 679.2ms/step | 29.4 samples/s +Step 3230/44000 | Loss: 3.4747 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s +Step 3240/44000 | Loss: 3.4899 | LR: 4.96e-04 | 678.4ms/step | 29.5 samples/s +Step 3250/44000 | Loss: 3.4701 | LR: 4.96e-04 | 678.3ms/step | 29.5 samples/s +Validation at step 3250: Loss = 3.4909, Perplexity = 32.82 +Step 3260/44000 | Loss: 3.5190 | LR: 4.96e-04 | 1243.6ms/step | 16.1 samples/s +Step 3270/44000 | Loss: 3.4723 | LR: 4.96e-04 | 681.4ms/step | 29.4 samples/s +Step 3280/44000 | Loss: 3.4731 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s +Step 3290/44000 | Loss: 3.4796 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s +Step 3300/44000 | Loss: 3.4657 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s +Step 3310/44000 | Loss: 3.4818 | LR: 4.96e-04 | 678.8ms/step | 29.5 samples/s +Step 3320/44000 | Loss: 3.4657 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s +Step 3330/44000 | Loss: 3.4865 | LR: 4.96e-04 | 679.2ms/step | 29.4 samples/s +Step 3340/44000 | Loss: 3.4610 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s +Step 3350/44000 | Loss: 3.4906 | LR: 4.96e-04 | 679.6ms/step | 29.4 samples/s +Step 3360/44000 | Loss: 3.4482 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s +Step 3370/44000 | Loss: 3.4783 | LR: 4.95e-04 | 680.1ms/step | 29.4 samples/s +Step 3380/44000 | Loss: 3.4658 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s +Step 3390/44000 | Loss: 3.4788 | LR: 4.95e-04 | 679.5ms/step | 29.4 samples/s +Step 3400/44000 | Loss: 3.4584 | LR: 4.95e-04 | 680.0ms/step | 29.4 samples/s +Step 3410/44000 | Loss: 3.4768 | LR: 4.95e-04 | 679.1ms/step | 29.4 samples/s +Step 3420/44000 | Loss: 3.3836 | LR: 4.95e-04 | 680.6ms/step | 29.4 samples/s +Step 3430/44000 | Loss: 3.4377 | LR: 4.95e-04 | 680.8ms/step | 29.4 samples/s +Step 3440/44000 | Loss: 3.4689 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s +Step 3450/44000 | Loss: 3.4345 | LR: 4.95e-04 | 678.8ms/step | 29.5 samples/s +Step 3460/44000 | Loss: 3.4805 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s +Step 3470/44000 | Loss: 3.4827 | LR: 4.95e-04 | 680.1ms/step | 29.4 samples/s +Step 3480/44000 | Loss: 3.4605 | LR: 4.95e-04 | 680.3ms/step | 29.4 samples/s +Step 3490/44000 | Loss: 3.4687 | LR: 4.95e-04 | 679.3ms/step | 29.4 samples/s +Step 3500/44000 | Loss: 3.4217 | LR: 4.95e-04 | 680.3ms/step | 29.4 samples/s +Validation at step 3500: Loss = 3.4627, Perplexity = 31.90 +Step 3510/44000 | Loss: 3.4708 | LR: 4.95e-04 | 1234.0ms/step | 16.2 samples/s +Step 3520/44000 | Loss: 3.4309 | LR: 4.95e-04 | 679.2ms/step | 29.4 samples/s +Step 3530/44000 | Loss: 3.4098 | LR: 4.95e-04 | 679.4ms/step | 29.4 samples/s +Step 3540/44000 | Loss: 3.4374 | LR: 4.95e-04 | 679.3ms/step | 29.4 samples/s +Step 3550/44000 | Loss: 3.4533 | LR: 4.95e-04 | 680.2ms/step | 29.4 samples/s +Step 3560/44000 | Loss: 3.4913 | LR: 4.95e-04 | 680.6ms/step | 29.4 samples/s +Step 3570/44000 | Loss: 3.4153 | LR: 4.95e-04 | 680.2ms/step | 29.4 samples/s +Step 3580/44000 | Loss: 3.4416 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s +Step 3590/44000 | Loss: 3.4475 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s +Step 3600/44000 | Loss: 3.4678 | LR: 4.95e-04 | 678.7ms/step | 29.5 samples/s +Step 3610/44000 | Loss: 3.4923 | LR: 4.95e-04 | 681.2ms/step | 29.4 samples/s +Step 3620/44000 | Loss: 3.4472 | LR: 4.95e-04 | 678.6ms/step | 29.5 samples/s +Step 3630/44000 | Loss: 3.4275 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s +Step 3640/44000 | Loss: 3.4488 | LR: 4.95e-04 | 679.7ms/step | 29.4 samples/s +Step 3650/44000 | Loss: 3.4783 | LR: 4.94e-04 | 679.3ms/step | 29.4 samples/s +Step 3660/44000 | Loss: 3.4170 | LR: 4.94e-04 | 678.3ms/step | 29.5 samples/s +Step 3670/44000 | Loss: 3.4606 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s +Step 3680/44000 | Loss: 3.4019 | LR: 4.94e-04 | 678.2ms/step | 29.5 samples/s +Step 3690/44000 | Loss: 3.4091 | LR: 4.94e-04 | 679.1ms/step | 29.4 samples/s +Step 3700/44000 | Loss: 3.4554 | LR: 4.94e-04 | 678.5ms/step | 29.5 samples/s +Step 3710/44000 | Loss: 3.4502 | LR: 4.94e-04 | 679.8ms/step | 29.4 samples/s +Step 3720/44000 | Loss: 3.4324 | LR: 4.94e-04 | 679.4ms/step | 29.4 samples/s +Step 3730/44000 | Loss: 3.4059 | LR: 4.94e-04 | 680.7ms/step | 29.4 samples/s +Step 3740/44000 | Loss: 3.4658 | LR: 4.94e-04 | 679.9ms/step | 29.4 samples/s +Step 3750/44000 | Loss: 3.4212 | LR: 4.94e-04 | 682.5ms/step | 29.3 samples/s +Validation at step 3750: Loss = 3.4375, Perplexity = 31.11 +Step 3760/44000 | Loss: 3.4491 | LR: 4.94e-04 | 1234.2ms/step | 16.2 samples/s +Step 3770/44000 | Loss: 3.4293 | LR: 4.94e-04 | 679.2ms/step | 29.4 samples/s +Step 3780/44000 | Loss: 3.4265 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s +Step 3790/44000 | Loss: 3.4338 | LR: 4.94e-04 | 680.9ms/step | 29.4 samples/s +Step 3800/44000 | Loss: 3.4025 | LR: 4.94e-04 | 680.3ms/step | 29.4 samples/s +Step 3810/44000 | Loss: 3.3858 | LR: 4.94e-04 | 680.1ms/step | 29.4 samples/s +Step 3820/44000 | Loss: 3.3710 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s +Step 3830/44000 | Loss: 3.4053 | LR: 4.94e-04 | 680.0ms/step | 29.4 samples/s +Step 3840/44000 | Loss: 3.4694 | LR: 4.94e-04 | 679.6ms/step | 29.4 samples/s +Step 3850/44000 | Loss: 3.4273 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s +Step 3860/44000 | Loss: 3.4149 | LR: 4.94e-04 | 679.7ms/step | 29.4 samples/s +Step 3870/44000 | Loss: 3.4650 | LR: 4.94e-04 | 678.9ms/step | 29.5 samples/s +Step 3880/44000 | Loss: 3.4368 | LR: 4.94e-04 | 678.9ms/step | 29.5 samples/s +Step 3890/44000 | Loss: 3.4628 | LR: 4.94e-04 | 681.1ms/step | 29.4 samples/s +Step 3900/44000 | Loss: 3.4075 | LR: 4.93e-04 | 679.9ms/step | 29.4 samples/s +Step 3910/44000 | Loss: 3.4479 | LR: 4.93e-04 | 677.4ms/step | 29.5 samples/s +Step 3920/44000 | Loss: 3.4100 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s +Step 3930/44000 | Loss: 3.4134 | LR: 4.93e-04 | 679.7ms/step | 29.4 samples/s +Step 3940/44000 | Loss: 3.4183 | LR: 4.93e-04 | 680.4ms/step | 29.4 samples/s +Step 3950/44000 | Loss: 3.3578 | LR: 4.93e-04 | 681.2ms/step | 29.4 samples/s +Step 3960/44000 | Loss: 3.4280 | LR: 4.93e-04 | 680.6ms/step | 29.4 samples/s +Step 3970/44000 | Loss: 3.4507 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s +Step 3980/44000 | Loss: 3.3956 | LR: 4.93e-04 | 679.0ms/step | 29.5 samples/s +Step 3990/44000 | Loss: 3.3910 | LR: 4.93e-04 | 680.8ms/step | 29.4 samples/s +Step 4000/44000 | Loss: 3.4093 | LR: 4.93e-04 | 679.1ms/step | 29.4 samples/s +Validation at step 4000: Loss = 3.4173, Perplexity = 30.49 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-4000 +Step 4010/44000 | Loss: 3.4274 | LR: 4.93e-04 | 1541.5ms/step | 13.0 samples/s +Step 4020/44000 | Loss: 3.4592 | LR: 4.93e-04 | 680.5ms/step | 29.4 samples/s +Step 4030/44000 | Loss: 3.3700 | LR: 4.93e-04 | 679.1ms/step | 29.5 samples/s +Step 4040/44000 | Loss: 3.3770 | LR: 4.93e-04 | 679.2ms/step | 29.4 samples/s +Step 4050/44000 | Loss: 3.3930 | LR: 4.93e-04 | 677.3ms/step | 29.5 samples/s +Step 4060/44000 | Loss: 3.3864 | LR: 4.93e-04 | 678.8ms/step | 29.5 samples/s +Step 4070/44000 | Loss: 3.4125 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s +Step 4080/44000 | Loss: 3.4098 | LR: 4.93e-04 | 681.7ms/step | 29.3 samples/s +Step 4090/44000 | Loss: 3.4028 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s +Step 4100/44000 | Loss: 3.3925 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s +Step 4110/44000 | Loss: 3.4158 | LR: 4.93e-04 | 679.5ms/step | 29.4 samples/s +Step 4120/44000 | Loss: 3.4153 | LR: 4.93e-04 | 680.1ms/step | 29.4 samples/s +Step 4130/44000 | Loss: 3.4069 | LR: 4.93e-04 | 680.1ms/step | 29.4 samples/s +Step 4140/44000 | Loss: 3.4074 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s +Step 4150/44000 | Loss: 3.3860 | LR: 4.92e-04 | 678.3ms/step | 29.5 samples/s +Step 4160/44000 | Loss: 3.4024 | LR: 4.92e-04 | 677.9ms/step | 29.5 samples/s +Step 4170/44000 | Loss: 3.3716 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s +Step 4180/44000 | Loss: 3.3577 | LR: 4.92e-04 | 679.4ms/step | 29.4 samples/s +Step 4190/44000 | Loss: 3.4122 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s +Step 4200/44000 | Loss: 3.3779 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s +Step 4210/44000 | Loss: 3.4211 | LR: 4.92e-04 | 679.9ms/step | 29.4 samples/s +Step 4220/44000 | Loss: 3.3901 | LR: 4.92e-04 | 680.5ms/step | 29.4 samples/s +Step 4230/44000 | Loss: 3.4316 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s +Step 4240/44000 | Loss: 3.3808 | LR: 4.92e-04 | 680.2ms/step | 29.4 samples/s +Step 4250/44000 | Loss: 3.4069 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s +Validation at step 4250: Loss = 3.3935, Perplexity = 29.77 +Step 4260/44000 | Loss: 3.3471 | LR: 4.92e-04 | 1234.2ms/step | 16.2 samples/s +Step 4270/44000 | Loss: 3.3606 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s +Step 4280/44000 | Loss: 3.4021 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s +Step 4290/44000 | Loss: 3.3672 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s +Step 4300/44000 | Loss: 3.4024 | LR: 4.92e-04 | 678.4ms/step | 29.5 samples/s +Step 4310/44000 | Loss: 3.3616 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s +Step 4320/44000 | Loss: 3.3728 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s +Step 4330/44000 | Loss: 3.3557 | LR: 4.92e-04 | 677.8ms/step | 29.5 samples/s +Step 4340/44000 | Loss: 3.3848 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s +Step 4350/44000 | Loss: 3.3405 | LR: 4.92e-04 | 679.4ms/step | 29.4 samples/s +Step 4360/44000 | Loss: 3.3781 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s +Step 4370/44000 | Loss: 3.3946 | LR: 4.91e-04 | 680.6ms/step | 29.4 samples/s +Step 4380/44000 | Loss: 3.3441 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s +Step 4390/44000 | Loss: 3.3535 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s +Step 4400/44000 | Loss: 3.4129 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s +Step 4410/44000 | Loss: 3.3832 | LR: 4.91e-04 | 677.8ms/step | 29.5 samples/s +Step 4420/44000 | Loss: 3.3845 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s +Step 4430/44000 | Loss: 3.3751 | LR: 4.91e-04 | 679.1ms/step | 29.5 samples/s +Step 4440/44000 | Loss: 3.3846 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s +Step 4450/44000 | Loss: 3.3631 | LR: 4.91e-04 | 678.6ms/step | 29.5 samples/s +Step 4460/44000 | Loss: 3.3365 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s +Step 4470/44000 | Loss: 3.3994 | LR: 4.91e-04 | 679.4ms/step | 29.4 samples/s +Step 4480/44000 | Loss: 3.3678 | LR: 4.91e-04 | 680.4ms/step | 29.4 samples/s +Step 4490/44000 | Loss: 3.3312 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s +Step 4500/44000 | Loss: 3.3581 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s +Validation at step 4500: Loss = 3.3756, Perplexity = 29.24 +Step 4510/44000 | Loss: 3.3550 | LR: 4.91e-04 | 1235.7ms/step | 16.2 samples/s +Step 4520/44000 | Loss: 3.3489 | LR: 4.91e-04 | 680.2ms/step | 29.4 samples/s +Step 4530/44000 | Loss: 3.3322 | LR: 4.91e-04 | 680.4ms/step | 29.4 samples/s +Step 4540/44000 | Loss: 3.3758 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s +Step 4550/44000 | Loss: 3.3705 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s +Step 4560/44000 | Loss: 3.3866 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s +Step 4570/44000 | Loss: 3.3803 | LR: 4.90e-04 | 679.3ms/step | 29.4 samples/s +Step 4580/44000 | Loss: 3.3635 | LR: 4.90e-04 | 679.8ms/step | 29.4 samples/s +Step 4590/44000 | Loss: 3.3132 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s +Step 4600/44000 | Loss: 3.3504 | LR: 4.90e-04 | 679.7ms/step | 29.4 samples/s +Step 4610/44000 | Loss: 3.3584 | LR: 4.90e-04 | 678.7ms/step | 29.5 samples/s +Step 4620/44000 | Loss: 3.3225 | LR: 4.90e-04 | 679.3ms/step | 29.4 samples/s +Step 4630/44000 | Loss: 3.3467 | LR: 4.90e-04 | 678.1ms/step | 29.5 samples/s +Step 4640/44000 | Loss: 3.3604 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s +Step 4650/44000 | Loss: 3.3247 | LR: 4.90e-04 | 680.5ms/step | 29.4 samples/s +Step 4660/44000 | Loss: 3.3196 | LR: 4.90e-04 | 679.1ms/step | 29.5 samples/s +Step 4670/44000 | Loss: 3.3940 | LR: 4.90e-04 | 678.5ms/step | 29.5 samples/s +Step 4680/44000 | Loss: 3.3543 | LR: 4.90e-04 | 680.0ms/step | 29.4 samples/s +Step 4690/44000 | Loss: 3.4117 | LR: 4.90e-04 | 678.3ms/step | 29.5 samples/s +Step 4700/44000 | Loss: 3.3923 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s +Step 4710/44000 | Loss: 3.3810 | LR: 4.90e-04 | 678.0ms/step | 29.5 samples/s +Step 4720/44000 | Loss: 3.3317 | LR: 4.90e-04 | 678.5ms/step | 29.5 samples/s +Step 4730/44000 | Loss: 3.3653 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s +Step 4740/44000 | Loss: 3.3559 | LR: 4.90e-04 | 678.2ms/step | 29.5 samples/s +Step 4750/44000 | Loss: 3.3137 | LR: 4.90e-04 | 678.6ms/step | 29.5 samples/s +Validation at step 4750: Loss = 3.3607, Perplexity = 28.81 +Step 4760/44000 | Loss: 3.3728 | LR: 4.89e-04 | 1234.3ms/step | 16.2 samples/s +Step 4770/44000 | Loss: 3.3637 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s +Step 4780/44000 | Loss: 3.3674 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s +Step 4790/44000 | Loss: 3.3470 | LR: 4.89e-04 | 678.9ms/step | 29.5 samples/s +Step 4800/44000 | Loss: 3.3552 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s +Step 4810/44000 | Loss: 3.3407 | LR: 4.89e-04 | 679.7ms/step | 29.4 samples/s +Step 4820/44000 | Loss: 3.3467 | LR: 4.89e-04 | 680.5ms/step | 29.4 samples/s +Step 4830/44000 | Loss: 3.3377 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s +Step 4840/44000 | Loss: 3.3188 | LR: 4.89e-04 | 679.7ms/step | 29.4 samples/s +Step 4850/44000 | Loss: 3.3334 | LR: 4.89e-04 | 683.2ms/step | 29.3 samples/s +Step 4860/44000 | Loss: 3.3322 | LR: 4.89e-04 | 681.0ms/step | 29.4 samples/s +Step 4870/44000 | Loss: 3.3272 | LR: 4.89e-04 | 680.9ms/step | 29.4 samples/s +Step 4880/44000 | Loss: 3.3476 | LR: 4.89e-04 | 681.2ms/step | 29.4 samples/s +Step 4890/44000 | Loss: 3.3145 | LR: 4.89e-04 | 680.5ms/step | 29.4 samples/s +Step 4900/44000 | Loss: 3.3838 | LR: 4.89e-04 | 685.0ms/step | 29.2 samples/s +Step 4910/44000 | Loss: 3.3637 | LR: 4.89e-04 | 688.5ms/step | 29.1 samples/s +Step 4920/44000 | Loss: 3.3630 | LR: 4.89e-04 | 689.4ms/step | 29.0 samples/s +Step 4930/44000 | Loss: 3.3524 | LR: 4.89e-04 | 687.1ms/step | 29.1 samples/s +Step 4940/44000 | Loss: 3.3524 | LR: 4.89e-04 | 683.6ms/step | 29.3 samples/s +Step 4950/44000 | Loss: 3.3078 | LR: 4.88e-04 | 683.8ms/step | 29.2 samples/s +Step 4960/44000 | Loss: 3.3596 | LR: 4.88e-04 | 684.7ms/step | 29.2 samples/s +Step 4970/44000 | Loss: 3.3109 | LR: 4.88e-04 | 684.1ms/step | 29.2 samples/s +Step 4980/44000 | Loss: 3.3340 | LR: 4.88e-04 | 683.6ms/step | 29.3 samples/s +Step 4990/44000 | Loss: 3.3376 | LR: 4.88e-04 | 682.6ms/step | 29.3 samples/s +Step 5000/44000 | Loss: 3.3362 | LR: 4.88e-04 | 684.5ms/step | 29.2 samples/s +Validation at step 5000: Loss = 3.3428, Perplexity = 28.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-5000 +Step 5010/44000 | Loss: 3.3187 | LR: 4.88e-04 | 1530.9ms/step | 13.1 samples/s +Step 5020/44000 | Loss: 3.3284 | LR: 4.88e-04 | 683.4ms/step | 29.3 samples/s +Step 5030/44000 | Loss: 3.3320 | LR: 4.88e-04 | 682.6ms/step | 29.3 samples/s +Step 5040/44000 | Loss: 3.3599 | LR: 4.88e-04 | 682.4ms/step | 29.3 samples/s +Step 5050/44000 | Loss: 3.3305 | LR: 4.88e-04 | 681.7ms/step | 29.3 samples/s +Step 5060/44000 | Loss: 3.3416 | LR: 4.88e-04 | 683.2ms/step | 29.3 samples/s +Step 5070/44000 | Loss: 3.3174 | LR: 4.88e-04 | 685.3ms/step | 29.2 samples/s +Step 5080/44000 | Loss: 3.3351 | LR: 4.88e-04 | 684.8ms/step | 29.2 samples/s +Step 5090/44000 | Loss: 3.3322 | LR: 4.88e-04 | 683.2ms/step | 29.3 samples/s +Step 5100/44000 | Loss: 3.3526 | LR: 4.88e-04 | 682.2ms/step | 29.3 samples/s +Step 5110/44000 | Loss: 3.3092 | LR: 4.88e-04 | 684.6ms/step | 29.2 samples/s +Step 5120/44000 | Loss: 3.3815 | LR: 4.88e-04 | 683.5ms/step | 29.3 samples/s +Step 5130/44000 | Loss: 3.3388 | LR: 4.87e-04 | 683.6ms/step | 29.3 samples/s +Step 5140/44000 | Loss: 3.3184 | LR: 4.87e-04 | 683.6ms/step | 29.3 samples/s +Step 5150/44000 | Loss: 3.3296 | LR: 4.87e-04 | 683.3ms/step | 29.3 samples/s +Step 5160/44000 | Loss: 3.3362 | LR: 4.87e-04 | 683.3ms/step | 29.3 samples/s +Step 5170/44000 | Loss: 3.2838 | LR: 4.87e-04 | 684.1ms/step | 29.2 samples/s +Step 5180/44000 | Loss: 3.3286 | LR: 4.87e-04 | 682.8ms/step | 29.3 samples/s +Step 5190/44000 | Loss: 3.3495 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s +Step 5200/44000 | Loss: 3.3265 | LR: 4.87e-04 | 682.3ms/step | 29.3 samples/s +Step 5210/44000 | Loss: 3.3450 | LR: 4.87e-04 | 682.5ms/step | 29.3 samples/s +Step 5220/44000 | Loss: 3.3372 | LR: 4.87e-04 | 684.2ms/step | 29.2 samples/s +Step 5230/44000 | Loss: 3.3439 | LR: 4.87e-04 | 683.1ms/step | 29.3 samples/s +Step 5240/44000 | Loss: 3.3951 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s +Step 5250/44000 | Loss: 3.3348 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s +Validation at step 5250: Loss = 3.3283, Perplexity = 27.89 +Step 5260/44000 | Loss: 3.3578 | LR: 4.87e-04 | 1248.7ms/step | 16.0 samples/s +Step 5270/44000 | Loss: 3.3456 | LR: 4.87e-04 | 687.0ms/step | 29.1 samples/s +Step 5280/44000 | Loss: 3.3424 | LR: 4.87e-04 | 685.5ms/step | 29.2 samples/s +Step 5290/44000 | Loss: 3.3156 | LR: 4.87e-04 | 683.2ms/step | 29.3 samples/s +Step 5300/44000 | Loss: 3.3169 | LR: 4.86e-04 | 684.7ms/step | 29.2 samples/s +Step 5310/44000 | Loss: 3.3450 | LR: 4.86e-04 | 683.3ms/step | 29.3 samples/s +Step 5320/44000 | Loss: 3.2940 | LR: 4.86e-04 | 683.7ms/step | 29.3 samples/s +Step 5330/44000 | Loss: 3.3424 | LR: 4.86e-04 | 684.4ms/step | 29.2 samples/s +Step 5340/44000 | Loss: 3.3076 | LR: 4.86e-04 | 683.0ms/step | 29.3 samples/s +Step 5350/44000 | Loss: 3.2986 | LR: 4.86e-04 | 684.6ms/step | 29.2 samples/s +Step 5360/44000 | Loss: 3.3042 | LR: 4.86e-04 | 682.7ms/step | 29.3 samples/s +Step 5370/44000 | Loss: 3.3298 | LR: 4.86e-04 | 684.5ms/step | 29.2 samples/s +Step 5380/44000 | Loss: 3.3142 | LR: 4.86e-04 | 684.7ms/step | 29.2 samples/s +Step 5390/44000 | Loss: 3.2867 | LR: 4.86e-04 | 682.6ms/step | 29.3 samples/s +Step 5400/44000 | Loss: 3.3236 | LR: 4.86e-04 | 682.8ms/step | 29.3 samples/s +Step 5410/44000 | Loss: 3.2990 | LR: 4.86e-04 | 683.0ms/step | 29.3 samples/s +Step 5420/44000 | Loss: 3.3215 | LR: 4.86e-04 | 680.0ms/step | 29.4 samples/s +Step 5430/44000 | Loss: 3.3168 | LR: 4.86e-04 | 680.1ms/step | 29.4 samples/s +Step 5440/44000 | Loss: 3.3473 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s +Step 5450/44000 | Loss: 3.2748 | LR: 4.86e-04 | 680.3ms/step | 29.4 samples/s +Step 5460/44000 | Loss: 3.2848 | LR: 4.86e-04 | 679.7ms/step | 29.4 samples/s +Step 5470/44000 | Loss: 3.2872 | LR: 4.85e-04 | 679.0ms/step | 29.5 samples/s +Step 5480/44000 | Loss: 3.3316 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s +Step 5490/44000 | Loss: 3.3487 | LR: 4.85e-04 | 679.2ms/step | 29.4 samples/s +Step 5500/44000 | Loss: 3.3209 | LR: 4.85e-04 | 679.5ms/step | 29.4 samples/s +Validation at step 5500: Loss = 3.3158, Perplexity = 27.55 +Step 5510/44000 | Loss: 3.2648 | LR: 4.85e-04 | 1242.4ms/step | 16.1 samples/s +Step 5520/44000 | Loss: 3.2905 | LR: 4.85e-04 | 683.8ms/step | 29.2 samples/s +Step 5530/44000 | Loss: 3.2811 | LR: 4.85e-04 | 682.1ms/step | 29.3 samples/s +Step 5540/44000 | Loss: 3.2864 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s +Step 5550/44000 | Loss: 3.3184 | LR: 4.85e-04 | 682.8ms/step | 29.3 samples/s +Step 5560/44000 | Loss: 3.3184 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s +Step 5570/44000 | Loss: 3.2975 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s +Step 5580/44000 | Loss: 3.3139 | LR: 4.85e-04 | 682.6ms/step | 29.3 samples/s +Step 5590/44000 | Loss: 3.3363 | LR: 4.85e-04 | 681.7ms/step | 29.3 samples/s +Step 5600/44000 | Loss: 3.3200 | LR: 4.85e-04 | 682.2ms/step | 29.3 samples/s +Step 5610/44000 | Loss: 3.2936 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s +Step 5620/44000 | Loss: 3.2959 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s +Step 5630/44000 | Loss: 3.2888 | LR: 4.84e-04 | 679.6ms/step | 29.4 samples/s +Step 5640/44000 | Loss: 3.3318 | LR: 4.84e-04 | 682.2ms/step | 29.3 samples/s +Step 5650/44000 | Loss: 3.2412 | LR: 4.84e-04 | 682.8ms/step | 29.3 samples/s +Step 5660/44000 | Loss: 3.2890 | LR: 4.84e-04 | 682.7ms/step | 29.3 samples/s +Step 5670/44000 | Loss: 3.3116 | LR: 4.84e-04 | 682.6ms/step | 29.3 samples/s +Step 5680/44000 | Loss: 3.2914 | LR: 4.84e-04 | 683.7ms/step | 29.3 samples/s +Step 5690/44000 | Loss: 3.2913 | LR: 4.84e-04 | 682.6ms/step | 29.3 samples/s +Step 5700/44000 | Loss: 3.2328 | LR: 4.84e-04 | 681.8ms/step | 29.3 samples/s +Step 5710/44000 | Loss: 3.2992 | LR: 4.84e-04 | 683.0ms/step | 29.3 samples/s +Step 5720/44000 | Loss: 3.3119 | LR: 4.84e-04 | 681.8ms/step | 29.3 samples/s +Step 5730/44000 | Loss: 3.2860 | LR: 4.84e-04 | 682.5ms/step | 29.3 samples/s +Step 5740/44000 | Loss: 3.2499 | LR: 4.84e-04 | 682.7ms/step | 29.3 samples/s +Step 5750/44000 | Loss: 3.2661 | LR: 4.84e-04 | 681.6ms/step | 29.3 samples/s +Validation at step 5750: Loss = 3.2998, Perplexity = 27.11 +Step 5760/44000 | Loss: 3.3369 | LR: 4.84e-04 | 1237.9ms/step | 16.2 samples/s +Step 5770/44000 | Loss: 3.3070 | LR: 4.84e-04 | 680.1ms/step | 29.4 samples/s +Step 5780/44000 | Loss: 3.2880 | LR: 4.83e-04 | 679.3ms/step | 29.4 samples/s +Step 5790/44000 | Loss: 3.2553 | LR: 4.83e-04 | 680.5ms/step | 29.4 samples/s +Step 5800/44000 | Loss: 3.3024 | LR: 4.83e-04 | 680.0ms/step | 29.4 samples/s +Step 5810/44000 | Loss: 3.3459 | LR: 4.83e-04 | 680.1ms/step | 29.4 samples/s +Step 5820/44000 | Loss: 3.3006 | LR: 4.83e-04 | 680.9ms/step | 29.4 samples/s +Step 5830/44000 | Loss: 3.3122 | LR: 4.83e-04 | 680.9ms/step | 29.4 samples/s +Step 5840/44000 | Loss: 3.2838 | LR: 4.83e-04 | 680.8ms/step | 29.4 samples/s +Step 5850/44000 | Loss: 3.2904 | LR: 4.83e-04 | 681.9ms/step | 29.3 samples/s +Step 5860/44000 | Loss: 3.2707 | LR: 4.83e-04 | 679.1ms/step | 29.5 samples/s +Step 5870/44000 | Loss: 3.2643 | LR: 4.83e-04 | 680.5ms/step | 29.4 samples/s +Step 5880/44000 | Loss: 3.2992 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s +Step 5890/44000 | Loss: 3.3205 | LR: 4.83e-04 | 681.4ms/step | 29.4 samples/s +Step 5900/44000 | Loss: 3.3075 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s +Step 5910/44000 | Loss: 3.3168 | LR: 4.83e-04 | 680.2ms/step | 29.4 samples/s +Step 5920/44000 | Loss: 3.2993 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s +Step 5930/44000 | Loss: 3.2549 | LR: 4.83e-04 | 680.4ms/step | 29.4 samples/s +Step 5940/44000 | Loss: 3.2812 | LR: 4.82e-04 | 679.9ms/step | 29.4 samples/s +Step 5950/44000 | Loss: 3.2856 | LR: 4.82e-04 | 680.8ms/step | 29.4 samples/s +Step 5960/44000 | Loss: 3.2767 | LR: 4.82e-04 | 678.6ms/step | 29.5 samples/s +Step 5970/44000 | Loss: 3.3060 | LR: 4.82e-04 | 680.9ms/step | 29.4 samples/s +Step 5980/44000 | Loss: 3.2944 | LR: 4.82e-04 | 681.5ms/step | 29.3 samples/s +Step 5990/44000 | Loss: 3.2877 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s +Step 6000/44000 | Loss: 3.2886 | LR: 4.82e-04 | 681.8ms/step | 29.3 samples/s +Validation at step 6000: Loss = 3.2881, Perplexity = 26.79 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-6000 +Step 6010/44000 | Loss: 3.3164 | LR: 4.82e-04 | 1529.9ms/step | 13.1 samples/s +Step 6020/44000 | Loss: 3.3169 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s +Step 6030/44000 | Loss: 3.2748 | LR: 4.82e-04 | 681.6ms/step | 29.3 samples/s +Step 6040/44000 | Loss: 3.2842 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s +Step 6050/44000 | Loss: 3.2479 | LR: 4.82e-04 | 681.6ms/step | 29.3 samples/s +Step 6060/44000 | Loss: 3.2579 | LR: 4.82e-04 | 682.3ms/step | 29.3 samples/s +Step 6070/44000 | Loss: 3.2966 | LR: 4.82e-04 | 682.2ms/step | 29.3 samples/s +Step 6080/44000 | Loss: 3.2917 | LR: 4.81e-04 | 681.9ms/step | 29.3 samples/s +Step 6090/44000 | Loss: 3.3131 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s +Step 6100/44000 | Loss: 3.2950 | LR: 4.81e-04 | 683.6ms/step | 29.3 samples/s +Step 6110/44000 | Loss: 3.2380 | LR: 4.81e-04 | 681.4ms/step | 29.4 samples/s +Step 6120/44000 | Loss: 3.2746 | LR: 4.81e-04 | 682.8ms/step | 29.3 samples/s +Step 6130/44000 | Loss: 3.2626 | LR: 4.81e-04 | 682.2ms/step | 29.3 samples/s +Step 6140/44000 | Loss: 3.2603 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s +Step 6150/44000 | Loss: 3.2538 | LR: 4.81e-04 | 683.3ms/step | 29.3 samples/s +Step 6160/44000 | Loss: 3.2764 | LR: 4.81e-04 | 683.2ms/step | 29.3 samples/s +Step 6170/44000 | Loss: 3.2947 | LR: 4.81e-04 | 681.6ms/step | 29.3 samples/s +Step 6180/44000 | Loss: 3.3079 | LR: 4.81e-04 | 680.8ms/step | 29.4 samples/s +Step 6190/44000 | Loss: 3.2699 | LR: 4.81e-04 | 682.7ms/step | 29.3 samples/s +Step 6200/44000 | Loss: 3.2474 | LR: 4.81e-04 | 681.2ms/step | 29.4 samples/s +Step 6210/44000 | Loss: 3.2506 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s +Step 6220/44000 | Loss: 3.2324 | LR: 4.81e-04 | 681.5ms/step | 29.3 samples/s +Step 6230/44000 | Loss: 3.2779 | LR: 4.80e-04 | 681.7ms/step | 29.3 samples/s +Step 6240/44000 | Loss: 3.2363 | LR: 4.80e-04 | 680.4ms/step | 29.4 samples/s +Step 6250/44000 | Loss: 3.2897 | LR: 4.80e-04 | 682.7ms/step | 29.3 samples/s +Validation at step 6250: Loss = 3.2784, Perplexity = 26.53 +Step 6260/44000 | Loss: 3.2699 | LR: 4.80e-04 | 1239.7ms/step | 16.1 samples/s +Step 6270/44000 | Loss: 3.2876 | LR: 4.80e-04 | 681.1ms/step | 29.4 samples/s +Step 6280/44000 | Loss: 3.2835 | LR: 4.80e-04 | 681.9ms/step | 29.3 samples/s +Step 6290/44000 | Loss: 3.2915 | LR: 4.80e-04 | 682.5ms/step | 29.3 samples/s +Step 6300/44000 | Loss: 3.2651 | LR: 4.80e-04 | 681.2ms/step | 29.4 samples/s +Step 6310/44000 | Loss: 3.3021 | LR: 4.80e-04 | 682.1ms/step | 29.3 samples/s +Step 6320/44000 | Loss: 3.2442 | LR: 4.80e-04 | 682.3ms/step | 29.3 samples/s +Step 6330/44000 | Loss: 3.3132 | LR: 4.80e-04 | 682.4ms/step | 29.3 samples/s +Step 6340/44000 | Loss: 3.2685 | LR: 4.80e-04 | 681.7ms/step | 29.3 samples/s +Step 6350/44000 | Loss: 3.3161 | LR: 4.80e-04 | 682.1ms/step | 29.3 samples/s +Step 6360/44000 | Loss: 3.2853 | LR: 4.80e-04 | 681.2ms/step | 29.4 samples/s +Step 6370/44000 | Loss: 3.2403 | LR: 4.79e-04 | 681.5ms/step | 29.3 samples/s +Step 6380/44000 | Loss: 3.2693 | LR: 4.79e-04 | 683.7ms/step | 29.3 samples/s +Step 6390/44000 | Loss: 3.2859 | LR: 4.79e-04 | 682.0ms/step | 29.3 samples/s +Step 6400/44000 | Loss: 3.2454 | LR: 4.79e-04 | 681.8ms/step | 29.3 samples/s +Step 6410/44000 | Loss: 3.2697 | LR: 4.79e-04 | 683.0ms/step | 29.3 samples/s +Step 6420/44000 | Loss: 3.2635 | LR: 4.79e-04 | 681.2ms/step | 29.4 samples/s +Step 6430/44000 | Loss: 3.2927 | LR: 4.79e-04 | 682.3ms/step | 29.3 samples/s +Step 6440/44000 | Loss: 3.2558 | LR: 4.79e-04 | 681.4ms/step | 29.4 samples/s +Step 6450/44000 | Loss: 3.2614 | LR: 4.79e-04 | 682.0ms/step | 29.3 samples/s +Step 6460/44000 | Loss: 3.2724 | LR: 4.79e-04 | 682.3ms/step | 29.3 samples/s +Step 6470/44000 | Loss: 3.2957 | LR: 4.79e-04 | 682.6ms/step | 29.3 samples/s +Step 6480/44000 | Loss: 3.3140 | LR: 4.79e-04 | 682.1ms/step | 29.3 samples/s +Step 6490/44000 | Loss: 3.2279 | LR: 4.79e-04 | 680.6ms/step | 29.4 samples/s +Step 6500/44000 | Loss: 3.3076 | LR: 4.79e-04 | 682.7ms/step | 29.3 samples/s +Validation at step 6500: Loss = 3.2658, Perplexity = 26.20 +Step 6510/44000 | Loss: 3.2742 | LR: 4.78e-04 | 1242.0ms/step | 16.1 samples/s +Step 6520/44000 | Loss: 3.2622 | LR: 4.78e-04 | 681.7ms/step | 29.3 samples/s +Step 6530/44000 | Loss: 3.2865 | LR: 4.78e-04 | 681.1ms/step | 29.4 samples/s +Step 6540/44000 | Loss: 3.2561 | LR: 4.78e-04 | 681.4ms/step | 29.4 samples/s +Step 6550/44000 | Loss: 3.2848 | LR: 4.78e-04 | 682.8ms/step | 29.3 samples/s +Step 6560/44000 | Loss: 3.2322 | LR: 4.78e-04 | 682.1ms/step | 29.3 samples/s +Step 6570/44000 | Loss: 3.2234 | LR: 4.78e-04 | 683.3ms/step | 29.3 samples/s +Step 6580/44000 | Loss: 3.2458 | LR: 4.78e-04 | 682.2ms/step | 29.3 samples/s +Step 6590/44000 | Loss: 3.2562 | LR: 4.78e-04 | 682.4ms/step | 29.3 samples/s +Step 6600/44000 | Loss: 3.2259 | LR: 4.78e-04 | 682.5ms/step | 29.3 samples/s +Step 6610/44000 | Loss: 3.2552 | LR: 4.78e-04 | 682.0ms/step | 29.3 samples/s +Step 6620/44000 | Loss: 3.2465 | LR: 4.78e-04 | 682.3ms/step | 29.3 samples/s +Step 6630/44000 | Loss: 3.2883 | LR: 4.78e-04 | 682.3ms/step | 29.3 samples/s +Step 6640/44000 | Loss: 3.2659 | LR: 4.77e-04 | 683.8ms/step | 29.2 samples/s +Step 6650/44000 | Loss: 3.2864 | LR: 4.77e-04 | 681.4ms/step | 29.4 samples/s +Step 6660/44000 | Loss: 3.2136 | LR: 4.77e-04 | 681.9ms/step | 29.3 samples/s +Step 6670/44000 | Loss: 3.2661 | LR: 4.77e-04 | 682.2ms/step | 29.3 samples/s +Step 6680/44000 | Loss: 3.2368 | LR: 4.77e-04 | 681.5ms/step | 29.3 samples/s +Step 6690/44000 | Loss: 3.2720 | LR: 4.77e-04 | 682.4ms/step | 29.3 samples/s +Step 6700/44000 | Loss: 3.2586 | LR: 4.77e-04 | 680.6ms/step | 29.4 samples/s +Step 6710/44000 | Loss: 3.2374 | LR: 4.77e-04 | 680.0ms/step | 29.4 samples/s +Step 6720/44000 | Loss: 3.2537 | LR: 4.77e-04 | 682.3ms/step | 29.3 samples/s +Step 6730/44000 | Loss: 3.3197 | LR: 4.77e-04 | 681.2ms/step | 29.4 samples/s +Step 6740/44000 | Loss: 3.2437 | LR: 4.77e-04 | 681.2ms/step | 29.4 samples/s +Step 6750/44000 | Loss: 3.2199 | LR: 4.77e-04 | 682.5ms/step | 29.3 samples/s +Validation at step 6750: Loss = 3.2597, Perplexity = 26.04 +Step 6760/44000 | Loss: 3.2379 | LR: 4.77e-04 | 1240.3ms/step | 16.1 samples/s +Step 6770/44000 | Loss: 3.2695 | LR: 4.76e-04 | 682.8ms/step | 29.3 samples/s +Step 6780/44000 | Loss: 3.2444 | LR: 4.76e-04 | 681.2ms/step | 29.4 samples/s +Step 6790/44000 | Loss: 3.2643 | LR: 4.76e-04 | 682.6ms/step | 29.3 samples/s +Step 6800/44000 | Loss: 3.2226 | LR: 4.76e-04 | 682.1ms/step | 29.3 samples/s +Step 6810/44000 | Loss: 3.2656 | LR: 4.76e-04 | 681.0ms/step | 29.4 samples/s +Step 6820/44000 | Loss: 3.2647 | LR: 4.76e-04 | 681.9ms/step | 29.3 samples/s +Step 6830/44000 | Loss: 3.2415 | LR: 4.76e-04 | 680.8ms/step | 29.4 samples/s +Step 6840/44000 | Loss: 3.2216 | LR: 4.76e-04 | 681.0ms/step | 29.4 samples/s +Step 6850/44000 | Loss: 3.2313 | LR: 4.76e-04 | 681.1ms/step | 29.4 samples/s +Step 6860/44000 | Loss: 3.2074 | LR: 4.76e-04 | 681.7ms/step | 29.3 samples/s +Step 6870/44000 | Loss: 3.2803 | LR: 4.76e-04 | 680.0ms/step | 29.4 samples/s +Step 6880/44000 | Loss: 3.2353 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s +Step 6890/44000 | Loss: 3.2318 | LR: 4.76e-04 | 681.3ms/step | 29.4 samples/s +Step 6900/44000 | Loss: 3.2416 | LR: 4.75e-04 | 681.1ms/step | 29.4 samples/s +Step 6910/44000 | Loss: 3.2817 | LR: 4.75e-04 | 680.9ms/step | 29.4 samples/s +Step 6920/44000 | Loss: 3.2866 | LR: 4.75e-04 | 681.9ms/step | 29.3 samples/s +Step 6930/44000 | Loss: 3.2466 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s +Step 6940/44000 | Loss: 3.2453 | LR: 4.75e-04 | 682.6ms/step | 29.3 samples/s +Step 6950/44000 | Loss: 3.2409 | LR: 4.75e-04 | 682.5ms/step | 29.3 samples/s +Step 6960/44000 | Loss: 3.2616 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s +Step 6970/44000 | Loss: 3.2052 | LR: 4.75e-04 | 682.7ms/step | 29.3 samples/s +Step 6980/44000 | Loss: 3.2595 | LR: 4.75e-04 | 681.7ms/step | 29.3 samples/s +Step 6990/44000 | Loss: 3.2086 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s +Step 7000/44000 | Loss: 3.2390 | LR: 4.75e-04 | 681.5ms/step | 29.3 samples/s +Validation at step 7000: Loss = 3.2479, Perplexity = 25.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-7000 +Step 7010/44000 | Loss: 3.2416 | LR: 4.75e-04 | 1513.6ms/step | 13.2 samples/s +Step 7020/44000 | Loss: 3.2459 | LR: 4.75e-04 | 681.5ms/step | 29.3 samples/s +Step 7030/44000 | Loss: 3.2243 | LR: 4.74e-04 | 679.4ms/step | 29.4 samples/s +Step 7040/44000 | Loss: 3.2355 | LR: 4.74e-04 | 680.1ms/step | 29.4 samples/s +Step 7050/44000 | Loss: 3.2453 | LR: 4.74e-04 | 681.1ms/step | 29.4 samples/s +Step 7060/44000 | Loss: 3.2557 | LR: 4.74e-04 | 680.9ms/step | 29.4 samples/s +Step 7070/44000 | Loss: 3.1812 | LR: 4.74e-04 | 679.9ms/step | 29.4 samples/s +Step 7080/44000 | Loss: 3.2296 | LR: 4.74e-04 | 678.6ms/step | 29.5 samples/s +Step 7090/44000 | Loss: 3.2749 | LR: 4.74e-04 | 678.8ms/step | 29.5 samples/s +Step 7100/44000 | Loss: 3.2316 | LR: 4.74e-04 | 678.9ms/step | 29.5 samples/s +Step 7110/44000 | Loss: 3.2163 | LR: 4.74e-04 | 677.4ms/step | 29.5 samples/s +Step 7120/44000 | Loss: 3.2116 | LR: 4.74e-04 | 676.9ms/step | 29.5 samples/s +Step 7130/44000 | Loss: 3.1702 | LR: 4.74e-04 | 677.5ms/step | 29.5 samples/s +Step 7140/44000 | Loss: 3.2219 | LR: 4.74e-04 | 677.5ms/step | 29.5 samples/s +Step 7150/44000 | Loss: 3.2445 | LR: 4.73e-04 | 678.5ms/step | 29.5 samples/s +Step 7160/44000 | Loss: 3.2340 | LR: 4.73e-04 | 677.7ms/step | 29.5 samples/s +Step 7170/44000 | Loss: 3.2090 | LR: 4.73e-04 | 677.9ms/step | 29.5 samples/s +Step 7180/44000 | Loss: 3.2265 | LR: 4.73e-04 | 680.2ms/step | 29.4 samples/s +Step 7190/44000 | Loss: 3.2487 | LR: 4.73e-04 | 679.9ms/step | 29.4 samples/s +Step 7200/44000 | Loss: 3.2554 | LR: 4.73e-04 | 679.0ms/step | 29.5 samples/s +Step 7210/44000 | Loss: 3.2330 | LR: 4.73e-04 | 679.6ms/step | 29.4 samples/s +Step 7220/44000 | Loss: 3.2147 | LR: 4.73e-04 | 678.6ms/step | 29.5 samples/s +Step 7230/44000 | Loss: 3.2292 | LR: 4.73e-04 | 679.8ms/step | 29.4 samples/s +Step 7240/44000 | Loss: 3.2154 | LR: 4.73e-04 | 679.5ms/step | 29.4 samples/s +Step 7250/44000 | Loss: 3.2517 | LR: 4.73e-04 | 678.5ms/step | 29.5 samples/s +Validation at step 7250: Loss = 3.2378, Perplexity = 25.48 +Step 7260/44000 | Loss: 3.2445 | LR: 4.73e-04 | 1235.9ms/step | 16.2 samples/s +Step 7270/44000 | Loss: 3.2137 | LR: 4.72e-04 | 679.9ms/step | 29.4 samples/s +Step 7280/44000 | Loss: 3.2321 | LR: 4.72e-04 | 679.2ms/step | 29.4 samples/s +Step 7290/44000 | Loss: 3.2509 | LR: 4.72e-04 | 678.9ms/step | 29.5 samples/s +Step 7300/44000 | Loss: 3.2473 | LR: 4.72e-04 | 677.3ms/step | 29.5 samples/s +Step 7310/44000 | Loss: 3.2488 | LR: 4.72e-04 | 678.8ms/step | 29.5 samples/s +Step 7320/44000 | Loss: 3.2084 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s +Step 7330/44000 | Loss: 3.2435 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s +Step 7340/44000 | Loss: 3.2316 | LR: 4.72e-04 | 678.2ms/step | 29.5 samples/s +Step 7350/44000 | Loss: 3.2156 | LR: 4.72e-04 | 678.1ms/step | 29.5 samples/s +Step 7360/44000 | Loss: 3.2001 | LR: 4.72e-04 | 678.6ms/step | 29.5 samples/s +Step 7370/44000 | Loss: 3.2559 | LR: 4.72e-04 | 678.2ms/step | 29.5 samples/s +Step 7380/44000 | Loss: 3.2513 | LR: 4.72e-04 | 678.6ms/step | 29.5 samples/s +Step 7390/44000 | Loss: 3.2778 | LR: 4.71e-04 | 678.8ms/step | 29.5 samples/s +Step 7400/44000 | Loss: 3.2280 | LR: 4.71e-04 | 677.9ms/step | 29.5 samples/s +Step 7410/44000 | Loss: 3.2428 | LR: 4.71e-04 | 678.4ms/step | 29.5 samples/s +Step 7420/44000 | Loss: 3.2287 | LR: 4.71e-04 | 678.7ms/step | 29.5 samples/s +Step 7430/44000 | Loss: 3.2288 | LR: 4.71e-04 | 679.1ms/step | 29.5 samples/s +Step 7440/44000 | Loss: 3.2346 | LR: 4.71e-04 | 677.6ms/step | 29.5 samples/s +Step 7450/44000 | Loss: 3.2619 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s +Step 7460/44000 | Loss: 3.2427 | LR: 4.71e-04 | 679.7ms/step | 29.4 samples/s +Step 7470/44000 | Loss: 3.2438 | LR: 4.71e-04 | 678.4ms/step | 29.5 samples/s +Step 7480/44000 | Loss: 3.2501 | LR: 4.71e-04 | 677.5ms/step | 29.5 samples/s +Step 7490/44000 | Loss: 3.2011 | LR: 4.71e-04 | 678.5ms/step | 29.5 samples/s +Step 7500/44000 | Loss: 3.1936 | LR: 4.71e-04 | 678.5ms/step | 29.5 samples/s +Validation at step 7500: Loss = 3.2298, Perplexity = 25.27 +Step 7510/44000 | Loss: 3.2095 | LR: 4.70e-04 | 1239.6ms/step | 16.1 samples/s +Step 7520/44000 | Loss: 3.2317 | LR: 4.70e-04 | 682.3ms/step | 29.3 samples/s +Step 7530/44000 | Loss: 3.2518 | LR: 4.70e-04 | 682.0ms/step | 29.3 samples/s +Step 7540/44000 | Loss: 3.2086 | LR: 4.70e-04 | 683.8ms/step | 29.3 samples/s +Step 7550/44000 | Loss: 3.2021 | LR: 4.70e-04 | 682.6ms/step | 29.3 samples/s +Step 7560/44000 | Loss: 3.2269 | LR: 4.70e-04 | 680.4ms/step | 29.4 samples/s +Step 7570/44000 | Loss: 3.1721 | LR: 4.70e-04 | 679.8ms/step | 29.4 samples/s +Step 7580/44000 | Loss: 3.1467 | LR: 4.70e-04 | 682.4ms/step | 29.3 samples/s +Step 7590/44000 | Loss: 3.2108 | LR: 4.70e-04 | 681.0ms/step | 29.4 samples/s +Step 7600/44000 | Loss: 3.2213 | LR: 4.70e-04 | 681.0ms/step | 29.4 samples/s +Step 7610/44000 | Loss: 3.2189 | LR: 4.70e-04 | 682.2ms/step | 29.3 samples/s +Step 7620/44000 | Loss: 3.2377 | LR: 4.70e-04 | 682.4ms/step | 29.3 samples/s +Step 7630/44000 | Loss: 3.2080 | LR: 4.69e-04 | 680.9ms/step | 29.4 samples/s +Step 7640/44000 | Loss: 3.1766 | LR: 4.69e-04 | 679.9ms/step | 29.4 samples/s +Step 7650/44000 | Loss: 3.1990 | LR: 4.69e-04 | 681.9ms/step | 29.3 samples/s +Step 7660/44000 | Loss: 3.2231 | LR: 4.69e-04 | 682.3ms/step | 29.3 samples/s +Step 7670/44000 | Loss: 3.1943 | LR: 4.69e-04 | 683.1ms/step | 29.3 samples/s +Step 7680/44000 | Loss: 3.2572 | LR: 4.69e-04 | 682.7ms/step | 29.3 samples/s +Step 7690/44000 | Loss: 3.2244 | LR: 4.69e-04 | 682.1ms/step | 29.3 samples/s +Step 7700/44000 | Loss: 3.1975 | LR: 4.69e-04 | 681.8ms/step | 29.3 samples/s +Step 7710/44000 | Loss: 3.1890 | LR: 4.69e-04 | 681.1ms/step | 29.4 samples/s +Step 7720/44000 | Loss: 3.2167 | LR: 4.69e-04 | 683.3ms/step | 29.3 samples/s +Step 7730/44000 | Loss: 3.2047 | LR: 4.69e-04 | 681.9ms/step | 29.3 samples/s +Step 7740/44000 | Loss: 3.1646 | LR: 4.68e-04 | 682.6ms/step | 29.3 samples/s +Step 7750/44000 | Loss: 3.2065 | LR: 4.68e-04 | 684.8ms/step | 29.2 samples/s +Validation at step 7750: Loss = 3.2211, Perplexity = 25.06 +Step 7760/44000 | Loss: 3.2048 | LR: 4.68e-04 | 1241.5ms/step | 16.1 samples/s +Step 7770/44000 | Loss: 3.2135 | LR: 4.68e-04 | 682.3ms/step | 29.3 samples/s +Step 7780/44000 | Loss: 3.2123 | LR: 4.68e-04 | 682.3ms/step | 29.3 samples/s +Step 7790/44000 | Loss: 3.2149 | LR: 4.68e-04 | 681.1ms/step | 29.4 samples/s +Step 7800/44000 | Loss: 3.2257 | LR: 4.68e-04 | 681.3ms/step | 29.4 samples/s +Step 7810/44000 | Loss: 3.1717 | LR: 4.68e-04 | 680.5ms/step | 29.4 samples/s +Step 7820/44000 | Loss: 3.2175 | LR: 4.68e-04 | 681.5ms/step | 29.3 samples/s +Step 7830/44000 | Loss: 3.2034 | LR: 4.68e-04 | 681.7ms/step | 29.3 samples/s +Step 7840/44000 | Loss: 3.1849 | LR: 4.68e-04 | 682.0ms/step | 29.3 samples/s +Step 7850/44000 | Loss: 3.1836 | LR: 4.67e-04 | 681.3ms/step | 29.4 samples/s +Step 7860/44000 | Loss: 3.2015 | LR: 4.67e-04 | 682.0ms/step | 29.3 samples/s +Step 7870/44000 | Loss: 3.2070 | LR: 4.67e-04 | 683.2ms/step | 29.3 samples/s +Step 7880/44000 | Loss: 3.2347 | LR: 4.67e-04 | 681.5ms/step | 29.3 samples/s +Step 7890/44000 | Loss: 3.2008 | LR: 4.67e-04 | 682.0ms/step | 29.3 samples/s +Step 7900/44000 | Loss: 3.2439 | LR: 4.67e-04 | 681.8ms/step | 29.3 samples/s +Step 7910/44000 | Loss: 3.2194 | LR: 4.67e-04 | 682.7ms/step | 29.3 samples/s +Step 7920/44000 | Loss: 3.2346 | LR: 4.67e-04 | 683.1ms/step | 29.3 samples/s +Step 7930/44000 | Loss: 3.2598 | LR: 4.67e-04 | 682.1ms/step | 29.3 samples/s +Step 7940/44000 | Loss: 3.2585 | LR: 4.67e-04 | 682.5ms/step | 29.3 samples/s +Step 7950/44000 | Loss: 3.1714 | LR: 4.67e-04 | 682.1ms/step | 29.3 samples/s +Step 7960/44000 | Loss: 3.2250 | LR: 4.66e-04 | 682.0ms/step | 29.3 samples/s +Step 7970/44000 | Loss: 3.1593 | LR: 4.66e-04 | 682.2ms/step | 29.3 samples/s +Step 7980/44000 | Loss: 3.1908 | LR: 4.66e-04 | 682.0ms/step | 29.3 samples/s +Step 7990/44000 | Loss: 3.2133 | LR: 4.66e-04 | 682.5ms/step | 29.3 samples/s +Step 8000/44000 | Loss: 3.2482 | LR: 4.66e-04 | 682.3ms/step | 29.3 samples/s +Validation at step 8000: Loss = 3.2132, Perplexity = 24.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-8000 +Step 8010/44000 | Loss: 3.2003 | LR: 4.66e-04 | 1522.8ms/step | 13.1 samples/s +Step 8020/44000 | Loss: 3.2183 | LR: 4.66e-04 | 682.3ms/step | 29.3 samples/s +Step 8030/44000 | Loss: 3.2136 | LR: 4.66e-04 | 682.5ms/step | 29.3 samples/s +Step 8040/44000 | Loss: 3.2349 | LR: 4.66e-04 | 680.9ms/step | 29.4 samples/s +Step 8050/44000 | Loss: 3.2064 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s +Step 8060/44000 | Loss: 3.2226 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s +Step 8070/44000 | Loss: 3.1707 | LR: 4.65e-04 | 681.7ms/step | 29.3 samples/s +Step 8080/44000 | Loss: 3.1858 | LR: 4.65e-04 | 682.5ms/step | 29.3 samples/s +Step 8090/44000 | Loss: 3.2295 | LR: 4.65e-04 | 681.9ms/step | 29.3 samples/s +Step 8100/44000 | Loss: 3.2361 | LR: 4.65e-04 | 680.9ms/step | 29.4 samples/s +Step 8110/44000 | Loss: 3.2172 | LR: 4.65e-04 | 683.2ms/step | 29.3 samples/s +Step 8120/44000 | Loss: 3.1681 | LR: 4.65e-04 | 681.7ms/step | 29.3 samples/s +W0708 22:28:09.644000 254 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 254 got signal: 15 diff --git a/epfl_final/logs/l8k365m_pool100.log b/epfl_final/logs/l8k365m_pool100.log new file mode 100644 index 0000000000000000000000000000000000000000..d39a30a29a5c749e108353a4298dc0279bdbb36b --- /dev/null +++ b/epfl_final/logs/l8k365m_pool100.log @@ -0,0 +1,1072 @@ +W0708 20:48:06.039000 255 site-packages/torch/distributed/run.py:851] +W0708 20:48:06.039000 255 site-packages/torch/distributed/run.py:851] ***************************************** +W0708 20:48:06.039000 255 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0708 20:48:06.039000 255 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m pool100 8k nrb31 freqabl", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "resampled", + "freqablation", + "pool100" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 100, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32", + "sparse_mask_seed": 0, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 100 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) + +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 82re6m3i +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_205109-82re6m3i +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m pool100 8k nrb31 freqabl +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/82re6m3i +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0398 | LR: 6.67e-06 | 6736.3ms/step | 3.0 samples/s +Step 20/44000 | Loss: 11.9261 | LR: 1.33e-05 | 677.8ms/step | 29.5 samples/s +Step 30/44000 | Loss: 11.5485 | LR: 2.00e-05 | 680.1ms/step | 29.4 samples/s +Step 40/44000 | Loss: 11.0225 | LR: 2.67e-05 | 674.7ms/step | 29.6 samples/s +Step 50/44000 | Loss: 10.7137 | LR: 3.33e-05 | 671.4ms/step | 29.8 samples/s +Step 60/44000 | Loss: 10.4833 | LR: 4.00e-05 | 675.4ms/step | 29.6 samples/s +Step 70/44000 | Loss: 10.2636 | LR: 4.67e-05 | 672.6ms/step | 29.7 samples/s +Step 80/44000 | Loss: 9.9451 | LR: 5.33e-05 | 673.1ms/step | 29.7 samples/s +Step 90/44000 | Loss: 9.5554 | LR: 6.00e-05 | 677.6ms/step | 29.5 samples/s +Step 100/44000 | Loss: 9.1247 | LR: 6.67e-05 | 675.0ms/step | 29.6 samples/s +Step 110/44000 | Loss: 8.5965 | LR: 7.33e-05 | 674.8ms/step | 29.6 samples/s +Step 120/44000 | Loss: 8.1657 | LR: 8.00e-05 | 672.8ms/step | 29.7 samples/s +Step 130/44000 | Loss: 7.7923 | LR: 8.67e-05 | 673.8ms/step | 29.7 samples/s +Step 140/44000 | Loss: 7.5092 | LR: 9.33e-05 | 675.6ms/step | 29.6 samples/s +Step 150/44000 | Loss: 7.3410 | LR: 1.00e-04 | 674.4ms/step | 29.7 samples/s +Step 160/44000 | Loss: 7.1668 | LR: 1.07e-04 | 676.6ms/step | 29.6 samples/s +Step 170/44000 | Loss: 7.1116 | LR: 1.13e-04 | 677.2ms/step | 29.5 samples/s +Step 180/44000 | Loss: 7.0514 | LR: 1.20e-04 | 678.2ms/step | 29.5 samples/s +Step 190/44000 | Loss: 6.9679 | LR: 1.27e-04 | 678.7ms/step | 29.5 samples/s +Step 200/44000 | Loss: 6.8446 | LR: 1.33e-04 | 680.1ms/step | 29.4 samples/s +Step 210/44000 | Loss: 6.7549 | LR: 1.40e-04 | 677.8ms/step | 29.5 samples/s +Step 220/44000 | Loss: 6.6656 | LR: 1.47e-04 | 680.1ms/step | 29.4 samples/s +Step 230/44000 | Loss: 6.5696 | LR: 1.53e-04 | 679.5ms/step | 29.4 samples/s +Step 240/44000 | Loss: 6.5532 | LR: 1.60e-04 | 680.1ms/step | 29.4 samples/s +Step 250/44000 | Loss: 6.4335 | LR: 1.67e-04 | 680.9ms/step | 29.4 samples/s +Validation at step 250: Loss = 6.4401, Perplexity = 626.49 +Step 260/44000 | Loss: 6.4447 | LR: 1.73e-04 | 3834.4ms/step | 5.2 samples/s +Step 270/44000 | Loss: 6.3853 | LR: 1.80e-04 | 680.5ms/step | 29.4 samples/s +Step 280/44000 | Loss: 6.2990 | LR: 1.87e-04 | 681.3ms/step | 29.4 samples/s +Step 290/44000 | Loss: 6.2483 | LR: 1.93e-04 | 679.4ms/step | 29.4 samples/s +Step 300/44000 | Loss: 6.2525 | LR: 2.00e-04 | 680.0ms/step | 29.4 samples/s +Step 310/44000 | Loss: 6.2115 | LR: 2.07e-04 | 680.8ms/step | 29.4 samples/s +Step 320/44000 | Loss: 6.1328 | LR: 2.13e-04 | 680.6ms/step | 29.4 samples/s +Step 330/44000 | Loss: 6.0866 | LR: 2.20e-04 | 680.0ms/step | 29.4 samples/s +Step 340/44000 | Loss: 6.0659 | LR: 2.27e-04 | 683.8ms/step | 29.2 samples/s +Step 350/44000 | Loss: 5.9946 | LR: 2.33e-04 | 683.4ms/step | 29.3 samples/s +Step 360/44000 | Loss: 5.9899 | LR: 2.40e-04 | 680.1ms/step | 29.4 samples/s +Step 370/44000 | Loss: 5.8784 | LR: 2.47e-04 | 679.2ms/step | 29.4 samples/s +Step 380/44000 | Loss: 5.8986 | LR: 2.53e-04 | 679.1ms/step | 29.4 samples/s +Step 390/44000 | Loss: 5.8632 | LR: 2.60e-04 | 681.4ms/step | 29.4 samples/s +Step 400/44000 | Loss: 5.8139 | LR: 2.67e-04 | 681.8ms/step | 29.3 samples/s +Step 410/44000 | Loss: 5.7832 | LR: 2.73e-04 | 681.2ms/step | 29.4 samples/s +Step 420/44000 | Loss: 5.7248 | LR: 2.80e-04 | 680.6ms/step | 29.4 samples/s +Step 430/44000 | Loss: 5.7328 | LR: 2.87e-04 | 681.6ms/step | 29.3 samples/s +Step 440/44000 | Loss: 5.7227 | LR: 2.93e-04 | 681.9ms/step | 29.3 samples/s +Step 450/44000 | Loss: 5.6031 | LR: 3.00e-04 | 682.6ms/step | 29.3 samples/s +Step 460/44000 | Loss: 5.6141 | LR: 3.07e-04 | 681.7ms/step | 29.3 samples/s +Step 470/44000 | Loss: 5.5835 | LR: 3.13e-04 | 680.8ms/step | 29.4 samples/s +Step 480/44000 | Loss: 5.5358 | LR: 3.20e-04 | 680.0ms/step | 29.4 samples/s +Step 490/44000 | Loss: 5.5257 | LR: 3.27e-04 | 680.0ms/step | 29.4 samples/s +Step 500/44000 | Loss: 5.4933 | LR: 3.33e-04 | 680.0ms/step | 29.4 samples/s +Validation at step 500: Loss = 5.4703, Perplexity = 237.54 +Step 510/44000 | Loss: 5.3939 | LR: 3.40e-04 | 1247.2ms/step | 16.0 samples/s +Step 520/44000 | Loss: 5.3948 | LR: 3.47e-04 | 684.7ms/step | 29.2 samples/s +Step 530/44000 | Loss: 5.3818 | LR: 3.53e-04 | 682.7ms/step | 29.3 samples/s +Step 540/44000 | Loss: 5.3525 | LR: 3.60e-04 | 682.7ms/step | 29.3 samples/s +Step 550/44000 | Loss: 5.3688 | LR: 3.67e-04 | 683.7ms/step | 29.3 samples/s +Step 560/44000 | Loss: 5.3101 | LR: 3.73e-04 | 681.4ms/step | 29.3 samples/s +Step 570/44000 | Loss: 5.3016 | LR: 3.80e-04 | 680.8ms/step | 29.4 samples/s +Step 580/44000 | Loss: 5.2444 | LR: 3.87e-04 | 680.8ms/step | 29.4 samples/s +Step 590/44000 | Loss: 5.2354 | LR: 3.93e-04 | 681.7ms/step | 29.3 samples/s +Step 600/44000 | Loss: 5.2160 | LR: 4.00e-04 | 682.2ms/step | 29.3 samples/s +Step 610/44000 | Loss: 5.1322 | LR: 4.07e-04 | 681.8ms/step | 29.3 samples/s +Step 620/44000 | Loss: 5.1132 | LR: 4.13e-04 | 682.4ms/step | 29.3 samples/s +Step 630/44000 | Loss: 5.1564 | LR: 4.20e-04 | 681.6ms/step | 29.3 samples/s +Step 640/44000 | Loss: 5.1229 | LR: 4.27e-04 | 681.7ms/step | 29.3 samples/s +Step 650/44000 | Loss: 5.0708 | LR: 4.33e-04 | 680.7ms/step | 29.4 samples/s +Step 660/44000 | Loss: 5.0615 | LR: 4.40e-04 | 680.9ms/step | 29.4 samples/s +Step 670/44000 | Loss: 5.0260 | LR: 4.47e-04 | 680.7ms/step | 29.4 samples/s +Step 680/44000 | Loss: 5.0432 | LR: 4.53e-04 | 683.5ms/step | 29.3 samples/s +Step 690/44000 | Loss: 5.0021 | LR: 4.60e-04 | 681.8ms/step | 29.3 samples/s +Step 700/44000 | Loss: 4.9098 | LR: 4.67e-04 | 683.3ms/step | 29.3 samples/s +Step 710/44000 | Loss: 4.9521 | LR: 4.73e-04 | 679.8ms/step | 29.4 samples/s +Step 720/44000 | Loss: 4.8679 | LR: 4.80e-04 | 683.1ms/step | 29.3 samples/s +Step 730/44000 | Loss: 4.8684 | LR: 4.87e-04 | 682.5ms/step | 29.3 samples/s +Step 740/44000 | Loss: 4.8072 | LR: 4.93e-04 | 681.6ms/step | 29.3 samples/s +Step 750/44000 | Loss: 4.7878 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s +Validation at step 750: Loss = 4.8024, Perplexity = 121.80 +Step 760/44000 | Loss: 4.7690 | LR: 5.00e-04 | 1242.5ms/step | 16.1 samples/s +Step 770/44000 | Loss: 4.7786 | LR: 5.00e-04 | 682.1ms/step | 29.3 samples/s +Step 780/44000 | Loss: 4.6998 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s +Step 790/44000 | Loss: 4.7071 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s +Step 800/44000 | Loss: 4.7447 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s +Step 810/44000 | Loss: 4.6695 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s +Step 820/44000 | Loss: 4.6305 | LR: 5.00e-04 | 683.6ms/step | 29.3 samples/s +Step 830/44000 | Loss: 4.6075 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s +Step 840/44000 | Loss: 4.5450 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s +Step 850/44000 | Loss: 4.5586 | LR: 5.00e-04 | 681.8ms/step | 29.3 samples/s +Step 860/44000 | Loss: 4.5464 | LR: 5.00e-04 | 682.2ms/step | 29.3 samples/s +Step 870/44000 | Loss: 4.5607 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s +Step 880/44000 | Loss: 4.4897 | LR: 5.00e-04 | 682.5ms/step | 29.3 samples/s +Step 890/44000 | Loss: 4.4793 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s +Step 900/44000 | Loss: 4.4737 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s +Step 910/44000 | Loss: 4.4645 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s +Step 920/44000 | Loss: 4.4159 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s +Step 930/44000 | Loss: 4.4167 | LR: 5.00e-04 | 684.6ms/step | 29.2 samples/s +Step 940/44000 | Loss: 4.3917 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s +Step 950/44000 | Loss: 4.4081 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s +Step 960/44000 | Loss: 4.3439 | LR: 5.00e-04 | 681.0ms/step | 29.4 samples/s +Step 970/44000 | Loss: 4.3519 | LR: 5.00e-04 | 681.6ms/step | 29.3 samples/s +Step 980/44000 | Loss: 4.3508 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s +Step 990/44000 | Loss: 4.3235 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s +Step 1000/44000 | Loss: 4.3257 | LR: 5.00e-04 | 681.5ms/step | 29.3 samples/s +Validation at step 1000: Loss = 4.3220, Perplexity = 75.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-1000 +Step 1010/44000 | Loss: 4.3460 | LR: 5.00e-04 | 1526.2ms/step | 13.1 samples/s +Step 1020/44000 | Loss: 4.2801 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s +Step 1030/44000 | Loss: 4.2858 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s +Step 1040/44000 | Loss: 4.2180 | LR: 5.00e-04 | 684.1ms/step | 29.2 samples/s +Step 1050/44000 | Loss: 4.2685 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s +Step 1060/44000 | Loss: 4.2945 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s +Step 1070/44000 | Loss: 4.2088 | LR: 5.00e-04 | 683.5ms/step | 29.3 samples/s +Step 1080/44000 | Loss: 4.2116 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s +Step 1090/44000 | Loss: 4.2323 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s +Step 1100/44000 | Loss: 4.2100 | LR: 5.00e-04 | 680.2ms/step | 29.4 samples/s +Step 1110/44000 | Loss: 4.2157 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s +Step 1120/44000 | Loss: 4.1160 | LR: 5.00e-04 | 681.8ms/step | 29.3 samples/s +Step 1130/44000 | Loss: 4.1421 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s +Step 1140/44000 | Loss: 4.1314 | LR: 5.00e-04 | 683.8ms/step | 29.2 samples/s +Step 1150/44000 | Loss: 4.1335 | LR: 5.00e-04 | 680.6ms/step | 29.4 samples/s +Step 1160/44000 | Loss: 4.1396 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s +Step 1170/44000 | Loss: 4.1272 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s +Step 1180/44000 | Loss: 4.1398 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s +Step 1190/44000 | Loss: 4.1006 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s +Step 1200/44000 | Loss: 4.1044 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s +Step 1210/44000 | Loss: 4.1278 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s +Step 1220/44000 | Loss: 4.1129 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s +Step 1230/44000 | Loss: 4.0767 | LR: 5.00e-04 | 684.0ms/step | 29.2 samples/s +Step 1240/44000 | Loss: 4.0811 | LR: 5.00e-04 | 684.2ms/step | 29.2 samples/s +Step 1250/44000 | Loss: 4.0561 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s +Validation at step 1250: Loss = 4.0741, Perplexity = 58.80 +Step 1260/44000 | Loss: 4.1101 | LR: 5.00e-04 | 1243.5ms/step | 16.1 samples/s +Step 1270/44000 | Loss: 4.0343 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s +Step 1280/44000 | Loss: 4.0511 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s +Step 1290/44000 | Loss: 4.0439 | LR: 5.00e-04 | 682.5ms/step | 29.3 samples/s +Step 1300/44000 | Loss: 4.0494 | LR: 5.00e-04 | 680.3ms/step | 29.4 samples/s +Step 1310/44000 | Loss: 3.9890 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s +Step 1320/44000 | Loss: 3.9759 | LR: 5.00e-04 | 684.4ms/step | 29.2 samples/s +Step 1330/44000 | Loss: 3.9987 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s +Step 1340/44000 | Loss: 4.0415 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s +Step 1350/44000 | Loss: 3.9718 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s +Step 1360/44000 | Loss: 3.9880 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s +Step 1370/44000 | Loss: 3.9674 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s +Step 1380/44000 | Loss: 3.9866 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s +Step 1390/44000 | Loss: 3.9734 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s +Step 1400/44000 | Loss: 3.9759 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s +Step 1410/44000 | Loss: 3.9504 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s +Step 1420/44000 | Loss: 3.9534 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s +Step 1430/44000 | Loss: 3.9652 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s +Step 1440/44000 | Loss: 3.8943 | LR: 5.00e-04 | 683.2ms/step | 29.3 samples/s +Step 1450/44000 | Loss: 3.9158 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s +Step 1460/44000 | Loss: 3.9641 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s +Step 1470/44000 | Loss: 3.9075 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s +Step 1480/44000 | Loss: 3.8820 | LR: 5.00e-04 | 683.5ms/step | 29.3 samples/s +Step 1490/44000 | Loss: 3.8792 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s +Step 1500/44000 | Loss: 3.9346 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s +Validation at step 1500: Loss = 3.9142, Perplexity = 50.11 +Step 1510/44000 | Loss: 3.9090 | LR: 5.00e-04 | 1238.7ms/step | 16.1 samples/s +Step 1520/44000 | Loss: 3.8981 | LR: 5.00e-04 | 683.3ms/step | 29.3 samples/s +Step 1530/44000 | Loss: 3.9228 | LR: 5.00e-04 | 682.7ms/step | 29.3 samples/s +Step 1540/44000 | Loss: 3.8443 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s +Step 1550/44000 | Loss: 3.9214 | LR: 5.00e-04 | 684.9ms/step | 29.2 samples/s +Step 1560/44000 | Loss: 3.8821 | LR: 5.00e-04 | 681.7ms/step | 29.3 samples/s +Step 1570/44000 | Loss: 3.9016 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s +Step 1580/44000 | Loss: 3.8545 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s +Step 1590/44000 | Loss: 3.8580 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s +Step 1600/44000 | Loss: 3.8513 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s +Step 1610/44000 | Loss: 3.8633 | LR: 5.00e-04 | 681.7ms/step | 29.3 samples/s +Step 1620/44000 | Loss: 3.8403 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s +Step 1630/44000 | Loss: 3.8547 | LR: 4.99e-04 | 680.4ms/step | 29.4 samples/s +Step 1640/44000 | Loss: 3.8142 | LR: 4.99e-04 | 682.2ms/step | 29.3 samples/s +Step 1650/44000 | Loss: 3.8556 | LR: 4.99e-04 | 681.9ms/step | 29.3 samples/s +Step 1660/44000 | Loss: 3.8659 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s +Step 1670/44000 | Loss: 3.8726 | LR: 4.99e-04 | 680.3ms/step | 29.4 samples/s +Step 1680/44000 | Loss: 3.8139 | LR: 4.99e-04 | 683.3ms/step | 29.3 samples/s +Step 1690/44000 | Loss: 3.7974 | LR: 4.99e-04 | 680.8ms/step | 29.4 samples/s +Step 1700/44000 | Loss: 3.8406 | LR: 4.99e-04 | 681.2ms/step | 29.4 samples/s +Step 1710/44000 | Loss: 3.7913 | LR: 4.99e-04 | 682.3ms/step | 29.3 samples/s +Step 1720/44000 | Loss: 3.8454 | LR: 4.99e-04 | 681.0ms/step | 29.4 samples/s +Step 1730/44000 | Loss: 3.8109 | LR: 4.99e-04 | 681.5ms/step | 29.3 samples/s +Step 1740/44000 | Loss: 3.7950 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s +Step 1750/44000 | Loss: 3.7722 | LR: 4.99e-04 | 682.2ms/step | 29.3 samples/s +Validation at step 1750: Loss = 3.8081, Perplexity = 45.06 +Step 1760/44000 | Loss: 3.8124 | LR: 4.99e-04 | 1244.5ms/step | 16.1 samples/s +Step 1770/44000 | Loss: 3.8140 | LR: 4.99e-04 | 683.5ms/step | 29.3 samples/s +Step 1780/44000 | Loss: 3.8096 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s +Step 1790/44000 | Loss: 3.8036 | LR: 4.99e-04 | 682.3ms/step | 29.3 samples/s +Step 1800/44000 | Loss: 3.7815 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s +Step 1810/44000 | Loss: 3.7622 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s +Step 1820/44000 | Loss: 3.7738 | LR: 4.99e-04 | 684.6ms/step | 29.2 samples/s +Step 1830/44000 | Loss: 3.7561 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s +Step 1840/44000 | Loss: 3.7443 | LR: 4.99e-04 | 684.2ms/step | 29.2 samples/s +Step 1850/44000 | Loss: 3.7809 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s +Step 1860/44000 | Loss: 3.8033 | LR: 4.99e-04 | 683.7ms/step | 29.3 samples/s +Step 1870/44000 | Loss: 3.7895 | LR: 4.99e-04 | 684.3ms/step | 29.2 samples/s +Step 1880/44000 | Loss: 3.7483 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s +Step 1890/44000 | Loss: 3.7258 | LR: 4.99e-04 | 684.1ms/step | 29.2 samples/s +Step 1900/44000 | Loss: 3.7610 | LR: 4.99e-04 | 681.5ms/step | 29.3 samples/s +Step 1910/44000 | Loss: 3.7677 | LR: 4.99e-04 | 681.1ms/step | 29.4 samples/s +Step 1920/44000 | Loss: 3.7417 | LR: 4.99e-04 | 682.8ms/step | 29.3 samples/s +Step 1930/44000 | Loss: 3.7539 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s +Step 1940/44000 | Loss: 3.7155 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s +Step 1950/44000 | Loss: 3.6881 | LR: 4.99e-04 | 686.7ms/step | 29.1 samples/s +Step 1960/44000 | Loss: 3.7373 | LR: 4.99e-04 | 684.7ms/step | 29.2 samples/s +Step 1970/44000 | Loss: 3.7051 | LR: 4.99e-04 | 682.5ms/step | 29.3 samples/s +Step 1980/44000 | Loss: 3.6944 | LR: 4.99e-04 | 682.8ms/step | 29.3 samples/s +Step 1990/44000 | Loss: 3.7627 | LR: 4.99e-04 | 682.4ms/step | 29.3 samples/s +Step 2000/44000 | Loss: 3.7249 | LR: 4.99e-04 | 683.7ms/step | 29.3 samples/s +Validation at step 2000: Loss = 3.7284, Perplexity = 41.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-2000 +Step 2010/44000 | Loss: 3.7331 | LR: 4.99e-04 | 1527.0ms/step | 13.1 samples/s +Step 2020/44000 | Loss: 3.7163 | LR: 4.99e-04 | 683.5ms/step | 29.3 samples/s +Step 2030/44000 | Loss: 3.6928 | LR: 4.99e-04 | 680.5ms/step | 29.4 samples/s +Step 2040/44000 | Loss: 3.7274 | LR: 4.99e-04 | 680.8ms/step | 29.4 samples/s +Step 2050/44000 | Loss: 3.7085 | LR: 4.99e-04 | 680.4ms/step | 29.4 samples/s +Step 2060/44000 | Loss: 3.7294 | LR: 4.99e-04 | 683.1ms/step | 29.3 samples/s +Step 2070/44000 | Loss: 3.7066 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s +Step 2080/44000 | Loss: 3.7279 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s +Step 2090/44000 | Loss: 3.7261 | LR: 4.99e-04 | 681.6ms/step | 29.3 samples/s +Step 2100/44000 | Loss: 3.6805 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s +Step 2110/44000 | Loss: 3.7071 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s +Step 2120/44000 | Loss: 3.6959 | LR: 4.99e-04 | 681.3ms/step | 29.4 samples/s +Step 2130/44000 | Loss: 3.6898 | LR: 4.99e-04 | 684.0ms/step | 29.2 samples/s +Step 2140/44000 | Loss: 3.6711 | LR: 4.99e-04 | 681.0ms/step | 29.4 samples/s +Step 2150/44000 | Loss: 3.6808 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s +Step 2160/44000 | Loss: 3.6943 | LR: 4.99e-04 | 684.8ms/step | 29.2 samples/s +Step 2170/44000 | Loss: 3.6677 | LR: 4.99e-04 | 683.1ms/step | 29.3 samples/s +Step 2180/44000 | Loss: 3.6871 | LR: 4.99e-04 | 683.6ms/step | 29.3 samples/s +Step 2190/44000 | Loss: 3.6678 | LR: 4.99e-04 | 684.5ms/step | 29.2 samples/s +Step 2200/44000 | Loss: 3.6547 | LR: 4.99e-04 | 682.4ms/step | 29.3 samples/s +Step 2210/44000 | Loss: 3.6378 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s +Step 2220/44000 | Loss: 3.6328 | LR: 4.99e-04 | 680.6ms/step | 29.4 samples/s +Step 2230/44000 | Loss: 3.6869 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s +Step 2240/44000 | Loss: 3.6823 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s +Step 2250/44000 | Loss: 3.6684 | LR: 4.99e-04 | 684.5ms/step | 29.2 samples/s +Validation at step 2250: Loss = 3.6606, Perplexity = 38.88 +Step 2260/44000 | Loss: 3.6578 | LR: 4.98e-04 | 1239.2ms/step | 16.1 samples/s +Step 2270/44000 | Loss: 3.6413 | LR: 4.98e-04 | 681.9ms/step | 29.3 samples/s +Step 2280/44000 | Loss: 3.6586 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s +Step 2290/44000 | Loss: 3.6776 | LR: 4.98e-04 | 684.0ms/step | 29.2 samples/s +Step 2300/44000 | Loss: 3.6054 | LR: 4.98e-04 | 682.7ms/step | 29.3 samples/s +Step 2310/44000 | Loss: 3.6456 | LR: 4.98e-04 | 683.2ms/step | 29.3 samples/s +Step 2320/44000 | Loss: 3.6547 | LR: 4.98e-04 | 683.7ms/step | 29.3 samples/s +Step 2330/44000 | Loss: 3.6483 | LR: 4.98e-04 | 683.6ms/step | 29.3 samples/s +Step 2340/44000 | Loss: 3.6473 | LR: 4.98e-04 | 683.1ms/step | 29.3 samples/s +Step 2350/44000 | Loss: 3.6125 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s +Step 2360/44000 | Loss: 3.6355 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s +Step 2370/44000 | Loss: 3.6366 | LR: 4.98e-04 | 679.0ms/step | 29.5 samples/s +Step 2380/44000 | Loss: 3.5960 | LR: 4.98e-04 | 682.2ms/step | 29.3 samples/s +Step 2390/44000 | Loss: 3.5817 | LR: 4.98e-04 | 681.4ms/step | 29.4 samples/s +Step 2400/44000 | Loss: 3.6369 | LR: 4.98e-04 | 681.5ms/step | 29.3 samples/s +Step 2410/44000 | Loss: 3.6284 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s +Step 2420/44000 | Loss: 3.6282 | LR: 4.98e-04 | 682.3ms/step | 29.3 samples/s +Step 2430/44000 | Loss: 3.6212 | LR: 4.98e-04 | 681.0ms/step | 29.4 samples/s +Step 2440/44000 | Loss: 3.6275 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s +Step 2450/44000 | Loss: 3.6034 | LR: 4.98e-04 | 680.9ms/step | 29.4 samples/s +Step 2460/44000 | Loss: 3.6057 | LR: 4.98e-04 | 679.2ms/step | 29.4 samples/s +Step 2470/44000 | Loss: 3.6119 | LR: 4.98e-04 | 682.4ms/step | 29.3 samples/s +Step 2480/44000 | Loss: 3.6256 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s +Step 2490/44000 | Loss: 3.5894 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s +Step 2500/44000 | Loss: 3.6012 | LR: 4.98e-04 | 681.7ms/step | 29.3 samples/s +Validation at step 2500: Loss = 3.6062, Perplexity = 36.83 +Step 2510/44000 | Loss: 3.6057 | LR: 4.98e-04 | 1247.7ms/step | 16.0 samples/s +Step 2520/44000 | Loss: 3.5934 | LR: 4.98e-04 | 682.6ms/step | 29.3 samples/s +Step 2530/44000 | Loss: 3.6051 | LR: 4.98e-04 | 683.6ms/step | 29.3 samples/s +Step 2540/44000 | Loss: 3.5801 | LR: 4.98e-04 | 681.8ms/step | 29.3 samples/s +Step 2550/44000 | Loss: 3.5463 | LR: 4.98e-04 | 680.3ms/step | 29.4 samples/s +Step 2560/44000 | Loss: 3.5690 | LR: 4.98e-04 | 681.8ms/step | 29.3 samples/s +Step 2570/44000 | Loss: 3.5804 | LR: 4.98e-04 | 683.3ms/step | 29.3 samples/s +Step 2580/44000 | Loss: 3.6015 | LR: 4.98e-04 | 683.3ms/step | 29.3 samples/s +Step 2590/44000 | Loss: 3.5855 | LR: 4.98e-04 | 681.3ms/step | 29.4 samples/s +Step 2600/44000 | Loss: 3.5511 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s +Step 2610/44000 | Loss: 3.5176 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s +Step 2620/44000 | Loss: 3.5812 | LR: 4.98e-04 | 684.3ms/step | 29.2 samples/s +Step 2630/44000 | Loss: 3.5415 | LR: 4.98e-04 | 681.5ms/step | 29.3 samples/s +Step 2640/44000 | Loss: 3.5606 | LR: 4.98e-04 | 680.9ms/step | 29.4 samples/s +Step 2650/44000 | Loss: 3.5750 | LR: 4.98e-04 | 683.7ms/step | 29.3 samples/s +Step 2660/44000 | Loss: 3.5457 | LR: 4.98e-04 | 683.1ms/step | 29.3 samples/s +Step 2670/44000 | Loss: 3.5489 | LR: 4.98e-04 | 683.9ms/step | 29.2 samples/s +Step 2680/44000 | Loss: 3.5814 | LR: 4.98e-04 | 680.7ms/step | 29.4 samples/s +Step 2690/44000 | Loss: 3.5940 | LR: 4.98e-04 | 682.2ms/step | 29.3 samples/s +Step 2700/44000 | Loss: 3.5307 | LR: 4.97e-04 | 682.5ms/step | 29.3 samples/s +Step 2710/44000 | Loss: 3.5850 | LR: 4.97e-04 | 682.3ms/step | 29.3 samples/s +Step 2720/44000 | Loss: 3.5988 | LR: 4.97e-04 | 684.4ms/step | 29.2 samples/s +Step 2730/44000 | Loss: 3.5787 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s +Step 2740/44000 | Loss: 3.5510 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s +Step 2750/44000 | Loss: 3.5302 | LR: 4.97e-04 | 682.7ms/step | 29.3 samples/s +Validation at step 2750: Loss = 3.5617, Perplexity = 35.22 +Step 2760/44000 | Loss: 3.5235 | LR: 4.97e-04 | 1241.4ms/step | 16.1 samples/s +Step 2770/44000 | Loss: 3.6002 | LR: 4.97e-04 | 682.0ms/step | 29.3 samples/s +Step 2780/44000 | Loss: 3.5929 | LR: 4.97e-04 | 683.1ms/step | 29.3 samples/s +Step 2790/44000 | Loss: 3.5685 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s +Step 2800/44000 | Loss: 3.5706 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s +Step 2810/44000 | Loss: 3.5281 | LR: 4.97e-04 | 683.2ms/step | 29.3 samples/s +Step 2820/44000 | Loss: 3.5558 | LR: 4.97e-04 | 682.6ms/step | 29.3 samples/s +Step 2830/44000 | Loss: 3.5401 | LR: 4.97e-04 | 683.1ms/step | 29.3 samples/s +Step 2840/44000 | Loss: 3.5146 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s +Step 2850/44000 | Loss: 3.5859 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s +Step 2860/44000 | Loss: 3.5575 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s +Step 2870/44000 | Loss: 3.5329 | LR: 4.97e-04 | 681.6ms/step | 29.3 samples/s +Step 2880/44000 | Loss: 3.4896 | LR: 4.97e-04 | 681.3ms/step | 29.4 samples/s +Step 2890/44000 | Loss: 3.5236 | LR: 4.97e-04 | 682.2ms/step | 29.3 samples/s +Step 2900/44000 | Loss: 3.5765 | LR: 4.97e-04 | 682.9ms/step | 29.3 samples/s +Step 2910/44000 | Loss: 3.5248 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s +Step 2920/44000 | Loss: 3.5438 | LR: 4.97e-04 | 679.7ms/step | 29.4 samples/s +Step 2930/44000 | Loss: 3.5284 | LR: 4.97e-04 | 681.1ms/step | 29.4 samples/s +Step 2940/44000 | Loss: 3.5211 | LR: 4.97e-04 | 681.1ms/step | 29.4 samples/s +Step 2950/44000 | Loss: 3.5509 | LR: 4.97e-04 | 681.7ms/step | 29.3 samples/s +Step 2960/44000 | Loss: 3.5302 | LR: 4.97e-04 | 682.7ms/step | 29.3 samples/s +Step 2970/44000 | Loss: 3.5520 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s +Step 2980/44000 | Loss: 3.6119 | LR: 4.97e-04 | 680.9ms/step | 29.4 samples/s +Step 2990/44000 | Loss: 3.5136 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s +Step 3000/44000 | Loss: 3.5309 | LR: 4.97e-04 | 683.2ms/step | 29.3 samples/s +Validation at step 3000: Loss = 3.5276, Perplexity = 34.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-3000 +Step 3010/44000 | Loss: 3.5184 | LR: 4.97e-04 | 1522.3ms/step | 13.1 samples/s +Step 3020/44000 | Loss: 3.5128 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s +Step 3030/44000 | Loss: 3.5592 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s +Step 3040/44000 | Loss: 3.5153 | LR: 4.97e-04 | 680.5ms/step | 29.4 samples/s +Step 3050/44000 | Loss: 3.5552 | LR: 4.97e-04 | 682.6ms/step | 29.3 samples/s +Step 3060/44000 | Loss: 3.4928 | LR: 4.96e-04 | 683.5ms/step | 29.3 samples/s +Step 3070/44000 | Loss: 3.5235 | LR: 4.96e-04 | 681.3ms/step | 29.4 samples/s +Step 3080/44000 | Loss: 3.5322 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s +Step 3090/44000 | Loss: 3.4932 | LR: 4.96e-04 | 681.9ms/step | 29.3 samples/s +Step 3100/44000 | Loss: 3.5170 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s +Step 3110/44000 | Loss: 3.4852 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s +Step 3120/44000 | Loss: 3.4994 | LR: 4.96e-04 | 680.1ms/step | 29.4 samples/s +Step 3130/44000 | Loss: 3.4935 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s +Step 3140/44000 | Loss: 3.4892 | LR: 4.96e-04 | 680.6ms/step | 29.4 samples/s +Step 3150/44000 | Loss: 3.4834 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s +Step 3160/44000 | Loss: 3.5026 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s +Step 3170/44000 | Loss: 3.4916 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s +Step 3180/44000 | Loss: 3.5293 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s +Step 3190/44000 | Loss: 3.5257 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s +Step 3200/44000 | Loss: 3.4865 | LR: 4.96e-04 | 681.5ms/step | 29.3 samples/s +Step 3210/44000 | Loss: 3.4721 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s +Step 3220/44000 | Loss: 3.5098 | LR: 4.96e-04 | 682.8ms/step | 29.3 samples/s +Step 3230/44000 | Loss: 3.4769 | LR: 4.96e-04 | 681.6ms/step | 29.3 samples/s +Step 3240/44000 | Loss: 3.4926 | LR: 4.96e-04 | 680.8ms/step | 29.4 samples/s +Step 3250/44000 | Loss: 3.4725 | LR: 4.96e-04 | 679.6ms/step | 29.4 samples/s +Validation at step 3250: Loss = 3.4923, Perplexity = 32.86 +Step 3260/44000 | Loss: 3.5206 | LR: 4.96e-04 | 1247.0ms/step | 16.0 samples/s +Step 3270/44000 | Loss: 3.4743 | LR: 4.96e-04 | 682.1ms/step | 29.3 samples/s +Step 3280/44000 | Loss: 3.4761 | LR: 4.96e-04 | 683.5ms/step | 29.3 samples/s +Step 3290/44000 | Loss: 3.4805 | LR: 4.96e-04 | 683.7ms/step | 29.3 samples/s +Step 3300/44000 | Loss: 3.4668 | LR: 4.96e-04 | 682.3ms/step | 29.3 samples/s +Step 3310/44000 | Loss: 3.4832 | LR: 4.96e-04 | 681.9ms/step | 29.3 samples/s +Step 3320/44000 | Loss: 3.4688 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s +Step 3330/44000 | Loss: 3.4872 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s +Step 3340/44000 | Loss: 3.4631 | LR: 4.96e-04 | 681.3ms/step | 29.4 samples/s +Step 3350/44000 | Loss: 3.4930 | LR: 4.96e-04 | 682.6ms/step | 29.3 samples/s +Step 3360/44000 | Loss: 3.4527 | LR: 4.96e-04 | 683.3ms/step | 29.3 samples/s +Step 3370/44000 | Loss: 3.4798 | LR: 4.95e-04 | 683.8ms/step | 29.3 samples/s +Step 3380/44000 | Loss: 3.4679 | LR: 4.95e-04 | 682.5ms/step | 29.3 samples/s +Step 3390/44000 | Loss: 3.4798 | LR: 4.95e-04 | 684.0ms/step | 29.2 samples/s +Step 3400/44000 | Loss: 3.4594 | LR: 4.95e-04 | 682.8ms/step | 29.3 samples/s +Step 3410/44000 | Loss: 3.4802 | LR: 4.95e-04 | 680.9ms/step | 29.4 samples/s +Step 3420/44000 | Loss: 3.3880 | LR: 4.95e-04 | 681.2ms/step | 29.4 samples/s +Step 3430/44000 | Loss: 3.4410 | LR: 4.95e-04 | 679.7ms/step | 29.4 samples/s +Step 3440/44000 | Loss: 3.4704 | LR: 4.95e-04 | 682.3ms/step | 29.3 samples/s +Step 3450/44000 | Loss: 3.4364 | LR: 4.95e-04 | 681.3ms/step | 29.4 samples/s +Step 3460/44000 | Loss: 3.4823 | LR: 4.95e-04 | 682.2ms/step | 29.3 samples/s +Step 3470/44000 | Loss: 3.4830 | LR: 4.95e-04 | 681.6ms/step | 29.3 samples/s +Step 3480/44000 | Loss: 3.4622 | LR: 4.95e-04 | 683.7ms/step | 29.3 samples/s +Step 3490/44000 | Loss: 3.4707 | LR: 4.95e-04 | 681.4ms/step | 29.4 samples/s +Step 3500/44000 | Loss: 3.4223 | LR: 4.95e-04 | 682.5ms/step | 29.3 samples/s +Validation at step 3500: Loss = 3.4642, Perplexity = 31.95 +Step 3510/44000 | Loss: 3.4733 | LR: 4.95e-04 | 1252.3ms/step | 16.0 samples/s +Step 3520/44000 | Loss: 3.4341 | LR: 4.95e-04 | 685.1ms/step | 29.2 samples/s +Step 3530/44000 | Loss: 3.4105 | LR: 4.95e-04 | 683.0ms/step | 29.3 samples/s +Step 3540/44000 | Loss: 3.4402 | LR: 4.95e-04 | 682.7ms/step | 29.3 samples/s +Step 3550/44000 | Loss: 3.4576 | LR: 4.95e-04 | 682.6ms/step | 29.3 samples/s +Step 3560/44000 | Loss: 3.4948 | LR: 4.95e-04 | 681.3ms/step | 29.4 samples/s +Step 3570/44000 | Loss: 3.4166 | LR: 4.95e-04 | 681.1ms/step | 29.4 samples/s +Step 3580/44000 | Loss: 3.4434 | LR: 4.95e-04 | 681.7ms/step | 29.3 samples/s +Step 3590/44000 | Loss: 3.4437 | LR: 4.95e-04 | 682.6ms/step | 29.3 samples/s +Step 3600/44000 | Loss: 3.4644 | LR: 4.95e-04 | 682.3ms/step | 29.3 samples/s +Step 3610/44000 | Loss: 3.4921 | LR: 4.95e-04 | 683.2ms/step | 29.3 samples/s +Step 3620/44000 | Loss: 3.4481 | LR: 4.95e-04 | 681.9ms/step | 29.3 samples/s +Step 3630/44000 | Loss: 3.4289 | LR: 4.95e-04 | 681.9ms/step | 29.3 samples/s +Step 3640/44000 | Loss: 3.4517 | LR: 4.95e-04 | 681.0ms/step | 29.4 samples/s +Step 3650/44000 | Loss: 3.4796 | LR: 4.94e-04 | 680.6ms/step | 29.4 samples/s +Step 3660/44000 | Loss: 3.4184 | LR: 4.94e-04 | 680.2ms/step | 29.4 samples/s +Step 3670/44000 | Loss: 3.4607 | LR: 4.94e-04 | 680.8ms/step | 29.4 samples/s +Step 3680/44000 | Loss: 3.4027 | LR: 4.94e-04 | 680.4ms/step | 29.4 samples/s +Step 3690/44000 | Loss: 3.4122 | LR: 4.94e-04 | 679.8ms/step | 29.4 samples/s +Step 3700/44000 | Loss: 3.4601 | LR: 4.94e-04 | 679.5ms/step | 29.4 samples/s +Step 3710/44000 | Loss: 3.4568 | LR: 4.94e-04 | 681.4ms/step | 29.4 samples/s +Step 3720/44000 | Loss: 3.4345 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s +Step 3730/44000 | Loss: 3.4070 | LR: 4.94e-04 | 682.4ms/step | 29.3 samples/s +Step 3740/44000 | Loss: 3.4669 | LR: 4.94e-04 | 683.0ms/step | 29.3 samples/s +Step 3750/44000 | Loss: 3.4222 | LR: 4.94e-04 | 681.6ms/step | 29.3 samples/s +Validation at step 3750: Loss = 3.4390, Perplexity = 31.16 +Step 3760/44000 | Loss: 3.4512 | LR: 4.94e-04 | 1249.4ms/step | 16.0 samples/s +Step 3770/44000 | Loss: 3.4291 | LR: 4.94e-04 | 682.9ms/step | 29.3 samples/s +Step 3780/44000 | Loss: 3.4285 | LR: 4.94e-04 | 682.6ms/step | 29.3 samples/s +Step 3790/44000 | Loss: 3.4337 | LR: 4.94e-04 | 683.0ms/step | 29.3 samples/s +Step 3800/44000 | Loss: 3.4043 | LR: 4.94e-04 | 683.9ms/step | 29.2 samples/s +Step 3810/44000 | Loss: 3.3895 | LR: 4.94e-04 | 682.0ms/step | 29.3 samples/s +Step 3820/44000 | Loss: 3.3728 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s +Step 3830/44000 | Loss: 3.4081 | LR: 4.94e-04 | 683.8ms/step | 29.2 samples/s +Step 3840/44000 | Loss: 3.4717 | LR: 4.94e-04 | 683.2ms/step | 29.3 samples/s +Step 3850/44000 | Loss: 3.4292 | LR: 4.94e-04 | 683.5ms/step | 29.3 samples/s +Step 3860/44000 | Loss: 3.4177 | LR: 4.94e-04 | 681.0ms/step | 29.4 samples/s +Step 3870/44000 | Loss: 3.4681 | LR: 4.94e-04 | 682.8ms/step | 29.3 samples/s +Step 3880/44000 | Loss: 3.4396 | LR: 4.94e-04 | 685.1ms/step | 29.2 samples/s +Step 3890/44000 | Loss: 3.4616 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s +Step 3900/44000 | Loss: 3.4096 | LR: 4.93e-04 | 681.0ms/step | 29.4 samples/s +Step 3910/44000 | Loss: 3.4503 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s +Step 3920/44000 | Loss: 3.4124 | LR: 4.93e-04 | 679.4ms/step | 29.4 samples/s +Step 3930/44000 | Loss: 3.4170 | LR: 4.93e-04 | 680.3ms/step | 29.4 samples/s +Step 3940/44000 | Loss: 3.4201 | LR: 4.93e-04 | 679.7ms/step | 29.4 samples/s +Step 3950/44000 | Loss: 3.3592 | LR: 4.93e-04 | 679.5ms/step | 29.4 samples/s +Step 3960/44000 | Loss: 3.4302 | LR: 4.93e-04 | 679.4ms/step | 29.4 samples/s +Step 3970/44000 | Loss: 3.4533 | LR: 4.93e-04 | 679.9ms/step | 29.4 samples/s +Step 3980/44000 | Loss: 3.3968 | LR: 4.93e-04 | 678.2ms/step | 29.5 samples/s +Step 3990/44000 | Loss: 3.3934 | LR: 4.93e-04 | 677.5ms/step | 29.5 samples/s +Step 4000/44000 | Loss: 3.4114 | LR: 4.93e-04 | 677.6ms/step | 29.5 samples/s +Validation at step 4000: Loss = 3.4178, Perplexity = 30.50 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-4000 +Step 4010/44000 | Loss: 3.4281 | LR: 4.93e-04 | 1525.1ms/step | 13.1 samples/s +Step 4020/44000 | Loss: 3.4596 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s +Step 4030/44000 | Loss: 3.3729 | LR: 4.93e-04 | 678.4ms/step | 29.5 samples/s +Step 4040/44000 | Loss: 3.3798 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s +Step 4050/44000 | Loss: 3.3952 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s +Step 4060/44000 | Loss: 3.3878 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s +Step 4070/44000 | Loss: 3.4152 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s +Step 4080/44000 | Loss: 3.4150 | LR: 4.93e-04 | 678.4ms/step | 29.5 samples/s +Step 4090/44000 | Loss: 3.4078 | LR: 4.93e-04 | 679.0ms/step | 29.5 samples/s +Step 4100/44000 | Loss: 3.3983 | LR: 4.93e-04 | 680.2ms/step | 29.4 samples/s +Step 4110/44000 | Loss: 3.4223 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s +Step 4120/44000 | Loss: 3.4187 | LR: 4.93e-04 | 678.8ms/step | 29.5 samples/s +Step 4130/44000 | Loss: 3.4097 | LR: 4.93e-04 | 679.2ms/step | 29.4 samples/s +Step 4140/44000 | Loss: 3.4083 | LR: 4.92e-04 | 678.3ms/step | 29.5 samples/s +Step 4150/44000 | Loss: 3.3893 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s +Step 4160/44000 | Loss: 3.4045 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s +Step 4170/44000 | Loss: 3.3740 | LR: 4.92e-04 | 677.5ms/step | 29.5 samples/s +Step 4180/44000 | Loss: 3.3596 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s +Step 4190/44000 | Loss: 3.4140 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s +Step 4200/44000 | Loss: 3.3799 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s +Step 4210/44000 | Loss: 3.4241 | LR: 4.92e-04 | 680.1ms/step | 29.4 samples/s +Step 4220/44000 | Loss: 3.3931 | LR: 4.92e-04 | 679.3ms/step | 29.4 samples/s +Step 4230/44000 | Loss: 3.4336 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s +Step 4240/44000 | Loss: 3.3840 | LR: 4.92e-04 | 679.9ms/step | 29.4 samples/s +Step 4250/44000 | Loss: 3.4068 | LR: 4.92e-04 | 679.6ms/step | 29.4 samples/s +Validation at step 4250: Loss = 3.3950, Perplexity = 29.81 +Step 4260/44000 | Loss: 3.3487 | LR: 4.92e-04 | 1233.9ms/step | 16.2 samples/s +Step 4270/44000 | Loss: 3.3628 | LR: 4.92e-04 | 678.4ms/step | 29.5 samples/s +Step 4280/44000 | Loss: 3.4044 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s +Step 4290/44000 | Loss: 3.3746 | LR: 4.92e-04 | 680.6ms/step | 29.4 samples/s +Step 4300/44000 | Loss: 3.4053 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s +Step 4310/44000 | Loss: 3.3666 | LR: 4.92e-04 | 680.5ms/step | 29.4 samples/s +Step 4320/44000 | Loss: 3.3756 | LR: 4.92e-04 | 679.0ms/step | 29.5 samples/s +Step 4330/44000 | Loss: 3.3588 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s +Step 4340/44000 | Loss: 3.3874 | LR: 4.92e-04 | 679.3ms/step | 29.4 samples/s +Step 4350/44000 | Loss: 3.3448 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s +Step 4360/44000 | Loss: 3.3811 | LR: 4.91e-04 | 678.5ms/step | 29.5 samples/s +Step 4370/44000 | Loss: 3.3967 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s +Step 4380/44000 | Loss: 3.3458 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s +Step 4390/44000 | Loss: 3.3588 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s +Step 4400/44000 | Loss: 3.4147 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s +Step 4410/44000 | Loss: 3.3854 | LR: 4.91e-04 | 680.7ms/step | 29.4 samples/s +Step 4420/44000 | Loss: 3.3861 | LR: 4.91e-04 | 678.9ms/step | 29.5 samples/s +Step 4430/44000 | Loss: 3.3763 | LR: 4.91e-04 | 678.9ms/step | 29.5 samples/s +Step 4440/44000 | Loss: 3.3891 | LR: 4.91e-04 | 679.0ms/step | 29.5 samples/s +Step 4450/44000 | Loss: 3.3627 | LR: 4.91e-04 | 679.6ms/step | 29.4 samples/s +Step 4460/44000 | Loss: 3.3368 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s +Step 4470/44000 | Loss: 3.4033 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s +Step 4480/44000 | Loss: 3.3707 | LR: 4.91e-04 | 679.3ms/step | 29.4 samples/s +Step 4490/44000 | Loss: 3.3335 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s +Step 4500/44000 | Loss: 3.3609 | LR: 4.91e-04 | 678.6ms/step | 29.5 samples/s +Validation at step 4500: Loss = 3.3769, Perplexity = 29.28 +Step 4510/44000 | Loss: 3.3556 | LR: 4.91e-04 | 1234.8ms/step | 16.2 samples/s +Step 4520/44000 | Loss: 3.3521 | LR: 4.91e-04 | 679.6ms/step | 29.4 samples/s +Step 4530/44000 | Loss: 3.3342 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s +Step 4540/44000 | Loss: 3.3765 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s +Step 4550/44000 | Loss: 3.3709 | LR: 4.91e-04 | 678.2ms/step | 29.5 samples/s +Step 4560/44000 | Loss: 3.3867 | LR: 4.90e-04 | 678.6ms/step | 29.5 samples/s +Step 4570/44000 | Loss: 3.3832 | LR: 4.90e-04 | 677.5ms/step | 29.5 samples/s +Step 4580/44000 | Loss: 3.3673 | LR: 4.90e-04 | 678.4ms/step | 29.5 samples/s +Step 4590/44000 | Loss: 3.3171 | LR: 4.90e-04 | 678.3ms/step | 29.5 samples/s +Step 4600/44000 | Loss: 3.3516 | LR: 4.90e-04 | 679.5ms/step | 29.4 samples/s +Step 4610/44000 | Loss: 3.3609 | LR: 4.90e-04 | 679.1ms/step | 29.5 samples/s +Step 4620/44000 | Loss: 3.3258 | LR: 4.90e-04 | 677.5ms/step | 29.5 samples/s +Step 4630/44000 | Loss: 3.3486 | LR: 4.90e-04 | 678.4ms/step | 29.5 samples/s +Step 4640/44000 | Loss: 3.3631 | LR: 4.90e-04 | 677.2ms/step | 29.5 samples/s +Step 4650/44000 | Loss: 3.3265 | LR: 4.90e-04 | 678.7ms/step | 29.5 samples/s +Step 4660/44000 | Loss: 3.3211 | LR: 4.90e-04 | 679.8ms/step | 29.4 samples/s +Step 4670/44000 | Loss: 3.3957 | LR: 4.90e-04 | 680.1ms/step | 29.4 samples/s +Step 4680/44000 | Loss: 3.3563 | LR: 4.90e-04 | 678.8ms/step | 29.5 samples/s +Step 4690/44000 | Loss: 3.4132 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s +Step 4700/44000 | Loss: 3.3966 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s +Step 4710/44000 | Loss: 3.3839 | LR: 4.90e-04 | 679.2ms/step | 29.4 samples/s +Step 4720/44000 | Loss: 3.3321 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s +Step 4730/44000 | Loss: 3.3661 | LR: 4.90e-04 | 679.5ms/step | 29.4 samples/s +Step 4740/44000 | Loss: 3.3577 | LR: 4.90e-04 | 680.3ms/step | 29.4 samples/s +Step 4750/44000 | Loss: 3.3139 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s +Validation at step 4750: Loss = 3.3614, Perplexity = 28.83 +Step 4760/44000 | Loss: 3.3747 | LR: 4.89e-04 | 1233.8ms/step | 16.2 samples/s +Step 4770/44000 | Loss: 3.3652 | LR: 4.89e-04 | 679.9ms/step | 29.4 samples/s +Step 4780/44000 | Loss: 3.3694 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s +Step 4790/44000 | Loss: 3.3512 | LR: 4.89e-04 | 680.1ms/step | 29.4 samples/s +Step 4800/44000 | Loss: 3.3583 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s +Step 4810/44000 | Loss: 3.3428 | LR: 4.89e-04 | 680.2ms/step | 29.4 samples/s +Step 4820/44000 | Loss: 3.3505 | LR: 4.89e-04 | 679.0ms/step | 29.5 samples/s +Step 4830/44000 | Loss: 3.3398 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s +Step 4840/44000 | Loss: 3.3197 | LR: 4.89e-04 | 680.0ms/step | 29.4 samples/s +Step 4850/44000 | Loss: 3.3333 | LR: 4.89e-04 | 681.2ms/step | 29.4 samples/s +Step 4860/44000 | Loss: 3.3329 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s +Step 4870/44000 | Loss: 3.3272 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s +Step 4880/44000 | Loss: 3.3495 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s +Step 4890/44000 | Loss: 3.3169 | LR: 4.89e-04 | 678.7ms/step | 29.5 samples/s +Step 4900/44000 | Loss: 3.3860 | LR: 4.89e-04 | 678.3ms/step | 29.5 samples/s +Step 4910/44000 | Loss: 3.3662 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s +Step 4920/44000 | Loss: 3.3651 | LR: 4.89e-04 | 677.5ms/step | 29.5 samples/s +Step 4930/44000 | Loss: 3.3539 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s +Step 4940/44000 | Loss: 3.3554 | LR: 4.89e-04 | 678.7ms/step | 29.5 samples/s +Step 4950/44000 | Loss: 3.3109 | LR: 4.88e-04 | 680.9ms/step | 29.4 samples/s +Step 4960/44000 | Loss: 3.3611 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s +Step 4970/44000 | Loss: 3.3116 | LR: 4.88e-04 | 679.0ms/step | 29.5 samples/s +Step 4980/44000 | Loss: 3.3370 | LR: 4.88e-04 | 678.5ms/step | 29.5 samples/s +Step 4990/44000 | Loss: 3.3416 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s +Step 5000/44000 | Loss: 3.3401 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s +Validation at step 5000: Loss = 3.3442, Perplexity = 28.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-5000 +Step 5010/44000 | Loss: 3.3201 | LR: 4.88e-04 | 1521.3ms/step | 13.1 samples/s +Step 5020/44000 | Loss: 3.3306 | LR: 4.88e-04 | 678.3ms/step | 29.5 samples/s +Step 5030/44000 | Loss: 3.3327 | LR: 4.88e-04 | 677.6ms/step | 29.5 samples/s +Step 5040/44000 | Loss: 3.3609 | LR: 4.88e-04 | 678.3ms/step | 29.5 samples/s +Step 5050/44000 | Loss: 3.3322 | LR: 4.88e-04 | 677.3ms/step | 29.5 samples/s +Step 5060/44000 | Loss: 3.3437 | LR: 4.88e-04 | 679.3ms/step | 29.4 samples/s +Step 5070/44000 | Loss: 3.3212 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s +Step 5080/44000 | Loss: 3.3374 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s +Step 5090/44000 | Loss: 3.3319 | LR: 4.88e-04 | 679.7ms/step | 29.4 samples/s +Step 5100/44000 | Loss: 3.3516 | LR: 4.88e-04 | 679.6ms/step | 29.4 samples/s +Step 5110/44000 | Loss: 3.3098 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s +Step 5120/44000 | Loss: 3.3821 | LR: 4.88e-04 | 678.1ms/step | 29.5 samples/s +Step 5130/44000 | Loss: 3.3418 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s +Step 5140/44000 | Loss: 3.3210 | LR: 4.87e-04 | 679.5ms/step | 29.4 samples/s +Step 5150/44000 | Loss: 3.3319 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s +Step 5160/44000 | Loss: 3.3373 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s +Step 5170/44000 | Loss: 3.2874 | LR: 4.87e-04 | 678.7ms/step | 29.5 samples/s +Step 5180/44000 | Loss: 3.3305 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s +Step 5190/44000 | Loss: 3.3494 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s +Step 5200/44000 | Loss: 3.3278 | LR: 4.87e-04 | 678.9ms/step | 29.5 samples/s +Step 5210/44000 | Loss: 3.3478 | LR: 4.87e-04 | 677.8ms/step | 29.5 samples/s +Step 5220/44000 | Loss: 3.3379 | LR: 4.87e-04 | 678.1ms/step | 29.5 samples/s +Step 5230/44000 | Loss: 3.3458 | LR: 4.87e-04 | 678.0ms/step | 29.5 samples/s +Step 5240/44000 | Loss: 3.3979 | LR: 4.87e-04 | 677.1ms/step | 29.5 samples/s +Step 5250/44000 | Loss: 3.3364 | LR: 4.87e-04 | 678.5ms/step | 29.5 samples/s +Validation at step 5250: Loss = 3.3295, Perplexity = 27.93 +Step 5260/44000 | Loss: 3.3595 | LR: 4.87e-04 | 1234.7ms/step | 16.2 samples/s +Step 5270/44000 | Loss: 3.3485 | LR: 4.87e-04 | 677.6ms/step | 29.5 samples/s +Step 5280/44000 | Loss: 3.3431 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s +Step 5290/44000 | Loss: 3.3174 | LR: 4.87e-04 | 679.0ms/step | 29.5 samples/s +Step 5300/44000 | Loss: 3.3188 | LR: 4.86e-04 | 681.3ms/step | 29.4 samples/s +Step 5310/44000 | Loss: 3.3468 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s +Step 5320/44000 | Loss: 3.2971 | LR: 4.86e-04 | 678.9ms/step | 29.5 samples/s +Step 5330/44000 | Loss: 3.3454 | LR: 4.86e-04 | 679.3ms/step | 29.4 samples/s +Step 5340/44000 | Loss: 3.3099 | LR: 4.86e-04 | 679.7ms/step | 29.4 samples/s +Step 5350/44000 | Loss: 3.3012 | LR: 4.86e-04 | 680.3ms/step | 29.4 samples/s +Step 5360/44000 | Loss: 3.3069 | LR: 4.86e-04 | 679.6ms/step | 29.4 samples/s +Step 5370/44000 | Loss: 3.3328 | LR: 4.86e-04 | 680.9ms/step | 29.4 samples/s +Step 5380/44000 | Loss: 3.3176 | LR: 4.86e-04 | 679.3ms/step | 29.4 samples/s +Step 5390/44000 | Loss: 3.2883 | LR: 4.86e-04 | 680.2ms/step | 29.4 samples/s +Step 5400/44000 | Loss: 3.3253 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s +Step 5410/44000 | Loss: 3.2998 | LR: 4.86e-04 | 679.2ms/step | 29.4 samples/s +Step 5420/44000 | Loss: 3.3220 | LR: 4.86e-04 | 679.0ms/step | 29.5 samples/s +Step 5430/44000 | Loss: 3.3162 | LR: 4.86e-04 | 679.4ms/step | 29.4 samples/s +Step 5440/44000 | Loss: 3.3488 | LR: 4.86e-04 | 678.1ms/step | 29.5 samples/s +Step 5450/44000 | Loss: 3.2814 | LR: 4.86e-04 | 679.2ms/step | 29.4 samples/s +Step 5460/44000 | Loss: 3.2891 | LR: 4.86e-04 | 678.3ms/step | 29.5 samples/s +Step 5470/44000 | Loss: 3.2895 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s +Step 5480/44000 | Loss: 3.3356 | LR: 4.85e-04 | 678.8ms/step | 29.5 samples/s +Step 5490/44000 | Loss: 3.3519 | LR: 4.85e-04 | 678.6ms/step | 29.5 samples/s +Step 5500/44000 | Loss: 3.3232 | LR: 4.85e-04 | 679.5ms/step | 29.4 samples/s +Validation at step 5500: Loss = 3.3156, Perplexity = 27.54 +Step 5510/44000 | Loss: 3.2666 | LR: 4.85e-04 | 1233.2ms/step | 16.2 samples/s +Step 5520/44000 | Loss: 3.2916 | LR: 4.85e-04 | 678.6ms/step | 29.5 samples/s +Step 5530/44000 | Loss: 3.2834 | LR: 4.85e-04 | 678.9ms/step | 29.5 samples/s +Step 5540/44000 | Loss: 3.2878 | LR: 4.85e-04 | 679.0ms/step | 29.5 samples/s +Step 5550/44000 | Loss: 3.3214 | LR: 4.85e-04 | 679.9ms/step | 29.4 samples/s +Step 5560/44000 | Loss: 3.3214 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s +Step 5570/44000 | Loss: 3.3011 | LR: 4.85e-04 | 678.8ms/step | 29.5 samples/s +Step 5580/44000 | Loss: 3.3161 | LR: 4.85e-04 | 679.8ms/step | 29.4 samples/s +Step 5590/44000 | Loss: 3.3378 | LR: 4.85e-04 | 679.8ms/step | 29.4 samples/s +Step 5600/44000 | Loss: 3.3209 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s +Step 5610/44000 | Loss: 3.2975 | LR: 4.85e-04 | 679.3ms/step | 29.4 samples/s +Step 5620/44000 | Loss: 3.3005 | LR: 4.85e-04 | 678.3ms/step | 29.5 samples/s +Step 5630/44000 | Loss: 3.2914 | LR: 4.84e-04 | 678.4ms/step | 29.5 samples/s +Step 5640/44000 | Loss: 3.3362 | LR: 4.84e-04 | 678.0ms/step | 29.5 samples/s +Step 5650/44000 | Loss: 3.2457 | LR: 4.84e-04 | 678.3ms/step | 29.5 samples/s +Step 5660/44000 | Loss: 3.2908 | LR: 4.84e-04 | 678.8ms/step | 29.5 samples/s +Step 5670/44000 | Loss: 3.3123 | LR: 4.84e-04 | 678.7ms/step | 29.5 samples/s +Step 5680/44000 | Loss: 3.2936 | LR: 4.84e-04 | 680.3ms/step | 29.4 samples/s +Step 5690/44000 | Loss: 3.2909 | LR: 4.84e-04 | 679.8ms/step | 29.4 samples/s +Step 5700/44000 | Loss: 3.2340 | LR: 4.84e-04 | 679.9ms/step | 29.4 samples/s +Step 5710/44000 | Loss: 3.3004 | LR: 4.84e-04 | 680.5ms/step | 29.4 samples/s +Step 5720/44000 | Loss: 3.3141 | LR: 4.84e-04 | 679.5ms/step | 29.4 samples/s +Step 5730/44000 | Loss: 3.2876 | LR: 4.84e-04 | 679.8ms/step | 29.4 samples/s +Step 5740/44000 | Loss: 3.2518 | LR: 4.84e-04 | 679.4ms/step | 29.4 samples/s +Step 5750/44000 | Loss: 3.2692 | LR: 4.84e-04 | 679.0ms/step | 29.5 samples/s +Validation at step 5750: Loss = 3.3021, Perplexity = 27.17 +Step 5760/44000 | Loss: 3.3389 | LR: 4.84e-04 | 1232.6ms/step | 16.2 samples/s +Step 5770/44000 | Loss: 3.3094 | LR: 4.84e-04 | 679.0ms/step | 29.5 samples/s +Step 5780/44000 | Loss: 3.2895 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s +Step 5790/44000 | Loss: 3.2590 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s +Step 5800/44000 | Loss: 3.3042 | LR: 4.83e-04 | 678.2ms/step | 29.5 samples/s +Step 5810/44000 | Loss: 3.3517 | LR: 4.83e-04 | 680.2ms/step | 29.4 samples/s +Step 5820/44000 | Loss: 3.3069 | LR: 4.83e-04 | 678.7ms/step | 29.5 samples/s +Step 5830/44000 | Loss: 3.3161 | LR: 4.83e-04 | 679.4ms/step | 29.4 samples/s +Step 5840/44000 | Loss: 3.2882 | LR: 4.83e-04 | 679.3ms/step | 29.4 samples/s +Step 5850/44000 | Loss: 3.2929 | LR: 4.83e-04 | 677.5ms/step | 29.5 samples/s +Step 5860/44000 | Loss: 3.2733 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s +Step 5870/44000 | Loss: 3.2655 | LR: 4.83e-04 | 679.0ms/step | 29.5 samples/s +Step 5880/44000 | Loss: 3.3012 | LR: 4.83e-04 | 678.2ms/step | 29.5 samples/s +Step 5890/44000 | Loss: 3.3211 | LR: 4.83e-04 | 677.8ms/step | 29.5 samples/s +Step 5900/44000 | Loss: 3.3090 | LR: 4.83e-04 | 679.8ms/step | 29.4 samples/s +Step 5910/44000 | Loss: 3.3210 | LR: 4.83e-04 | 679.5ms/step | 29.4 samples/s +Step 5920/44000 | Loss: 3.3013 | LR: 4.83e-04 | 679.9ms/step | 29.4 samples/s +Step 5930/44000 | Loss: 3.2569 | LR: 4.83e-04 | 679.5ms/step | 29.4 samples/s +Step 5940/44000 | Loss: 3.2824 | LR: 4.82e-04 | 679.2ms/step | 29.4 samples/s +Step 5950/44000 | Loss: 3.2879 | LR: 4.82e-04 | 679.6ms/step | 29.4 samples/s +Step 5960/44000 | Loss: 3.2794 | LR: 4.82e-04 | 678.7ms/step | 29.5 samples/s +Step 5970/44000 | Loss: 3.3082 | LR: 4.82e-04 | 679.7ms/step | 29.4 samples/s +Step 5980/44000 | Loss: 3.2952 | LR: 4.82e-04 | 679.8ms/step | 29.4 samples/s +Step 5990/44000 | Loss: 3.2891 | LR: 4.82e-04 | 678.9ms/step | 29.5 samples/s +Step 6000/44000 | Loss: 3.2904 | LR: 4.82e-04 | 679.5ms/step | 29.4 samples/s +Validation at step 6000: Loss = 3.2899, Perplexity = 26.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-6000 +Step 6010/44000 | Loss: 3.3171 | LR: 4.82e-04 | 1517.4ms/step | 13.2 samples/s +Step 6020/44000 | Loss: 3.3210 | LR: 4.82e-04 | 678.6ms/step | 29.5 samples/s +Step 6030/44000 | Loss: 3.2763 | LR: 4.82e-04 | 678.5ms/step | 29.5 samples/s +Step 6040/44000 | Loss: 3.2888 | LR: 4.82e-04 | 680.5ms/step | 29.4 samples/s +Step 6050/44000 | Loss: 3.2496 | LR: 4.82e-04 | 678.9ms/step | 29.5 samples/s +Step 6060/44000 | Loss: 3.2594 | LR: 4.82e-04 | 679.1ms/step | 29.4 samples/s +Step 6070/44000 | Loss: 3.2980 | LR: 4.82e-04 | 680.3ms/step | 29.4 samples/s +Step 6080/44000 | Loss: 3.2949 | LR: 4.81e-04 | 678.5ms/step | 29.5 samples/s +Step 6090/44000 | Loss: 3.3161 | LR: 4.81e-04 | 678.7ms/step | 29.5 samples/s +Step 6100/44000 | Loss: 3.2980 | LR: 4.81e-04 | 679.2ms/step | 29.4 samples/s +Step 6110/44000 | Loss: 3.2402 | LR: 4.81e-04 | 679.3ms/step | 29.4 samples/s +Step 6120/44000 | Loss: 3.2767 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s +Step 6130/44000 | Loss: 3.2639 | LR: 4.81e-04 | 679.2ms/step | 29.4 samples/s +Step 6140/44000 | Loss: 3.2620 | LR: 4.81e-04 | 678.9ms/step | 29.5 samples/s +Step 6150/44000 | Loss: 3.2545 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s +Step 6160/44000 | Loss: 3.2771 | LR: 4.81e-04 | 677.4ms/step | 29.5 samples/s +Step 6170/44000 | Loss: 3.2956 | LR: 4.81e-04 | 676.4ms/step | 29.6 samples/s +Step 6180/44000 | Loss: 3.3112 | LR: 4.81e-04 | 678.4ms/step | 29.5 samples/s +Step 6190/44000 | Loss: 3.2732 | LR: 4.81e-04 | 678.4ms/step | 29.5 samples/s +Step 6200/44000 | Loss: 3.2504 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s +Step 6210/44000 | Loss: 3.2548 | LR: 4.81e-04 | 676.9ms/step | 29.5 samples/s +Step 6220/44000 | Loss: 3.2342 | LR: 4.81e-04 | 679.1ms/step | 29.4 samples/s +Step 6230/44000 | Loss: 3.2800 | LR: 4.80e-04 | 680.0ms/step | 29.4 samples/s +Step 6240/44000 | Loss: 3.2352 | LR: 4.80e-04 | 679.0ms/step | 29.5 samples/s +Step 6250/44000 | Loss: 3.2904 | LR: 4.80e-04 | 679.4ms/step | 29.4 samples/s +Validation at step 6250: Loss = 3.2793, Perplexity = 26.56 +Step 6260/44000 | Loss: 3.2713 | LR: 4.80e-04 | 1233.3ms/step | 16.2 samples/s +Step 6270/44000 | Loss: 3.2917 | LR: 4.80e-04 | 679.7ms/step | 29.4 samples/s +Step 6280/44000 | Loss: 3.2853 | LR: 4.80e-04 | 680.2ms/step | 29.4 samples/s +Step 6290/44000 | Loss: 3.2928 | LR: 4.80e-04 | 678.7ms/step | 29.5 samples/s +Step 6300/44000 | Loss: 3.2695 | LR: 4.80e-04 | 680.0ms/step | 29.4 samples/s +Step 6310/44000 | Loss: 3.3031 | LR: 4.80e-04 | 679.5ms/step | 29.4 samples/s +Step 6320/44000 | Loss: 3.2441 | LR: 4.80e-04 | 678.8ms/step | 29.5 samples/s +Step 6330/44000 | Loss: 3.3167 | LR: 4.80e-04 | 679.1ms/step | 29.5 samples/s +Step 6340/44000 | Loss: 3.2719 | LR: 4.80e-04 | 680.6ms/step | 29.4 samples/s +Step 6350/44000 | Loss: 3.3179 | LR: 4.80e-04 | 679.4ms/step | 29.4 samples/s +Step 6360/44000 | Loss: 3.2857 | LR: 4.80e-04 | 679.0ms/step | 29.5 samples/s +Step 6370/44000 | Loss: 3.2397 | LR: 4.79e-04 | 679.6ms/step | 29.4 samples/s +Step 6380/44000 | Loss: 3.2720 | LR: 4.79e-04 | 680.0ms/step | 29.4 samples/s +Step 6390/44000 | Loss: 3.2851 | LR: 4.79e-04 | 679.6ms/step | 29.4 samples/s +Step 6400/44000 | Loss: 3.2461 | LR: 4.79e-04 | 680.4ms/step | 29.4 samples/s +Step 6410/44000 | Loss: 3.2709 | LR: 4.79e-04 | 679.2ms/step | 29.4 samples/s +Step 6420/44000 | Loss: 3.2656 | LR: 4.79e-04 | 679.7ms/step | 29.4 samples/s +Step 6430/44000 | Loss: 3.2963 | LR: 4.79e-04 | 679.1ms/step | 29.4 samples/s +Step 6440/44000 | Loss: 3.2586 | LR: 4.79e-04 | 678.3ms/step | 29.5 samples/s +Step 6450/44000 | Loss: 3.2651 | LR: 4.79e-04 | 679.1ms/step | 29.5 samples/s +Step 6460/44000 | Loss: 3.2751 | LR: 4.79e-04 | 680.5ms/step | 29.4 samples/s +Step 6470/44000 | Loss: 3.2956 | LR: 4.79e-04 | 679.2ms/step | 29.4 samples/s +Step 6480/44000 | Loss: 3.3148 | LR: 4.79e-04 | 678.9ms/step | 29.5 samples/s +Step 6490/44000 | Loss: 3.2306 | LR: 4.79e-04 | 679.1ms/step | 29.4 samples/s +Step 6500/44000 | Loss: 3.3101 | LR: 4.79e-04 | 678.8ms/step | 29.5 samples/s +Validation at step 6500: Loss = 3.2681, Perplexity = 26.26 +Step 6510/44000 | Loss: 3.2772 | LR: 4.78e-04 | 1233.9ms/step | 16.2 samples/s +Step 6520/44000 | Loss: 3.2638 | LR: 4.78e-04 | 677.9ms/step | 29.5 samples/s +Step 6530/44000 | Loss: 3.2889 | LR: 4.78e-04 | 678.3ms/step | 29.5 samples/s +Step 6540/44000 | Loss: 3.2582 | LR: 4.78e-04 | 678.4ms/step | 29.5 samples/s +Step 6550/44000 | Loss: 3.2878 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s +Step 6560/44000 | Loss: 3.2331 | LR: 4.78e-04 | 678.5ms/step | 29.5 samples/s +Step 6570/44000 | Loss: 3.2258 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s +Step 6580/44000 | Loss: 3.2472 | LR: 4.78e-04 | 678.6ms/step | 29.5 samples/s +Step 6590/44000 | Loss: 3.2573 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s +Step 6600/44000 | Loss: 3.2273 | LR: 4.78e-04 | 677.9ms/step | 29.5 samples/s +Step 6610/44000 | Loss: 3.2580 | LR: 4.78e-04 | 679.4ms/step | 29.4 samples/s +Step 6620/44000 | Loss: 3.2500 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s +Step 6630/44000 | Loss: 3.2876 | LR: 4.78e-04 | 678.3ms/step | 29.5 samples/s +Step 6640/44000 | Loss: 3.2677 | LR: 4.77e-04 | 677.6ms/step | 29.5 samples/s +Step 6650/44000 | Loss: 3.2883 | LR: 4.77e-04 | 678.5ms/step | 29.5 samples/s +Step 6660/44000 | Loss: 3.2172 | LR: 4.77e-04 | 678.0ms/step | 29.5 samples/s +Step 6670/44000 | Loss: 3.2684 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s +Step 6680/44000 | Loss: 3.2422 | LR: 4.77e-04 | 678.4ms/step | 29.5 samples/s +Step 6690/44000 | Loss: 3.2733 | LR: 4.77e-04 | 678.5ms/step | 29.5 samples/s +Step 6700/44000 | Loss: 3.2605 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s +Step 6710/44000 | Loss: 3.2407 | LR: 4.77e-04 | 678.1ms/step | 29.5 samples/s +Step 6720/44000 | Loss: 3.2555 | LR: 4.77e-04 | 677.8ms/step | 29.5 samples/s +Step 6730/44000 | Loss: 3.3198 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s +Step 6740/44000 | Loss: 3.2470 | LR: 4.77e-04 | 678.2ms/step | 29.5 samples/s +Step 6750/44000 | Loss: 3.2217 | LR: 4.77e-04 | 679.2ms/step | 29.4 samples/s +Validation at step 6750: Loss = 3.2617, Perplexity = 26.10 +Step 6760/44000 | Loss: 3.2398 | LR: 4.77e-04 | 1234.5ms/step | 16.2 samples/s +Step 6770/44000 | Loss: 3.2717 | LR: 4.76e-04 | 679.8ms/step | 29.4 samples/s +Step 6780/44000 | Loss: 3.2467 | LR: 4.76e-04 | 680.6ms/step | 29.4 samples/s +Step 6790/44000 | Loss: 3.2670 | LR: 4.76e-04 | 679.7ms/step | 29.4 samples/s +Step 6800/44000 | Loss: 3.2245 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s +Step 6810/44000 | Loss: 3.2664 | LR: 4.76e-04 | 680.6ms/step | 29.4 samples/s +Step 6820/44000 | Loss: 3.2656 | LR: 4.76e-04 | 679.5ms/step | 29.4 samples/s +Step 6830/44000 | Loss: 3.2434 | LR: 4.76e-04 | 679.8ms/step | 29.4 samples/s +Step 6840/44000 | Loss: 3.2242 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s +Step 6850/44000 | Loss: 3.2333 | LR: 4.76e-04 | 678.3ms/step | 29.5 samples/s +Step 6860/44000 | Loss: 3.2091 | LR: 4.76e-04 | 678.8ms/step | 29.5 samples/s +Step 6870/44000 | Loss: 3.2832 | LR: 4.76e-04 | 679.0ms/step | 29.5 samples/s +Step 6880/44000 | Loss: 3.2373 | LR: 4.76e-04 | 679.5ms/step | 29.4 samples/s +Step 6890/44000 | Loss: 3.2338 | LR: 4.76e-04 | 678.7ms/step | 29.5 samples/s +Step 6900/44000 | Loss: 3.2429 | LR: 4.75e-04 | 681.7ms/step | 29.3 samples/s +Step 6910/44000 | Loss: 3.2843 | LR: 4.75e-04 | 679.7ms/step | 29.4 samples/s +Step 6920/44000 | Loss: 3.2883 | LR: 4.75e-04 | 678.5ms/step | 29.5 samples/s +Step 6930/44000 | Loss: 3.2477 | LR: 4.75e-04 | 680.7ms/step | 29.4 samples/s +Step 6940/44000 | Loss: 3.2472 | LR: 4.75e-04 | 680.6ms/step | 29.4 samples/s +Step 6950/44000 | Loss: 3.2425 | LR: 4.75e-04 | 680.5ms/step | 29.4 samples/s +Step 6960/44000 | Loss: 3.2633 | LR: 4.75e-04 | 678.1ms/step | 29.5 samples/s +Step 6970/44000 | Loss: 3.2067 | LR: 4.75e-04 | 679.0ms/step | 29.5 samples/s +Step 6980/44000 | Loss: 3.2616 | LR: 4.75e-04 | 678.8ms/step | 29.5 samples/s +Step 6990/44000 | Loss: 3.2119 | LR: 4.75e-04 | 679.1ms/step | 29.5 samples/s +Step 7000/44000 | Loss: 3.2409 | LR: 4.75e-04 | 678.9ms/step | 29.5 samples/s +Validation at step 7000: Loss = 3.2491, Perplexity = 25.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-7000 +Step 7010/44000 | Loss: 3.2437 | LR: 4.75e-04 | 1513.2ms/step | 13.2 samples/s +Step 7020/44000 | Loss: 3.2494 | LR: 4.75e-04 | 679.5ms/step | 29.4 samples/s +Step 7030/44000 | Loss: 3.2273 | LR: 4.74e-04 | 678.0ms/step | 29.5 samples/s +Step 7040/44000 | Loss: 3.2369 | LR: 4.74e-04 | 677.2ms/step | 29.5 samples/s +Step 7050/44000 | Loss: 3.2484 | LR: 4.74e-04 | 678.2ms/step | 29.5 samples/s +Step 7060/44000 | Loss: 3.2558 | LR: 4.74e-04 | 677.9ms/step | 29.5 samples/s +Step 7070/44000 | Loss: 3.1837 | LR: 4.74e-04 | 679.2ms/step | 29.4 samples/s +Step 7080/44000 | Loss: 3.2312 | LR: 4.74e-04 | 677.3ms/step | 29.5 samples/s +Step 7090/44000 | Loss: 3.2743 | LR: 4.74e-04 | 676.6ms/step | 29.6 samples/s +Step 7100/44000 | Loss: 3.2342 | LR: 4.74e-04 | 676.8ms/step | 29.6 samples/s +Step 7110/44000 | Loss: 3.2184 | LR: 4.74e-04 | 677.9ms/step | 29.5 samples/s +Step 7120/44000 | Loss: 3.2133 | LR: 4.74e-04 | 677.0ms/step | 29.5 samples/s +Step 7130/44000 | Loss: 3.1702 | LR: 4.74e-04 | 678.6ms/step | 29.5 samples/s +Step 7140/44000 | Loss: 3.2252 | LR: 4.74e-04 | 678.4ms/step | 29.5 samples/s +Step 7150/44000 | Loss: 3.2479 | LR: 4.73e-04 | 678.2ms/step | 29.5 samples/s +Step 7160/44000 | Loss: 3.2362 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s +Step 7170/44000 | Loss: 3.2109 | LR: 4.73e-04 | 678.4ms/step | 29.5 samples/s +Step 7180/44000 | Loss: 3.2289 | LR: 4.73e-04 | 678.7ms/step | 29.5 samples/s +Step 7190/44000 | Loss: 3.2519 | LR: 4.73e-04 | 678.4ms/step | 29.5 samples/s +Step 7200/44000 | Loss: 3.2563 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s +Step 7210/44000 | Loss: 3.2352 | LR: 4.73e-04 | 677.6ms/step | 29.5 samples/s +Step 7220/44000 | Loss: 3.2157 | LR: 4.73e-04 | 677.8ms/step | 29.5 samples/s +Step 7230/44000 | Loss: 3.2302 | LR: 4.73e-04 | 677.1ms/step | 29.5 samples/s +Step 7240/44000 | Loss: 3.2174 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s +Step 7250/44000 | Loss: 3.2539 | LR: 4.73e-04 | 677.8ms/step | 29.5 samples/s +Validation at step 7250: Loss = 3.2385, Perplexity = 25.50 +Step 7260/44000 | Loss: 3.2460 | LR: 4.73e-04 | 1233.0ms/step | 16.2 samples/s +Step 7270/44000 | Loss: 3.2156 | LR: 4.72e-04 | 678.9ms/step | 29.5 samples/s +Step 7280/44000 | Loss: 3.2339 | LR: 4.72e-04 | 679.9ms/step | 29.4 samples/s +Step 7290/44000 | Loss: 3.2530 | LR: 4.72e-04 | 680.5ms/step | 29.4 samples/s +Step 7300/44000 | Loss: 3.2496 | LR: 4.72e-04 | 680.4ms/step | 29.4 samples/s +Step 7310/44000 | Loss: 3.2499 | LR: 4.72e-04 | 679.6ms/step | 29.4 samples/s +Step 7320/44000 | Loss: 3.2096 | LR: 4.72e-04 | 680.8ms/step | 29.4 samples/s +Step 7330/44000 | Loss: 3.2435 | LR: 4.72e-04 | 679.0ms/step | 29.5 samples/s +Step 7340/44000 | Loss: 3.2338 | LR: 4.72e-04 | 680.0ms/step | 29.4 samples/s +Step 7350/44000 | Loss: 3.2175 | LR: 4.72e-04 | 680.6ms/step | 29.4 samples/s +Step 7360/44000 | Loss: 3.2012 | LR: 4.72e-04 | 679.2ms/step | 29.4 samples/s +Step 7370/44000 | Loss: 3.2560 | LR: 4.72e-04 | 678.3ms/step | 29.5 samples/s +Step 7380/44000 | Loss: 3.2539 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s +Step 7390/44000 | Loss: 3.2795 | LR: 4.71e-04 | 677.7ms/step | 29.5 samples/s +Step 7400/44000 | Loss: 3.2290 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s +Step 7410/44000 | Loss: 3.2432 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s +Step 7420/44000 | Loss: 3.2311 | LR: 4.71e-04 | 680.0ms/step | 29.4 samples/s +Step 7430/44000 | Loss: 3.2289 | LR: 4.71e-04 | 678.3ms/step | 29.5 samples/s +Step 7440/44000 | Loss: 3.2376 | LR: 4.71e-04 | 680.8ms/step | 29.4 samples/s +Step 7450/44000 | Loss: 3.2647 | LR: 4.71e-04 | 679.5ms/step | 29.4 samples/s +Step 7460/44000 | Loss: 3.2444 | LR: 4.71e-04 | 680.9ms/step | 29.4 samples/s +Step 7470/44000 | Loss: 3.2463 | LR: 4.71e-04 | 680.5ms/step | 29.4 samples/s +Step 7480/44000 | Loss: 3.2525 | LR: 4.71e-04 | 679.5ms/step | 29.4 samples/s +Step 7490/44000 | Loss: 3.2028 | LR: 4.71e-04 | 681.6ms/step | 29.3 samples/s +Step 7500/44000 | Loss: 3.1953 | LR: 4.71e-04 | 679.6ms/step | 29.4 samples/s +Validation at step 7500: Loss = 3.2309, Perplexity = 25.30 +Step 7510/44000 | Loss: 3.2101 | LR: 4.70e-04 | 1237.4ms/step | 16.2 samples/s +Step 7520/44000 | Loss: 3.2320 | LR: 4.70e-04 | 679.5ms/step | 29.4 samples/s +Step 7530/44000 | Loss: 3.2523 | LR: 4.70e-04 | 679.8ms/step | 29.4 samples/s +Step 7540/44000 | Loss: 3.2104 | LR: 4.70e-04 | 680.7ms/step | 29.4 samples/s +Step 7550/44000 | Loss: 3.2043 | LR: 4.70e-04 | 680.1ms/step | 29.4 samples/s +Step 7560/44000 | Loss: 3.2283 | LR: 4.70e-04 | 679.4ms/step | 29.4 samples/s +Step 7570/44000 | Loss: 3.1746 | LR: 4.70e-04 | 680.1ms/step | 29.4 samples/s +Step 7580/44000 | Loss: 3.1486 | LR: 4.70e-04 | 679.5ms/step | 29.4 samples/s +Step 7590/44000 | Loss: 3.2140 | LR: 4.70e-04 | 678.8ms/step | 29.5 samples/s +Step 7600/44000 | Loss: 3.2231 | LR: 4.70e-04 | 678.4ms/step | 29.5 samples/s +Step 7610/44000 | Loss: 3.2206 | LR: 4.70e-04 | 678.7ms/step | 29.5 samples/s +Step 7620/44000 | Loss: 3.2409 | LR: 4.70e-04 | 678.6ms/step | 29.5 samples/s +Step 7630/44000 | Loss: 3.2120 | LR: 4.69e-04 | 679.0ms/step | 29.5 samples/s +Step 7640/44000 | Loss: 3.1782 | LR: 4.69e-04 | 678.9ms/step | 29.5 samples/s +Step 7650/44000 | Loss: 3.1998 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s +Step 7660/44000 | Loss: 3.2252 | LR: 4.69e-04 | 679.8ms/step | 29.4 samples/s +Step 7670/44000 | Loss: 3.1973 | LR: 4.69e-04 | 678.9ms/step | 29.5 samples/s +Step 7680/44000 | Loss: 3.2567 | LR: 4.69e-04 | 679.4ms/step | 29.4 samples/s +Step 7690/44000 | Loss: 3.2256 | LR: 4.69e-04 | 680.2ms/step | 29.4 samples/s +Step 7700/44000 | Loss: 3.1972 | LR: 4.69e-04 | 680.0ms/step | 29.4 samples/s +Step 7710/44000 | Loss: 3.1891 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s +Step 7720/44000 | Loss: 3.2200 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s +Step 7730/44000 | Loss: 3.2071 | LR: 4.69e-04 | 678.8ms/step | 29.5 samples/s +Step 7740/44000 | Loss: 3.1694 | LR: 4.68e-04 | 680.0ms/step | 29.4 samples/s +Step 7750/44000 | Loss: 3.2097 | LR: 4.68e-04 | 681.5ms/step | 29.3 samples/s +Validation at step 7750: Loss = 3.2223, Perplexity = 25.09 +Step 7760/44000 | Loss: 3.2055 | LR: 4.68e-04 | 1233.8ms/step | 16.2 samples/s +Step 7770/44000 | Loss: 3.2165 | LR: 4.68e-04 | 678.0ms/step | 29.5 samples/s +Step 7780/44000 | Loss: 3.2132 | LR: 4.68e-04 | 679.0ms/step | 29.5 samples/s +Step 7790/44000 | Loss: 3.2165 | LR: 4.68e-04 | 679.5ms/step | 29.4 samples/s +Step 7800/44000 | Loss: 3.2271 | LR: 4.68e-04 | 679.9ms/step | 29.4 samples/s +Step 7810/44000 | Loss: 3.1735 | LR: 4.68e-04 | 678.1ms/step | 29.5 samples/s +Step 7820/44000 | Loss: 3.2187 | LR: 4.68e-04 | 677.3ms/step | 29.5 samples/s +Step 7830/44000 | Loss: 3.2056 | LR: 4.68e-04 | 678.4ms/step | 29.5 samples/s +Step 7840/44000 | Loss: 3.1888 | LR: 4.68e-04 | 678.7ms/step | 29.5 samples/s +Step 7850/44000 | Loss: 3.1858 | LR: 4.67e-04 | 679.5ms/step | 29.4 samples/s +Step 7860/44000 | Loss: 3.2051 | LR: 4.67e-04 | 680.9ms/step | 29.4 samples/s +Step 7870/44000 | Loss: 3.2081 | LR: 4.67e-04 | 679.9ms/step | 29.4 samples/s +Step 7880/44000 | Loss: 3.2378 | LR: 4.67e-04 | 679.6ms/step | 29.4 samples/s +Step 7890/44000 | Loss: 3.2040 | LR: 4.67e-04 | 680.0ms/step | 29.4 samples/s +Step 7900/44000 | Loss: 3.2436 | LR: 4.67e-04 | 680.0ms/step | 29.4 samples/s +Step 7910/44000 | Loss: 3.2239 | LR: 4.67e-04 | 678.6ms/step | 29.5 samples/s +Step 7920/44000 | Loss: 3.2372 | LR: 4.67e-04 | 678.4ms/step | 29.5 samples/s +Step 7930/44000 | Loss: 3.2606 | LR: 4.67e-04 | 678.1ms/step | 29.5 samples/s +Step 7940/44000 | Loss: 3.2614 | LR: 4.67e-04 | 678.3ms/step | 29.5 samples/s +Step 7950/44000 | Loss: 3.1729 | LR: 4.67e-04 | 679.1ms/step | 29.4 samples/s +Step 7960/44000 | Loss: 3.2248 | LR: 4.66e-04 | 679.1ms/step | 29.5 samples/s +Step 7970/44000 | Loss: 3.1600 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s +Step 7980/44000 | Loss: 3.1919 | LR: 4.66e-04 | 678.3ms/step | 29.5 samples/s +Step 7990/44000 | Loss: 3.2139 | LR: 4.66e-04 | 680.5ms/step | 29.4 samples/s +Step 8000/44000 | Loss: 3.2483 | LR: 4.66e-04 | 678.2ms/step | 29.5 samples/s +Validation at step 8000: Loss = 3.2159, Perplexity = 24.93 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-8000 +Step 8010/44000 | Loss: 3.2012 | LR: 4.66e-04 | 1521.3ms/step | 13.1 samples/s +Step 8020/44000 | Loss: 3.2198 | LR: 4.66e-04 | 678.7ms/step | 29.5 samples/s +Step 8030/44000 | Loss: 3.2153 | LR: 4.66e-04 | 679.2ms/step | 29.4 samples/s +Step 8040/44000 | Loss: 3.2348 | LR: 4.66e-04 | 677.0ms/step | 29.5 samples/s +Step 8050/44000 | Loss: 3.2063 | LR: 4.66e-04 | 676.9ms/step | 29.5 samples/s +Step 8060/44000 | Loss: 3.2254 | LR: 4.66e-04 | 677.9ms/step | 29.5 samples/s +Step 8070/44000 | Loss: 3.1721 | LR: 4.65e-04 | 678.9ms/step | 29.5 samples/s +Step 8080/44000 | Loss: 3.1908 | LR: 4.65e-04 | 678.1ms/step | 29.5 samples/s +Step 8090/44000 | Loss: 3.2304 | LR: 4.65e-04 | 679.3ms/step | 29.4 samples/s +Step 8100/44000 | Loss: 3.2392 | LR: 4.65e-04 | 679.7ms/step | 29.4 samples/s +Step 8110/44000 | Loss: 3.2200 | LR: 4.65e-04 | 679.5ms/step | 29.4 samples/s +Step 8120/44000 | Loss: 3.1675 | LR: 4.65e-04 | 678.8ms/step | 29.5 samples/s +W0708 22:28:09.644000 255 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 255 got signal: 15 diff --git a/epfl_final/logs/l8k365m_resampled.log b/epfl_final/logs/l8k365m_resampled.log new file mode 100644 index 0000000000000000000000000000000000000000..e23829c26622a63e7035df2e62f3c73e4f57cd38 --- /dev/null +++ b/epfl_final/logs/l8k365m_resampled.log @@ -0,0 +1,1705 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m resampled 8k nrb31", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "resampled" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32", + "sparse_mask_seed": 0, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_034748-304r0ilz +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m resampled 8k nrb31 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/304r0ilz +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0394 | LR: 6.67e-06 | 6018.3ms/step | 3.3 samples/s +Step 20/44000 | Loss: 11.9182 | LR: 1.33e-05 | 1296.0ms/step | 15.4 samples/s +Step 30/44000 | Loss: 11.5362 | LR: 2.00e-05 | 1296.3ms/step | 15.4 samples/s +Step 40/44000 | Loss: 11.0006 | LR: 2.67e-05 | 1293.7ms/step | 15.5 samples/s +Step 50/44000 | Loss: 10.7291 | LR: 3.33e-05 | 1295.3ms/step | 15.4 samples/s +Step 60/44000 | Loss: 10.4871 | LR: 4.00e-05 | 1291.6ms/step | 15.5 samples/s +Step 70/44000 | Loss: 10.2503 | LR: 4.67e-05 | 1289.9ms/step | 15.5 samples/s +Step 80/44000 | Loss: 9.9438 | LR: 5.33e-05 | 1288.7ms/step | 15.5 samples/s +Step 90/44000 | Loss: 9.5796 | LR: 6.00e-05 | 1293.2ms/step | 15.5 samples/s +Step 100/44000 | Loss: 9.1291 | LR: 6.67e-05 | 1296.7ms/step | 15.4 samples/s +Step 110/44000 | Loss: 8.5968 | LR: 7.33e-05 | 1290.5ms/step | 15.5 samples/s +Step 120/44000 | Loss: 8.1444 | LR: 8.00e-05 | 1292.3ms/step | 15.5 samples/s +Step 130/44000 | Loss: 7.7642 | LR: 8.67e-05 | 1292.7ms/step | 15.5 samples/s +Step 140/44000 | Loss: 7.5290 | LR: 9.33e-05 | 1293.6ms/step | 15.5 samples/s +Step 150/44000 | Loss: 7.3401 | LR: 1.00e-04 | 1294.1ms/step | 15.5 samples/s +Step 160/44000 | Loss: 7.2283 | LR: 1.07e-04 | 1296.6ms/step | 15.4 samples/s +Step 170/44000 | Loss: 7.1243 | LR: 1.13e-04 | 1295.9ms/step | 15.4 samples/s +Step 180/44000 | Loss: 7.0381 | LR: 1.20e-04 | 1296.2ms/step | 15.4 samples/s +Step 190/44000 | Loss: 6.9408 | LR: 1.27e-04 | 1298.7ms/step | 15.4 samples/s +Step 200/44000 | Loss: 6.8702 | LR: 1.33e-04 | 1296.1ms/step | 15.4 samples/s +Step 210/44000 | Loss: 6.7930 | LR: 1.40e-04 | 1294.5ms/step | 15.5 samples/s +Step 220/44000 | Loss: 6.6917 | LR: 1.47e-04 | 1294.8ms/step | 15.4 samples/s +Step 230/44000 | Loss: 6.5857 | LR: 1.53e-04 | 1297.1ms/step | 15.4 samples/s +Step 240/44000 | Loss: 6.5270 | LR: 1.60e-04 | 1296.0ms/step | 15.4 samples/s +Step 250/44000 | Loss: 6.4370 | LR: 1.67e-04 | 1298.0ms/step | 15.4 samples/s +Validation at step 250: Loss = 6.4365, Perplexity = 624.25 +Step 260/44000 | Loss: 6.4098 | LR: 1.73e-04 | 5544.7ms/step | 3.6 samples/s +Step 270/44000 | Loss: 6.3389 | LR: 1.80e-04 | 1303.2ms/step | 15.3 samples/s +Step 280/44000 | Loss: 6.3059 | LR: 1.87e-04 | 1301.0ms/step | 15.4 samples/s +Step 290/44000 | Loss: 6.2811 | LR: 1.93e-04 | 1301.1ms/step | 15.4 samples/s +Step 300/44000 | Loss: 6.2144 | LR: 2.00e-04 | 1302.2ms/step | 15.4 samples/s +Step 310/44000 | Loss: 6.1960 | LR: 2.07e-04 | 1306.6ms/step | 15.3 samples/s +Step 320/44000 | Loss: 6.1440 | LR: 2.13e-04 | 1307.8ms/step | 15.3 samples/s +Step 330/44000 | Loss: 6.0828 | LR: 2.20e-04 | 1308.0ms/step | 15.3 samples/s +Step 340/44000 | Loss: 6.0613 | LR: 2.27e-04 | 1306.2ms/step | 15.3 samples/s +Step 350/44000 | Loss: 5.9932 | LR: 2.33e-04 | 1305.9ms/step | 15.3 samples/s +Step 360/44000 | Loss: 5.9614 | LR: 2.40e-04 | 1304.8ms/step | 15.3 samples/s +Step 370/44000 | Loss: 5.9250 | LR: 2.47e-04 | 1304.1ms/step | 15.3 samples/s +Step 380/44000 | Loss: 5.9120 | LR: 2.53e-04 | 1306.0ms/step | 15.3 samples/s +Step 390/44000 | Loss: 5.8224 | LR: 2.60e-04 | 1307.1ms/step | 15.3 samples/s +Step 400/44000 | Loss: 5.8325 | LR: 2.67e-04 | 1303.8ms/step | 15.3 samples/s +Step 410/44000 | Loss: 5.8053 | LR: 2.73e-04 | 1304.1ms/step | 15.3 samples/s +Step 420/44000 | Loss: 5.7894 | LR: 2.80e-04 | 1304.1ms/step | 15.3 samples/s +Step 430/44000 | Loss: 5.7271 | LR: 2.87e-04 | 1306.0ms/step | 15.3 samples/s +Step 440/44000 | Loss: 5.6996 | LR: 2.93e-04 | 1306.7ms/step | 15.3 samples/s +Step 450/44000 | Loss: 5.6667 | LR: 3.00e-04 | 1305.3ms/step | 15.3 samples/s +Step 460/44000 | Loss: 5.5967 | LR: 3.07e-04 | 1307.7ms/step | 15.3 samples/s +Step 470/44000 | Loss: 5.6054 | LR: 3.13e-04 | 1305.5ms/step | 15.3 samples/s +Step 480/44000 | Loss: 5.5594 | LR: 3.20e-04 | 1304.3ms/step | 15.3 samples/s +Step 490/44000 | Loss: 5.5160 | LR: 3.27e-04 | 1307.3ms/step | 15.3 samples/s +Step 500/44000 | Loss: 5.4972 | LR: 3.33e-04 | 1306.6ms/step | 15.3 samples/s +Validation at step 500: Loss = 5.4824, Perplexity = 240.42 +Step 510/44000 | Loss: 5.4572 | LR: 3.40e-04 | 3501.9ms/step | 5.7 samples/s +Step 520/44000 | Loss: 5.4170 | LR: 3.47e-04 | 1304.8ms/step | 15.3 samples/s +Step 530/44000 | Loss: 5.4355 | LR: 3.53e-04 | 1305.2ms/step | 15.3 samples/s +Step 540/44000 | Loss: 5.3553 | LR: 3.60e-04 | 1305.6ms/step | 15.3 samples/s +Step 550/44000 | Loss: 5.3275 | LR: 3.67e-04 | 1305.4ms/step | 15.3 samples/s +Step 560/44000 | Loss: 5.3049 | LR: 3.73e-04 | 1302.6ms/step | 15.4 samples/s +Step 570/44000 | Loss: 5.3021 | LR: 3.80e-04 | 1302.8ms/step | 15.4 samples/s +Step 580/44000 | Loss: 5.2617 | LR: 3.87e-04 | 1302.6ms/step | 15.4 samples/s +Step 590/44000 | Loss: 5.2056 | LR: 3.93e-04 | 1304.8ms/step | 15.3 samples/s +Step 600/44000 | Loss: 5.2028 | LR: 4.00e-04 | 1304.0ms/step | 15.3 samples/s +Step 610/44000 | Loss: 5.1860 | LR: 4.07e-04 | 1305.9ms/step | 15.3 samples/s +Step 620/44000 | Loss: 5.1536 | LR: 4.13e-04 | 1309.8ms/step | 15.3 samples/s +Step 630/44000 | Loss: 5.0757 | LR: 4.20e-04 | 1305.3ms/step | 15.3 samples/s +Step 640/44000 | Loss: 5.0697 | LR: 4.27e-04 | 1303.5ms/step | 15.3 samples/s +Step 650/44000 | Loss: 5.0483 | LR: 4.33e-04 | 1301.1ms/step | 15.4 samples/s +Step 660/44000 | Loss: 5.0160 | LR: 4.40e-04 | 1300.6ms/step | 15.4 samples/s +Step 670/44000 | Loss: 4.9921 | LR: 4.47e-04 | 1300.8ms/step | 15.4 samples/s +Step 680/44000 | Loss: 4.9728 | LR: 4.53e-04 | 1311.6ms/step | 15.2 samples/s +Step 690/44000 | Loss: 4.9404 | LR: 4.60e-04 | 1306.2ms/step | 15.3 samples/s +Step 700/44000 | Loss: 4.9172 | LR: 4.67e-04 | 1307.4ms/step | 15.3 samples/s +Step 710/44000 | Loss: 4.8757 | LR: 4.73e-04 | 1305.5ms/step | 15.3 samples/s +Step 720/44000 | Loss: 4.8385 | LR: 4.80e-04 | 1306.1ms/step | 15.3 samples/s +Step 730/44000 | Loss: 4.8223 | LR: 4.87e-04 | 1305.2ms/step | 15.3 samples/s +Step 740/44000 | Loss: 4.7715 | LR: 4.93e-04 | 1305.5ms/step | 15.3 samples/s +Step 750/44000 | Loss: 4.7880 | LR: 5.00e-04 | 1307.3ms/step | 15.3 samples/s +Validation at step 750: Loss = 4.7761, Perplexity = 118.64 +Step 760/44000 | Loss: 4.7626 | LR: 5.00e-04 | 3503.0ms/step | 5.7 samples/s +Step 770/44000 | Loss: 4.7241 | LR: 5.00e-04 | 1306.7ms/step | 15.3 samples/s +Step 780/44000 | Loss: 4.7173 | LR: 5.00e-04 | 1302.6ms/step | 15.4 samples/s +Step 790/44000 | Loss: 4.6890 | LR: 5.00e-04 | 1304.3ms/step | 15.3 samples/s +Step 800/44000 | Loss: 4.6464 | LR: 5.00e-04 | 1304.7ms/step | 15.3 samples/s +Step 810/44000 | Loss: 4.6499 | LR: 5.00e-04 | 1305.6ms/step | 15.3 samples/s +Step 820/44000 | Loss: 4.6164 | LR: 5.00e-04 | 1301.5ms/step | 15.4 samples/s +Step 830/44000 | Loss: 4.5952 | LR: 5.00e-04 | 1301.1ms/step | 15.4 samples/s +Step 840/44000 | Loss: 4.5631 | LR: 5.00e-04 | 1300.3ms/step | 15.4 samples/s +Step 850/44000 | Loss: 4.5635 | LR: 5.00e-04 | 1300.6ms/step | 15.4 samples/s +Step 860/44000 | Loss: 4.5845 | LR: 5.00e-04 | 1303.4ms/step | 15.3 samples/s +Step 870/44000 | Loss: 4.5055 | LR: 5.00e-04 | 1301.9ms/step | 15.4 samples/s +Step 880/44000 | Loss: 4.5109 | LR: 5.00e-04 | 1302.3ms/step | 15.4 samples/s +Step 890/44000 | Loss: 4.4857 | LR: 5.00e-04 | 1301.8ms/step | 15.4 samples/s +Step 900/44000 | Loss: 4.4629 | LR: 5.00e-04 | 1302.8ms/step | 15.4 samples/s +Step 910/44000 | Loss: 4.4368 | LR: 5.00e-04 | 1302.1ms/step | 15.4 samples/s +Step 920/44000 | Loss: 4.4401 | LR: 5.00e-04 | 1302.2ms/step | 15.4 samples/s +Step 930/44000 | Loss: 4.4125 | LR: 5.00e-04 | 1299.9ms/step | 15.4 samples/s +Step 940/44000 | Loss: 4.4005 | LR: 5.00e-04 | 1299.4ms/step | 15.4 samples/s +Step 950/44000 | Loss: 4.3923 | LR: 5.00e-04 | 1300.1ms/step | 15.4 samples/s +Step 960/44000 | Loss: 4.3524 | LR: 5.00e-04 | 1305.7ms/step | 15.3 samples/s +Step 970/44000 | Loss: 4.3809 | LR: 5.00e-04 | 1303.2ms/step | 15.3 samples/s +Step 980/44000 | Loss: 4.3615 | LR: 5.00e-04 | 1304.8ms/step | 15.3 samples/s +Step 990/44000 | Loss: 4.3383 | LR: 5.00e-04 | 1303.5ms/step | 15.3 samples/s +Step 1000/44000 | Loss: 4.3341 | LR: 5.00e-04 | 1305.0ms/step | 15.3 samples/s +Validation at step 1000: Loss = 4.3523, Perplexity = 77.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-1000 +Step 1010/44000 | Loss: 4.2992 | LR: 5.00e-04 | 3796.5ms/step | 5.3 samples/s +Step 1020/44000 | Loss: 4.3026 | LR: 5.00e-04 | 1303.6ms/step | 15.3 samples/s +Step 1030/44000 | Loss: 4.2519 | LR: 5.00e-04 | 1302.3ms/step | 15.4 samples/s +Step 1040/44000 | Loss: 4.2625 | LR: 5.00e-04 | 1304.7ms/step | 15.3 samples/s +Step 1050/44000 | Loss: 4.2617 | LR: 5.00e-04 | 1303.0ms/step | 15.3 samples/s +Step 1060/44000 | Loss: 4.2519 | LR: 5.00e-04 | 1302.0ms/step | 15.4 samples/s +Step 1070/44000 | Loss: 4.2348 | LR: 5.00e-04 | 1303.2ms/step | 15.3 samples/s +Step 1080/44000 | Loss: 4.2245 | LR: 5.00e-04 | 1301.5ms/step | 15.4 samples/s +Step 1090/44000 | Loss: 4.2434 | LR: 5.00e-04 | 1300.6ms/step | 15.4 samples/s +Step 1100/44000 | Loss: 4.2196 | LR: 5.00e-04 | 1300.3ms/step | 15.4 samples/s +Step 1110/44000 | Loss: 4.2025 | LR: 5.00e-04 | 1301.5ms/step | 15.4 samples/s +Step 1120/44000 | Loss: 4.1843 | LR: 5.00e-04 | 1300.7ms/step | 15.4 samples/s +Step 1130/44000 | Loss: 4.1542 | LR: 5.00e-04 | 1300.2ms/step | 15.4 samples/s +Step 1140/44000 | Loss: 4.1795 | LR: 5.00e-04 | 1301.7ms/step | 15.4 samples/s +Step 1150/44000 | Loss: 4.1302 | LR: 5.00e-04 | 1299.4ms/step | 15.4 samples/s +Step 1160/44000 | Loss: 4.1348 | LR: 5.00e-04 | 1299.9ms/step | 15.4 samples/s +Step 1170/44000 | Loss: 4.1389 | LR: 5.00e-04 | 1299.7ms/step | 15.4 samples/s +Step 1180/44000 | Loss: 4.1184 | LR: 5.00e-04 | 1298.7ms/step | 15.4 samples/s +Step 1190/44000 | Loss: 4.1388 | LR: 5.00e-04 | 1302.7ms/step | 15.4 samples/s +Step 1200/44000 | Loss: 4.1388 | LR: 5.00e-04 | 1304.5ms/step | 15.3 samples/s +Step 1210/44000 | Loss: 4.1427 | LR: 5.00e-04 | 1305.3ms/step | 15.3 samples/s +Step 1220/44000 | Loss: 4.0960 | LR: 5.00e-04 | 1305.6ms/step | 15.3 samples/s +Step 1230/44000 | Loss: 4.0841 | LR: 5.00e-04 | 1302.3ms/step | 15.4 samples/s +Step 1240/44000 | Loss: 4.0851 | LR: 5.00e-04 | 1304.0ms/step | 15.3 samples/s +Step 1250/44000 | Loss: 4.0926 | LR: 5.00e-04 | 1304.5ms/step | 15.3 samples/s +Validation at step 1250: Loss = 4.0873, Perplexity = 59.58 +Step 1260/44000 | Loss: 4.0612 | LR: 5.00e-04 | 3495.1ms/step | 5.7 samples/s +Step 1270/44000 | Loss: 4.0596 | LR: 5.00e-04 | 1301.5ms/step | 15.4 samples/s +Step 1280/44000 | Loss: 4.0679 | LR: 5.00e-04 | 1302.0ms/step | 15.4 samples/s +Step 1290/44000 | Loss: 4.0178 | LR: 5.00e-04 | 1303.7ms/step | 15.3 samples/s +Step 1300/44000 | Loss: 4.0686 | LR: 5.00e-04 | 1302.6ms/step | 15.4 samples/s +Step 1310/44000 | Loss: 4.0080 | LR: 5.00e-04 | 1304.3ms/step | 15.3 samples/s +Step 1320/44000 | Loss: 4.0348 | LR: 5.00e-04 | 1302.1ms/step | 15.4 samples/s +Step 1330/44000 | Loss: 4.0399 | LR: 5.00e-04 | 1304.1ms/step | 15.3 samples/s +Step 1340/44000 | Loss: 4.0010 | LR: 5.00e-04 | 1304.2ms/step | 15.3 samples/s +Step 1350/44000 | Loss: 4.0022 | LR: 5.00e-04 | 1303.8ms/step | 15.3 samples/s +Step 1360/44000 | Loss: 3.9970 | LR: 5.00e-04 | 1303.1ms/step | 15.3 samples/s +Step 1370/44000 | Loss: 3.9738 | LR: 5.00e-04 | 1299.9ms/step | 15.4 samples/s +Step 1380/44000 | Loss: 3.9861 | LR: 5.00e-04 | 1301.8ms/step | 15.4 samples/s +Step 1390/44000 | Loss: 3.9894 | LR: 5.00e-04 | 1300.4ms/step | 15.4 samples/s +Step 1400/44000 | Loss: 3.9836 | LR: 5.00e-04 | 1299.8ms/step | 15.4 samples/s +Step 1410/44000 | Loss: 3.9765 | LR: 5.00e-04 | 1299.5ms/step | 15.4 samples/s +Step 1420/44000 | Loss: 3.9671 | LR: 5.00e-04 | 1308.2ms/step | 15.3 samples/s +Step 1430/44000 | Loss: 3.9829 | LR: 5.00e-04 | 1312.2ms/step | 15.2 samples/s +Step 1440/44000 | Loss: 3.9790 | LR: 5.00e-04 | 1308.7ms/step | 15.3 samples/s +Step 1450/44000 | Loss: 3.9631 | LR: 5.00e-04 | 1308.4ms/step | 15.3 samples/s +Step 1460/44000 | Loss: 3.9595 | LR: 5.00e-04 | 1311.9ms/step | 15.2 samples/s +Step 1470/44000 | Loss: 3.9698 | LR: 5.00e-04 | 1313.0ms/step | 15.2 samples/s +Step 1480/44000 | Loss: 3.9652 | LR: 5.00e-04 | 1303.7ms/step | 15.3 samples/s +Step 1490/44000 | Loss: 3.9621 | LR: 5.00e-04 | 1309.7ms/step | 15.3 samples/s +Step 1500/44000 | Loss: 3.9361 | LR: 5.00e-04 | 1304.1ms/step | 15.3 samples/s +Validation at step 1500: Loss = 3.9347, Perplexity = 51.15 +Step 1510/44000 | Loss: 3.9502 | LR: 5.00e-04 | 3493.3ms/step | 5.7 samples/s +Step 1520/44000 | Loss: 3.9492 | LR: 5.00e-04 | 1307.8ms/step | 15.3 samples/s +Step 1530/44000 | Loss: 3.9173 | LR: 5.00e-04 | 1305.9ms/step | 15.3 samples/s +Step 1540/44000 | Loss: 3.9269 | LR: 5.00e-04 | 1299.9ms/step | 15.4 samples/s +Step 1550/44000 | Loss: 3.8909 | LR: 5.00e-04 | 1301.0ms/step | 15.4 samples/s +Step 1560/44000 | Loss: 3.8813 | LR: 5.00e-04 | 1299.9ms/step | 15.4 samples/s +Step 1570/44000 | Loss: 3.8770 | LR: 5.00e-04 | 1301.6ms/step | 15.4 samples/s +Step 1580/44000 | Loss: 3.8985 | LR: 5.00e-04 | 1299.6ms/step | 15.4 samples/s +Step 1590/44000 | Loss: 3.8710 | LR: 5.00e-04 | 1301.0ms/step | 15.4 samples/s +Step 1600/44000 | Loss: 3.8777 | LR: 5.00e-04 | 1301.0ms/step | 15.4 samples/s +Step 1610/44000 | Loss: 3.8562 | LR: 5.00e-04 | 1299.8ms/step | 15.4 samples/s +Step 1620/44000 | Loss: 3.8420 | LR: 5.00e-04 | 1301.2ms/step | 15.4 samples/s +Step 1630/44000 | Loss: 3.8559 | LR: 4.99e-04 | 1305.0ms/step | 15.3 samples/s +Step 1640/44000 | Loss: 3.8600 | LR: 4.99e-04 | 1302.1ms/step | 15.4 samples/s +Step 1650/44000 | Loss: 3.8347 | LR: 4.99e-04 | 1301.5ms/step | 15.4 samples/s +Step 1660/44000 | Loss: 3.8391 | LR: 4.99e-04 | 1301.9ms/step | 15.4 samples/s +Step 1670/44000 | Loss: 3.8325 | LR: 4.99e-04 | 1303.9ms/step | 15.3 samples/s +Step 1680/44000 | Loss: 3.8370 | LR: 4.99e-04 | 1303.3ms/step | 15.3 samples/s +Step 1690/44000 | Loss: 3.8230 | LR: 4.99e-04 | 1301.4ms/step | 15.4 samples/s +Step 1700/44000 | Loss: 3.8620 | LR: 4.99e-04 | 1301.8ms/step | 15.4 samples/s +Step 1710/44000 | Loss: 3.8084 | LR: 4.99e-04 | 1301.1ms/step | 15.4 samples/s +Step 1720/44000 | Loss: 3.8206 | LR: 4.99e-04 | 1300.8ms/step | 15.4 samples/s +Step 1730/44000 | Loss: 3.8222 | LR: 4.99e-04 | 1306.9ms/step | 15.3 samples/s +Step 1740/44000 | Loss: 3.7861 | LR: 4.99e-04 | 1309.7ms/step | 15.3 samples/s +Step 1750/44000 | Loss: 3.8061 | LR: 4.99e-04 | 1309.3ms/step | 15.3 samples/s +Validation at step 1750: Loss = 3.8231, Perplexity = 45.74 +Step 1760/44000 | Loss: 3.7921 | LR: 4.99e-04 | 3505.5ms/step | 5.7 samples/s +Step 1770/44000 | Loss: 3.7879 | LR: 4.99e-04 | 1299.8ms/step | 15.4 samples/s +Step 1780/44000 | Loss: 3.8250 | LR: 4.99e-04 | 1298.6ms/step | 15.4 samples/s +Step 1790/44000 | Loss: 3.7882 | LR: 4.99e-04 | 1299.9ms/step | 15.4 samples/s +Step 1800/44000 | Loss: 3.7880 | LR: 4.99e-04 | 1300.9ms/step | 15.4 samples/s +Step 1810/44000 | Loss: 3.8032 | LR: 4.99e-04 | 1300.0ms/step | 15.4 samples/s +Step 1820/44000 | Loss: 3.7935 | LR: 4.99e-04 | 1299.6ms/step | 15.4 samples/s +Step 1830/44000 | Loss: 3.7805 | LR: 4.99e-04 | 1302.2ms/step | 15.4 samples/s +Step 1840/44000 | Loss: 3.7675 | LR: 4.99e-04 | 1297.9ms/step | 15.4 samples/s +Step 1850/44000 | Loss: 3.7939 | LR: 4.99e-04 | 1301.1ms/step | 15.4 samples/s +Step 1860/44000 | Loss: 3.7784 | LR: 4.99e-04 | 1302.5ms/step | 15.4 samples/s +Step 1870/44000 | Loss: 3.7820 | LR: 4.99e-04 | 1301.6ms/step | 15.4 samples/s +Step 1880/44000 | Loss: 3.7820 | LR: 4.99e-04 | 1300.9ms/step | 15.4 samples/s +Step 1890/44000 | Loss: 3.7743 | LR: 4.99e-04 | 1299.7ms/step | 15.4 samples/s +Step 1900/44000 | Loss: 3.7810 | LR: 4.99e-04 | 1301.9ms/step | 15.4 samples/s +Step 1910/44000 | Loss: 3.7546 | LR: 4.99e-04 | 1300.0ms/step | 15.4 samples/s +Step 1920/44000 | Loss: 3.7746 | LR: 4.99e-04 | 1298.1ms/step | 15.4 samples/s +Step 1930/44000 | Loss: 3.7647 | LR: 4.99e-04 | 1302.1ms/step | 15.4 samples/s +Step 1940/44000 | Loss: 3.7615 | LR: 4.99e-04 | 1309.6ms/step | 15.3 samples/s +Step 1950/44000 | Loss: 3.7595 | LR: 4.99e-04 | 1309.3ms/step | 15.3 samples/s +Step 1960/44000 | Loss: 3.7440 | LR: 4.99e-04 | 1303.3ms/step | 15.3 samples/s +Step 1970/44000 | Loss: 3.7497 | LR: 4.99e-04 | 1301.6ms/step | 15.4 samples/s +Step 1980/44000 | Loss: 3.7259 | LR: 4.99e-04 | 1301.1ms/step | 15.4 samples/s +Step 1990/44000 | Loss: 3.7090 | LR: 4.99e-04 | 1301.8ms/step | 15.4 samples/s +Step 2000/44000 | Loss: 3.7207 | LR: 4.99e-04 | 1302.8ms/step | 15.4 samples/s +Validation at step 2000: Loss = 3.7390, Perplexity = 42.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-2000 +Step 2010/44000 | Loss: 3.6986 | LR: 4.99e-04 | 3798.4ms/step | 5.3 samples/s +Step 2020/44000 | Loss: 3.7567 | LR: 4.99e-04 | 1299.5ms/step | 15.4 samples/s +Step 2030/44000 | Loss: 3.7322 | LR: 4.99e-04 | 1302.0ms/step | 15.4 samples/s +Step 2040/44000 | Loss: 3.7175 | LR: 4.99e-04 | 1305.0ms/step | 15.3 samples/s +Step 2050/44000 | Loss: 3.7083 | LR: 4.99e-04 | 1311.3ms/step | 15.3 samples/s +Step 2060/44000 | Loss: 3.6795 | LR: 4.99e-04 | 1307.5ms/step | 15.3 samples/s +Step 2070/44000 | Loss: 3.6994 | LR: 4.99e-04 | 1303.7ms/step | 15.3 samples/s +Step 2080/44000 | Loss: 3.6699 | LR: 4.99e-04 | 1301.4ms/step | 15.4 samples/s +Step 2090/44000 | Loss: 3.7091 | LR: 4.99e-04 | 1305.0ms/step | 15.3 samples/s +Step 2100/44000 | Loss: 3.6997 | LR: 4.99e-04 | 1298.1ms/step | 15.4 samples/s +Step 2110/44000 | Loss: 3.7174 | LR: 4.99e-04 | 1297.8ms/step | 15.4 samples/s +Step 2120/44000 | Loss: 3.7044 | LR: 4.99e-04 | 1298.0ms/step | 15.4 samples/s +Step 2130/44000 | Loss: 3.7129 | LR: 4.99e-04 | 1298.5ms/step | 15.4 samples/s +Step 2140/44000 | Loss: 3.6638 | LR: 4.99e-04 | 1298.3ms/step | 15.4 samples/s +Step 2150/44000 | Loss: 3.6773 | LR: 4.99e-04 | 1298.5ms/step | 15.4 samples/s +Step 2160/44000 | Loss: 3.6851 | LR: 4.99e-04 | 1298.6ms/step | 15.4 samples/s +Step 2170/44000 | Loss: 3.6856 | LR: 4.99e-04 | 1297.0ms/step | 15.4 samples/s +Step 2180/44000 | Loss: 3.6707 | LR: 4.99e-04 | 1298.4ms/step | 15.4 samples/s +Step 2190/44000 | Loss: 3.6805 | LR: 4.99e-04 | 1299.0ms/step | 15.4 samples/s +Step 2200/44000 | Loss: 3.6548 | LR: 4.99e-04 | 1300.9ms/step | 15.4 samples/s +Step 2210/44000 | Loss: 3.6573 | LR: 4.99e-04 | 1298.9ms/step | 15.4 samples/s +Step 2220/44000 | Loss: 3.6379 | LR: 4.99e-04 | 1300.0ms/step | 15.4 samples/s +Step 2230/44000 | Loss: 3.6800 | LR: 4.99e-04 | 1300.8ms/step | 15.4 samples/s +Step 2240/44000 | Loss: 3.6600 | LR: 4.99e-04 | 1300.3ms/step | 15.4 samples/s +Step 2250/44000 | Loss: 3.6617 | LR: 4.99e-04 | 1300.5ms/step | 15.4 samples/s +Validation at step 2250: Loss = 3.6727, Perplexity = 39.36 +Step 2260/44000 | Loss: 3.6584 | LR: 4.98e-04 | 3486.0ms/step | 5.7 samples/s +Step 2270/44000 | Loss: 3.6418 | LR: 4.98e-04 | 1301.5ms/step | 15.4 samples/s +Step 2280/44000 | Loss: 3.6702 | LR: 4.98e-04 | 1302.0ms/step | 15.4 samples/s +Step 2290/44000 | Loss: 3.6616 | LR: 4.98e-04 | 1301.8ms/step | 15.4 samples/s +Step 2300/44000 | Loss: 3.6391 | LR: 4.98e-04 | 1301.8ms/step | 15.4 samples/s +Step 2310/44000 | Loss: 3.6544 | LR: 4.98e-04 | 1304.0ms/step | 15.3 samples/s +Step 2320/44000 | Loss: 3.6417 | LR: 4.98e-04 | 1302.6ms/step | 15.4 samples/s +Step 2330/44000 | Loss: 3.6450 | LR: 4.98e-04 | 1305.6ms/step | 15.3 samples/s +Step 2340/44000 | Loss: 3.6210 | LR: 4.98e-04 | 1303.8ms/step | 15.3 samples/s +Step 2350/44000 | Loss: 3.6311 | LR: 4.98e-04 | 1304.1ms/step | 15.3 samples/s +Step 2360/44000 | Loss: 3.6147 | LR: 4.98e-04 | 1302.3ms/step | 15.4 samples/s +Step 2370/44000 | Loss: 3.6559 | LR: 4.98e-04 | 1303.1ms/step | 15.3 samples/s +Step 2380/44000 | Loss: 3.6332 | LR: 4.98e-04 | 1302.9ms/step | 15.4 samples/s +Step 2390/44000 | Loss: 3.5959 | LR: 4.98e-04 | 1299.5ms/step | 15.4 samples/s +Step 2400/44000 | Loss: 3.6098 | LR: 4.98e-04 | 1298.2ms/step | 15.4 samples/s +Step 2410/44000 | Loss: 3.6532 | LR: 4.98e-04 | 1298.1ms/step | 15.4 samples/s +Step 2420/44000 | Loss: 3.6605 | LR: 4.98e-04 | 1301.1ms/step | 15.4 samples/s +Step 2430/44000 | Loss: 3.6240 | LR: 4.98e-04 | 1301.4ms/step | 15.4 samples/s +Step 2440/44000 | Loss: 3.6022 | LR: 4.98e-04 | 1301.2ms/step | 15.4 samples/s +Step 2450/44000 | Loss: 3.6284 | LR: 4.98e-04 | 1302.0ms/step | 15.4 samples/s +Step 2460/44000 | Loss: 3.6235 | LR: 4.98e-04 | 1303.0ms/step | 15.3 samples/s +Step 2470/44000 | Loss: 3.6022 | LR: 4.98e-04 | 1303.6ms/step | 15.3 samples/s +Step 2480/44000 | Loss: 3.6327 | LR: 4.98e-04 | 1301.4ms/step | 15.4 samples/s +Step 2490/44000 | Loss: 3.5885 | LR: 4.98e-04 | 1299.8ms/step | 15.4 samples/s +Step 2500/44000 | Loss: 3.6047 | LR: 4.98e-04 | 1300.1ms/step | 15.4 samples/s +Validation at step 2500: Loss = 3.6213, Perplexity = 37.39 +Step 2510/44000 | Loss: 3.5889 | LR: 4.98e-04 | 3476.5ms/step | 5.8 samples/s +Step 2520/44000 | Loss: 3.6069 | LR: 4.98e-04 | 1301.3ms/step | 15.4 samples/s +Step 2530/44000 | Loss: 3.6066 | LR: 4.98e-04 | 1300.5ms/step | 15.4 samples/s +Step 2540/44000 | Loss: 3.6182 | LR: 4.98e-04 | 1299.2ms/step | 15.4 samples/s +Step 2550/44000 | Loss: 3.6072 | LR: 4.98e-04 | 1297.3ms/step | 15.4 samples/s +Step 2560/44000 | Loss: 3.6190 | LR: 4.98e-04 | 1297.2ms/step | 15.4 samples/s +Step 2570/44000 | Loss: 3.5925 | LR: 4.98e-04 | 1297.3ms/step | 15.4 samples/s +Step 2580/44000 | Loss: 3.6284 | LR: 4.98e-04 | 1295.1ms/step | 15.4 samples/s +Step 2590/44000 | Loss: 3.5843 | LR: 4.98e-04 | 1295.4ms/step | 15.4 samples/s +Step 2600/44000 | Loss: 3.5903 | LR: 4.98e-04 | 1299.5ms/step | 15.4 samples/s +Step 2610/44000 | Loss: 3.5858 | LR: 4.98e-04 | 1299.4ms/step | 15.4 samples/s +Step 2620/44000 | Loss: 3.5885 | LR: 4.98e-04 | 1299.5ms/step | 15.4 samples/s +Step 2630/44000 | Loss: 3.5801 | LR: 4.98e-04 | 1301.3ms/step | 15.4 samples/s +Step 2640/44000 | Loss: 3.5927 | LR: 4.98e-04 | 1301.1ms/step | 15.4 samples/s +Step 2650/44000 | Loss: 3.6111 | LR: 4.98e-04 | 1301.7ms/step | 15.4 samples/s +Step 2660/44000 | Loss: 3.5795 | LR: 4.98e-04 | 1304.5ms/step | 15.3 samples/s +Step 2670/44000 | Loss: 3.5705 | LR: 4.98e-04 | 1301.4ms/step | 15.4 samples/s +Step 2680/44000 | Loss: 3.5912 | LR: 4.98e-04 | 1299.5ms/step | 15.4 samples/s +Step 2690/44000 | Loss: 3.5801 | LR: 4.98e-04 | 1299.3ms/step | 15.4 samples/s +Step 2700/44000 | Loss: 3.6137 | LR: 4.97e-04 | 1298.5ms/step | 15.4 samples/s +Step 2710/44000 | Loss: 3.5794 | LR: 4.97e-04 | 1298.4ms/step | 15.4 samples/s +Step 2720/44000 | Loss: 3.5767 | LR: 4.97e-04 | 1298.3ms/step | 15.4 samples/s +Step 2730/44000 | Loss: 3.5742 | LR: 4.97e-04 | 1299.1ms/step | 15.4 samples/s +Step 2740/44000 | Loss: 3.5737 | LR: 4.97e-04 | 1301.0ms/step | 15.4 samples/s +Step 2750/44000 | Loss: 3.5625 | LR: 4.97e-04 | 1305.3ms/step | 15.3 samples/s +Validation at step 2750: Loss = 3.5776, Perplexity = 35.79 +Step 2760/44000 | Loss: 3.5352 | LR: 4.97e-04 | 3493.0ms/step | 5.7 samples/s +Step 2770/44000 | Loss: 3.5616 | LR: 4.97e-04 | 1303.8ms/step | 15.3 samples/s +Step 2780/44000 | Loss: 3.5375 | LR: 4.97e-04 | 1302.5ms/step | 15.4 samples/s +Step 2790/44000 | Loss: 3.5645 | LR: 4.97e-04 | 1301.5ms/step | 15.4 samples/s +Step 2800/44000 | Loss: 3.5564 | LR: 4.97e-04 | 1301.6ms/step | 15.4 samples/s +Step 2810/44000 | Loss: 3.5408 | LR: 4.97e-04 | 1301.9ms/step | 15.4 samples/s +Step 2820/44000 | Loss: 3.5510 | LR: 4.97e-04 | 1302.5ms/step | 15.4 samples/s +Step 2830/44000 | Loss: 3.5272 | LR: 4.97e-04 | 1301.9ms/step | 15.4 samples/s +Step 2840/44000 | Loss: 3.5366 | LR: 4.97e-04 | 1302.3ms/step | 15.4 samples/s +Step 2850/44000 | Loss: 3.5437 | LR: 4.97e-04 | 1301.4ms/step | 15.4 samples/s +Step 2860/44000 | Loss: 3.5401 | LR: 4.97e-04 | 1302.8ms/step | 15.4 samples/s +Step 2870/44000 | Loss: 3.5493 | LR: 4.97e-04 | 1302.7ms/step | 15.4 samples/s +Step 2880/44000 | Loss: 3.5114 | LR: 4.97e-04 | 1303.5ms/step | 15.3 samples/s +Step 2890/44000 | Loss: 3.5564 | LR: 4.97e-04 | 1303.1ms/step | 15.3 samples/s +Step 2900/44000 | Loss: 3.5539 | LR: 4.97e-04 | 1304.4ms/step | 15.3 samples/s +Step 2910/44000 | Loss: 3.5259 | LR: 4.97e-04 | 1300.8ms/step | 15.4 samples/s +Step 2920/44000 | Loss: 3.5139 | LR: 4.97e-04 | 1302.3ms/step | 15.4 samples/s +Step 2930/44000 | Loss: 3.5096 | LR: 4.97e-04 | 1301.7ms/step | 15.4 samples/s +Step 2940/44000 | Loss: 3.5324 | LR: 4.97e-04 | 1302.2ms/step | 15.4 samples/s +Step 2950/44000 | Loss: 3.5316 | LR: 4.97e-04 | 1300.6ms/step | 15.4 samples/s +Step 2960/44000 | Loss: 3.5487 | LR: 4.97e-04 | 1299.5ms/step | 15.4 samples/s +Step 2970/44000 | Loss: 3.5300 | LR: 4.97e-04 | 1300.1ms/step | 15.4 samples/s +Step 2980/44000 | Loss: 3.5309 | LR: 4.97e-04 | 1300.3ms/step | 15.4 samples/s +Step 2990/44000 | Loss: 3.5213 | LR: 4.97e-04 | 1299.8ms/step | 15.4 samples/s +Step 3000/44000 | Loss: 3.5516 | LR: 4.97e-04 | 1299.9ms/step | 15.4 samples/s +Validation at step 3000: Loss = 3.5383, Perplexity = 34.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-3000 +Step 3010/44000 | Loss: 3.5302 | LR: 4.97e-04 | 3778.5ms/step | 5.3 samples/s +Step 3020/44000 | Loss: 3.5098 | LR: 4.97e-04 | 1297.1ms/step | 15.4 samples/s +Step 3030/44000 | Loss: 3.5294 | LR: 4.97e-04 | 1300.0ms/step | 15.4 samples/s +Step 3040/44000 | Loss: 3.4909 | LR: 4.97e-04 | 1298.7ms/step | 15.4 samples/s +Step 3050/44000 | Loss: 3.5167 | LR: 4.97e-04 | 1299.0ms/step | 15.4 samples/s +Step 3060/44000 | Loss: 3.4880 | LR: 4.96e-04 | 1298.8ms/step | 15.4 samples/s +Step 3070/44000 | Loss: 3.5340 | LR: 4.96e-04 | 1297.5ms/step | 15.4 samples/s +Step 3080/44000 | Loss: 3.5547 | LR: 4.96e-04 | 1296.9ms/step | 15.4 samples/s +Step 3090/44000 | Loss: 3.5431 | LR: 4.96e-04 | 1299.3ms/step | 15.4 samples/s +Step 3100/44000 | Loss: 3.5031 | LR: 4.96e-04 | 1297.9ms/step | 15.4 samples/s +Step 3110/44000 | Loss: 3.5054 | LR: 4.96e-04 | 1300.8ms/step | 15.4 samples/s +Step 3120/44000 | Loss: 3.5247 | LR: 4.96e-04 | 1301.5ms/step | 15.4 samples/s +Step 3130/44000 | Loss: 3.4931 | LR: 4.96e-04 | 1300.4ms/step | 15.4 samples/s +Step 3140/44000 | Loss: 3.5109 | LR: 4.96e-04 | 1300.4ms/step | 15.4 samples/s +Step 3150/44000 | Loss: 3.5112 | LR: 4.96e-04 | 1299.9ms/step | 15.4 samples/s +Step 3160/44000 | Loss: 3.5316 | LR: 4.96e-04 | 1300.3ms/step | 15.4 samples/s +Step 3170/44000 | Loss: 3.4703 | LR: 4.96e-04 | 1299.0ms/step | 15.4 samples/s +Step 3180/44000 | Loss: 3.5102 | LR: 4.96e-04 | 1297.9ms/step | 15.4 samples/s +Step 3190/44000 | Loss: 3.4988 | LR: 4.96e-04 | 1298.4ms/step | 15.4 samples/s +Step 3200/44000 | Loss: 3.5242 | LR: 4.96e-04 | 1297.9ms/step | 15.4 samples/s +Step 3210/44000 | Loss: 3.4976 | LR: 4.96e-04 | 1303.1ms/step | 15.3 samples/s +Step 3220/44000 | Loss: 3.4995 | LR: 4.96e-04 | 1300.4ms/step | 15.4 samples/s +Step 3230/44000 | Loss: 3.4976 | LR: 4.96e-04 | 1301.3ms/step | 15.4 samples/s +Step 3240/44000 | Loss: 3.4960 | LR: 4.96e-04 | 1301.3ms/step | 15.4 samples/s +Step 3250/44000 | Loss: 3.4976 | LR: 4.96e-04 | 1306.1ms/step | 15.3 samples/s +Validation at step 3250: Loss = 3.5042, Perplexity = 33.26 +Step 3260/44000 | Loss: 3.5165 | LR: 4.96e-04 | 3479.9ms/step | 5.7 samples/s +Step 3270/44000 | Loss: 3.4817 | LR: 4.96e-04 | 1302.1ms/step | 15.4 samples/s +Step 3280/44000 | Loss: 3.4703 | LR: 4.96e-04 | 1299.1ms/step | 15.4 samples/s +Step 3290/44000 | Loss: 3.4901 | LR: 4.96e-04 | 1299.2ms/step | 15.4 samples/s +Step 3300/44000 | Loss: 3.5185 | LR: 4.96e-04 | 1298.4ms/step | 15.4 samples/s +Step 3310/44000 | Loss: 3.4926 | LR: 4.96e-04 | 1297.0ms/step | 15.4 samples/s +Step 3320/44000 | Loss: 3.5154 | LR: 4.96e-04 | 1301.7ms/step | 15.4 samples/s +Step 3330/44000 | Loss: 3.5156 | LR: 4.96e-04 | 1303.6ms/step | 15.3 samples/s +Step 3340/44000 | Loss: 3.4758 | LR: 4.96e-04 | 1302.0ms/step | 15.4 samples/s +Step 3350/44000 | Loss: 3.4878 | LR: 4.96e-04 | 1303.0ms/step | 15.3 samples/s +Step 3360/44000 | Loss: 3.4699 | LR: 4.96e-04 | 1300.7ms/step | 15.4 samples/s +Step 3370/44000 | Loss: 3.4436 | LR: 4.95e-04 | 1300.1ms/step | 15.4 samples/s +Step 3380/44000 | Loss: 3.4912 | LR: 4.95e-04 | 1297.2ms/step | 15.4 samples/s +Step 3390/44000 | Loss: 3.4703 | LR: 4.95e-04 | 1300.5ms/step | 15.4 samples/s +Step 3400/44000 | Loss: 3.5017 | LR: 4.95e-04 | 1301.2ms/step | 15.4 samples/s +Step 3410/44000 | Loss: 3.4775 | LR: 4.95e-04 | 1300.3ms/step | 15.4 samples/s +Step 3420/44000 | Loss: 3.4864 | LR: 4.95e-04 | 1301.3ms/step | 15.4 samples/s +Step 3430/44000 | Loss: 3.4779 | LR: 4.95e-04 | 1300.6ms/step | 15.4 samples/s +Step 3440/44000 | Loss: 3.4699 | LR: 4.95e-04 | 1301.3ms/step | 15.4 samples/s +Step 3450/44000 | Loss: 3.4549 | LR: 4.95e-04 | 1300.6ms/step | 15.4 samples/s +Step 3460/44000 | Loss: 3.4917 | LR: 4.95e-04 | 1299.9ms/step | 15.4 samples/s +Step 3470/44000 | Loss: 3.4649 | LR: 4.95e-04 | 1300.2ms/step | 15.4 samples/s +Step 3480/44000 | Loss: 3.5280 | LR: 4.95e-04 | 1300.1ms/step | 15.4 samples/s +Step 3490/44000 | Loss: 3.4460 | LR: 4.95e-04 | 1299.8ms/step | 15.4 samples/s +Step 3500/44000 | Loss: 3.4499 | LR: 4.95e-04 | 1300.3ms/step | 15.4 samples/s +Validation at step 3500: Loss = 3.4776, Perplexity = 32.38 +Step 3510/44000 | Loss: 3.4827 | LR: 4.95e-04 | 3480.1ms/step | 5.7 samples/s +Step 3520/44000 | Loss: 3.4691 | LR: 4.95e-04 | 1298.6ms/step | 15.4 samples/s +Step 3530/44000 | Loss: 3.4676 | LR: 4.95e-04 | 1298.7ms/step | 15.4 samples/s +Step 3540/44000 | Loss: 3.4738 | LR: 4.95e-04 | 1300.2ms/step | 15.4 samples/s +Step 3550/44000 | Loss: 3.4495 | LR: 4.95e-04 | 1302.2ms/step | 15.4 samples/s +Step 3560/44000 | Loss: 3.4373 | LR: 4.95e-04 | 1299.9ms/step | 15.4 samples/s +Step 3570/44000 | Loss: 3.4637 | LR: 4.95e-04 | 1302.4ms/step | 15.4 samples/s +Step 3580/44000 | Loss: 3.4417 | LR: 4.95e-04 | 1301.7ms/step | 15.4 samples/s +Step 3590/44000 | Loss: 3.4552 | LR: 4.95e-04 | 1300.6ms/step | 15.4 samples/s +Step 3600/44000 | Loss: 3.4433 | LR: 4.95e-04 | 1302.4ms/step | 15.4 samples/s +Step 3610/44000 | Loss: 3.4550 | LR: 4.95e-04 | 1301.0ms/step | 15.4 samples/s +Step 3620/44000 | Loss: 3.4538 | LR: 4.95e-04 | 1301.6ms/step | 15.4 samples/s +Step 3630/44000 | Loss: 3.4443 | LR: 4.95e-04 | 1300.2ms/step | 15.4 samples/s +Step 3640/44000 | Loss: 3.4562 | LR: 4.95e-04 | 1301.1ms/step | 15.4 samples/s +Step 3650/44000 | Loss: 3.4459 | LR: 4.94e-04 | 1299.9ms/step | 15.4 samples/s +Step 3660/44000 | Loss: 3.4416 | LR: 4.94e-04 | 1296.3ms/step | 15.4 samples/s +Step 3670/44000 | Loss: 3.4376 | LR: 4.94e-04 | 1296.9ms/step | 15.4 samples/s +Step 3680/44000 | Loss: 3.4492 | LR: 4.94e-04 | 1297.6ms/step | 15.4 samples/s +Step 3690/44000 | Loss: 3.4434 | LR: 4.94e-04 | 1298.4ms/step | 15.4 samples/s +Step 3700/44000 | Loss: 3.4568 | LR: 4.94e-04 | 1297.1ms/step | 15.4 samples/s +Step 3710/44000 | Loss: 3.4523 | LR: 4.94e-04 | 1300.6ms/step | 15.4 samples/s +Step 3720/44000 | Loss: 3.4052 | LR: 4.94e-04 | 1303.9ms/step | 15.3 samples/s +Step 3730/44000 | Loss: 3.4543 | LR: 4.94e-04 | 1303.5ms/step | 15.3 samples/s +Step 3740/44000 | Loss: 3.4493 | LR: 4.94e-04 | 1301.0ms/step | 15.4 samples/s +Step 3750/44000 | Loss: 3.4563 | LR: 4.94e-04 | 1303.9ms/step | 15.3 samples/s +Validation at step 3750: Loss = 3.4498, Perplexity = 31.50 +Step 3760/44000 | Loss: 3.4354 | LR: 4.94e-04 | 3494.8ms/step | 5.7 samples/s +Step 3770/44000 | Loss: 3.4307 | LR: 4.94e-04 | 1303.0ms/step | 15.3 samples/s +Step 3780/44000 | Loss: 3.4248 | LR: 4.94e-04 | 1301.8ms/step | 15.4 samples/s +Step 3790/44000 | Loss: 3.4462 | LR: 4.94e-04 | 1300.0ms/step | 15.4 samples/s +Step 3800/44000 | Loss: 3.4278 | LR: 4.94e-04 | 1298.1ms/step | 15.4 samples/s +Step 3810/44000 | Loss: 3.4343 | LR: 4.94e-04 | 1299.6ms/step | 15.4 samples/s +Step 3820/44000 | Loss: 3.4416 | LR: 4.94e-04 | 1301.0ms/step | 15.4 samples/s +Step 3830/44000 | Loss: 3.4317 | LR: 4.94e-04 | 1300.0ms/step | 15.4 samples/s +Step 3840/44000 | Loss: 3.4188 | LR: 4.94e-04 | 1302.5ms/step | 15.4 samples/s +Step 3850/44000 | Loss: 3.4269 | LR: 4.94e-04 | 1301.5ms/step | 15.4 samples/s +Step 3860/44000 | Loss: 3.4266 | LR: 4.94e-04 | 1299.5ms/step | 15.4 samples/s +Step 3870/44000 | Loss: 3.4323 | LR: 4.94e-04 | 1301.0ms/step | 15.4 samples/s +Step 3880/44000 | Loss: 3.4010 | LR: 4.94e-04 | 1300.1ms/step | 15.4 samples/s +Step 3890/44000 | Loss: 3.4581 | LR: 4.94e-04 | 1300.9ms/step | 15.4 samples/s +Step 3900/44000 | Loss: 3.4153 | LR: 4.93e-04 | 1301.2ms/step | 15.4 samples/s +Step 3910/44000 | Loss: 3.4067 | LR: 4.93e-04 | 1299.5ms/step | 15.4 samples/s +Step 3920/44000 | Loss: 3.4112 | LR: 4.93e-04 | 1301.3ms/step | 15.4 samples/s +Step 3930/44000 | Loss: 3.4208 | LR: 4.93e-04 | 1300.2ms/step | 15.4 samples/s +Step 3940/44000 | Loss: 3.4222 | LR: 4.93e-04 | 1300.0ms/step | 15.4 samples/s +Step 3950/44000 | Loss: 3.4472 | LR: 4.93e-04 | 1302.4ms/step | 15.4 samples/s +Step 3960/44000 | Loss: 3.4377 | LR: 4.93e-04 | 1295.4ms/step | 15.4 samples/s +Step 3970/44000 | Loss: 3.3927 | LR: 4.93e-04 | 1297.9ms/step | 15.4 samples/s +Step 3980/44000 | Loss: 3.4031 | LR: 4.93e-04 | 1298.6ms/step | 15.4 samples/s +Step 3990/44000 | Loss: 3.4310 | LR: 4.93e-04 | 1298.5ms/step | 15.4 samples/s +Step 4000/44000 | Loss: 3.4124 | LR: 4.93e-04 | 1300.0ms/step | 15.4 samples/s +Validation at step 4000: Loss = 3.4268, Perplexity = 30.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-4000 +Step 4010/44000 | Loss: 3.4641 | LR: 4.93e-04 | 3772.5ms/step | 5.3 samples/s +Step 4020/44000 | Loss: 3.4237 | LR: 4.93e-04 | 1304.0ms/step | 15.3 samples/s +Step 4030/44000 | Loss: 3.4008 | LR: 4.93e-04 | 1301.6ms/step | 15.4 samples/s +Step 4040/44000 | Loss: 3.4152 | LR: 4.93e-04 | 1305.9ms/step | 15.3 samples/s +Step 4050/44000 | Loss: 3.4496 | LR: 4.93e-04 | 1301.2ms/step | 15.4 samples/s +Step 4060/44000 | Loss: 3.3966 | LR: 4.93e-04 | 1303.2ms/step | 15.3 samples/s +Step 4070/44000 | Loss: 3.4185 | LR: 4.93e-04 | 1306.2ms/step | 15.3 samples/s +Step 4080/44000 | Loss: 3.4368 | LR: 4.93e-04 | 1303.1ms/step | 15.3 samples/s +Step 4090/44000 | Loss: 3.3870 | LR: 4.93e-04 | 1301.1ms/step | 15.4 samples/s +Step 4100/44000 | Loss: 3.4191 | LR: 4.93e-04 | 1299.6ms/step | 15.4 samples/s +Step 4110/44000 | Loss: 3.3893 | LR: 4.93e-04 | 1299.7ms/step | 15.4 samples/s +Step 4120/44000 | Loss: 3.4169 | LR: 4.93e-04 | 1296.3ms/step | 15.4 samples/s +Step 4130/44000 | Loss: 3.4176 | LR: 4.93e-04 | 1295.1ms/step | 15.4 samples/s +Step 4140/44000 | Loss: 3.3995 | LR: 4.92e-04 | 1298.1ms/step | 15.4 samples/s +Step 4150/44000 | Loss: 3.4439 | LR: 4.92e-04 | 1301.5ms/step | 15.4 samples/s +Step 4160/44000 | Loss: 3.4230 | LR: 4.92e-04 | 1302.6ms/step | 15.4 samples/s +Step 4170/44000 | Loss: 3.4078 | LR: 4.92e-04 | 1302.2ms/step | 15.4 samples/s +Step 4180/44000 | Loss: 3.3884 | LR: 4.92e-04 | 1301.2ms/step | 15.4 samples/s +Step 4190/44000 | Loss: 3.4005 | LR: 4.92e-04 | 1300.7ms/step | 15.4 samples/s +Step 4200/44000 | Loss: 3.3942 | LR: 4.92e-04 | 1301.1ms/step | 15.4 samples/s +Step 4210/44000 | Loss: 3.4248 | LR: 4.92e-04 | 1299.7ms/step | 15.4 samples/s +Step 4220/44000 | Loss: 3.3888 | LR: 4.92e-04 | 1300.4ms/step | 15.4 samples/s +Step 4230/44000 | Loss: 3.4160 | LR: 4.92e-04 | 1302.1ms/step | 15.4 samples/s +Step 4240/44000 | Loss: 3.4723 | LR: 4.92e-04 | 1301.5ms/step | 15.4 samples/s +Step 4250/44000 | Loss: 3.4404 | LR: 4.92e-04 | 1300.9ms/step | 15.4 samples/s +Validation at step 4250: Loss = 3.4088, Perplexity = 30.23 +Step 4260/44000 | Loss: 3.4134 | LR: 4.92e-04 | 3482.1ms/step | 5.7 samples/s +Step 4270/44000 | Loss: 3.3465 | LR: 4.92e-04 | 1301.7ms/step | 15.4 samples/s +Step 4280/44000 | Loss: 3.3710 | LR: 4.92e-04 | 1303.3ms/step | 15.3 samples/s +Step 4290/44000 | Loss: 3.4035 | LR: 4.92e-04 | 1300.9ms/step | 15.4 samples/s +Step 4300/44000 | Loss: 3.3957 | LR: 4.92e-04 | 1300.2ms/step | 15.4 samples/s +Step 4310/44000 | Loss: 3.3974 | LR: 4.92e-04 | 1301.5ms/step | 15.4 samples/s +Step 4320/44000 | Loss: 3.4022 | LR: 4.92e-04 | 1299.6ms/step | 15.4 samples/s +Step 4330/44000 | Loss: 3.3899 | LR: 4.92e-04 | 1308.0ms/step | 15.3 samples/s +Step 4340/44000 | Loss: 3.4033 | LR: 4.92e-04 | 1309.6ms/step | 15.3 samples/s +Step 4350/44000 | Loss: 3.4120 | LR: 4.92e-04 | 1304.2ms/step | 15.3 samples/s +Step 4360/44000 | Loss: 3.3611 | LR: 4.91e-04 | 1299.2ms/step | 15.4 samples/s +Step 4370/44000 | Loss: 3.3789 | LR: 4.91e-04 | 1309.1ms/step | 15.3 samples/s +Step 4380/44000 | Loss: 3.4250 | LR: 4.91e-04 | 1313.4ms/step | 15.2 samples/s +Step 4390/44000 | Loss: 3.4130 | LR: 4.91e-04 | 1309.7ms/step | 15.3 samples/s +Step 4400/44000 | Loss: 3.4109 | LR: 4.91e-04 | 1304.7ms/step | 15.3 samples/s +Step 4410/44000 | Loss: 3.3828 | LR: 4.91e-04 | 1313.8ms/step | 15.2 samples/s +Step 4420/44000 | Loss: 3.3908 | LR: 4.91e-04 | 1302.4ms/step | 15.4 samples/s +Step 4430/44000 | Loss: 3.3914 | LR: 4.91e-04 | 1301.9ms/step | 15.4 samples/s +Step 4440/44000 | Loss: 3.4020 | LR: 4.91e-04 | 1301.7ms/step | 15.4 samples/s +Step 4450/44000 | Loss: 3.3925 | LR: 4.91e-04 | 1305.6ms/step | 15.3 samples/s +Step 4460/44000 | Loss: 3.3914 | LR: 4.91e-04 | 1302.9ms/step | 15.4 samples/s +Step 4470/44000 | Loss: 3.3554 | LR: 4.91e-04 | 1303.2ms/step | 15.3 samples/s +Step 4480/44000 | Loss: 3.3897 | LR: 4.91e-04 | 1303.5ms/step | 15.3 samples/s +Step 4490/44000 | Loss: 3.3954 | LR: 4.91e-04 | 1303.2ms/step | 15.3 samples/s +Step 4500/44000 | Loss: 3.3878 | LR: 4.91e-04 | 1301.6ms/step | 15.4 samples/s +Validation at step 4500: Loss = 3.3894, Perplexity = 29.65 +Step 4510/44000 | Loss: 3.3777 | LR: 4.91e-04 | 3483.5ms/step | 5.7 samples/s +Step 4520/44000 | Loss: 3.3992 | LR: 4.91e-04 | 1300.6ms/step | 15.4 samples/s +Step 4530/44000 | Loss: 3.3990 | LR: 4.91e-04 | 1300.8ms/step | 15.4 samples/s +Step 4540/44000 | Loss: 3.3595 | LR: 4.91e-04 | 1301.3ms/step | 15.4 samples/s +Step 4550/44000 | Loss: 3.3536 | LR: 4.91e-04 | 1302.1ms/step | 15.4 samples/s +Step 4560/44000 | Loss: 3.3604 | LR: 4.90e-04 | 1301.5ms/step | 15.4 samples/s +Step 4570/44000 | Loss: 3.3796 | LR: 4.90e-04 | 1300.9ms/step | 15.4 samples/s +Step 4580/44000 | Loss: 3.3375 | LR: 4.90e-04 | 1300.8ms/step | 15.4 samples/s +Step 4590/44000 | Loss: 3.3395 | LR: 4.90e-04 | 1301.8ms/step | 15.4 samples/s +Step 4600/44000 | Loss: 3.3598 | LR: 4.90e-04 | 1302.3ms/step | 15.4 samples/s +Step 4610/44000 | Loss: 3.3698 | LR: 4.90e-04 | 1301.3ms/step | 15.4 samples/s +Step 4620/44000 | Loss: 3.3587 | LR: 4.90e-04 | 1300.9ms/step | 15.4 samples/s +Step 4630/44000 | Loss: 3.3690 | LR: 4.90e-04 | 1302.8ms/step | 15.4 samples/s +Step 4640/44000 | Loss: 3.3794 | LR: 4.90e-04 | 1303.2ms/step | 15.3 samples/s +Step 4650/44000 | Loss: 3.3690 | LR: 4.90e-04 | 1304.1ms/step | 15.3 samples/s +Step 4660/44000 | Loss: 3.3469 | LR: 4.90e-04 | 1301.3ms/step | 15.4 samples/s +Step 4670/44000 | Loss: 3.3403 | LR: 4.90e-04 | 1302.4ms/step | 15.4 samples/s +Step 4680/44000 | Loss: 3.3627 | LR: 4.90e-04 | 1300.6ms/step | 15.4 samples/s +Step 4690/44000 | Loss: 3.3575 | LR: 4.90e-04 | 1301.1ms/step | 15.4 samples/s +Step 4700/44000 | Loss: 3.3919 | LR: 4.90e-04 | 1301.4ms/step | 15.4 samples/s +Step 4710/44000 | Loss: 3.3673 | LR: 4.90e-04 | 1300.5ms/step | 15.4 samples/s +Step 4720/44000 | Loss: 3.3459 | LR: 4.90e-04 | 1300.0ms/step | 15.4 samples/s +Step 4730/44000 | Loss: 3.3650 | LR: 4.90e-04 | 1300.7ms/step | 15.4 samples/s +Step 4740/44000 | Loss: 3.3555 | LR: 4.90e-04 | 1301.2ms/step | 15.4 samples/s +Step 4750/44000 | Loss: 3.3667 | LR: 4.90e-04 | 1300.7ms/step | 15.4 samples/s +Validation at step 4750: Loss = 3.3708, Perplexity = 29.10 +Step 4760/44000 | Loss: 3.3447 | LR: 4.89e-04 | 3484.7ms/step | 5.7 samples/s +Step 4770/44000 | Loss: 3.3681 | LR: 4.89e-04 | 1297.5ms/step | 15.4 samples/s +Step 4780/44000 | Loss: 3.3656 | LR: 4.89e-04 | 1300.3ms/step | 15.4 samples/s +Step 4790/44000 | Loss: 3.3393 | LR: 4.89e-04 | 1302.0ms/step | 15.4 samples/s +Step 4800/44000 | Loss: 3.3638 | LR: 4.89e-04 | 1301.2ms/step | 15.4 samples/s +Step 4810/44000 | Loss: 3.3615 | LR: 4.89e-04 | 1304.9ms/step | 15.3 samples/s +Step 4820/44000 | Loss: 3.3634 | LR: 4.89e-04 | 1307.5ms/step | 15.3 samples/s +Step 4830/44000 | Loss: 3.3510 | LR: 4.89e-04 | 1309.0ms/step | 15.3 samples/s +Step 4840/44000 | Loss: 3.3637 | LR: 4.89e-04 | 1307.5ms/step | 15.3 samples/s +Step 4850/44000 | Loss: 3.3651 | LR: 4.89e-04 | 1302.1ms/step | 15.4 samples/s +Step 4860/44000 | Loss: 3.3722 | LR: 4.89e-04 | 1302.4ms/step | 15.4 samples/s +Step 4870/44000 | Loss: 3.3365 | LR: 4.89e-04 | 1302.0ms/step | 15.4 samples/s +Step 4880/44000 | Loss: 3.3357 | LR: 4.89e-04 | 1300.9ms/step | 15.4 samples/s +Step 4890/44000 | Loss: 3.3269 | LR: 4.89e-04 | 1299.7ms/step | 15.4 samples/s +Step 4900/44000 | Loss: 3.3109 | LR: 4.89e-04 | 1299.2ms/step | 15.4 samples/s +Step 4910/44000 | Loss: 3.3472 | LR: 4.89e-04 | 1302.2ms/step | 15.4 samples/s +Step 4920/44000 | Loss: 3.3473 | LR: 4.89e-04 | 1299.1ms/step | 15.4 samples/s +Step 4930/44000 | Loss: 3.3313 | LR: 4.89e-04 | 1300.9ms/step | 15.4 samples/s +Step 4940/44000 | Loss: 3.3407 | LR: 4.89e-04 | 1297.8ms/step | 15.4 samples/s +Step 4950/44000 | Loss: 3.3435 | LR: 4.88e-04 | 1301.2ms/step | 15.4 samples/s +Step 4960/44000 | Loss: 3.3329 | LR: 4.88e-04 | 1299.6ms/step | 15.4 samples/s +Step 4970/44000 | Loss: 3.3241 | LR: 4.88e-04 | 1300.5ms/step | 15.4 samples/s +Step 4980/44000 | Loss: 3.3211 | LR: 4.88e-04 | 1300.3ms/step | 15.4 samples/s +Step 4990/44000 | Loss: 3.3480 | LR: 4.88e-04 | 1301.1ms/step | 15.4 samples/s +Step 5000/44000 | Loss: 3.3455 | LR: 4.88e-04 | 1301.9ms/step | 15.4 samples/s +Validation at step 5000: Loss = 3.3540, Perplexity = 28.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-5000 +Step 5010/44000 | Loss: 3.3525 | LR: 4.88e-04 | 3773.8ms/step | 5.3 samples/s +Step 5020/44000 | Loss: 3.3325 | LR: 4.88e-04 | 1294.8ms/step | 15.4 samples/s +Step 5030/44000 | Loss: 3.3390 | LR: 4.88e-04 | 1296.4ms/step | 15.4 samples/s +Step 5040/44000 | Loss: 3.3497 | LR: 4.88e-04 | 1297.4ms/step | 15.4 samples/s +Step 5050/44000 | Loss: 3.3251 | LR: 4.88e-04 | 1297.4ms/step | 15.4 samples/s +Step 5060/44000 | Loss: 3.3552 | LR: 4.88e-04 | 1305.7ms/step | 15.3 samples/s +Step 5070/44000 | Loss: 3.3586 | LR: 4.88e-04 | 1309.2ms/step | 15.3 samples/s +Step 5080/44000 | Loss: 3.3500 | LR: 4.88e-04 | 1299.1ms/step | 15.4 samples/s +Step 5090/44000 | Loss: 3.3221 | LR: 4.88e-04 | 1300.8ms/step | 15.4 samples/s +Step 5100/44000 | Loss: 3.3491 | LR: 4.88e-04 | 1299.9ms/step | 15.4 samples/s +Step 5110/44000 | Loss: 3.3670 | LR: 4.88e-04 | 1298.7ms/step | 15.4 samples/s +Step 5120/44000 | Loss: 3.3669 | LR: 4.88e-04 | 1300.7ms/step | 15.4 samples/s +Step 5130/44000 | Loss: 3.3472 | LR: 4.87e-04 | 1300.9ms/step | 15.4 samples/s +Step 5140/44000 | Loss: 3.3223 | LR: 4.87e-04 | 1299.5ms/step | 15.4 samples/s +Step 5150/44000 | Loss: 3.3659 | LR: 4.87e-04 | 1297.5ms/step | 15.4 samples/s +Step 5160/44000 | Loss: 3.3646 | LR: 4.87e-04 | 1298.3ms/step | 15.4 samples/s +Step 5170/44000 | Loss: 3.3541 | LR: 4.87e-04 | 1299.9ms/step | 15.4 samples/s +Step 5180/44000 | Loss: 3.3107 | LR: 4.87e-04 | 1301.1ms/step | 15.4 samples/s +Step 5190/44000 | Loss: 3.3398 | LR: 4.87e-04 | 1308.5ms/step | 15.3 samples/s +Step 5200/44000 | Loss: 3.3060 | LR: 4.87e-04 | 1331.5ms/step | 15.0 samples/s +Step 5210/44000 | Loss: 3.3430 | LR: 4.87e-04 | 1307.6ms/step | 15.3 samples/s +Step 5220/44000 | Loss: 3.3357 | LR: 4.87e-04 | 1309.8ms/step | 15.3 samples/s +Step 5230/44000 | Loss: 3.2899 | LR: 4.87e-04 | 1302.0ms/step | 15.4 samples/s +Step 5240/44000 | Loss: 3.3195 | LR: 4.87e-04 | 1300.9ms/step | 15.4 samples/s +Step 5250/44000 | Loss: 3.3191 | LR: 4.87e-04 | 1312.0ms/step | 15.2 samples/s +Validation at step 5250: Loss = 3.3396, Perplexity = 28.21 +Step 5260/44000 | Loss: 3.3428 | LR: 4.87e-04 | 3507.9ms/step | 5.7 samples/s +Step 5270/44000 | Loss: 3.3083 | LR: 4.87e-04 | 1301.4ms/step | 15.4 samples/s +Step 5280/44000 | Loss: 3.3376 | LR: 4.87e-04 | 1301.5ms/step | 15.4 samples/s +Step 5290/44000 | Loss: 3.3225 | LR: 4.87e-04 | 1301.7ms/step | 15.4 samples/s +Step 5300/44000 | Loss: 3.2764 | LR: 4.86e-04 | 1302.5ms/step | 15.4 samples/s +Step 5310/44000 | Loss: 3.3170 | LR: 4.86e-04 | 1301.1ms/step | 15.4 samples/s +Step 5320/44000 | Loss: 3.3291 | LR: 4.86e-04 | 1300.3ms/step | 15.4 samples/s +Step 5330/44000 | Loss: 3.3249 | LR: 4.86e-04 | 1306.2ms/step | 15.3 samples/s +Step 5340/44000 | Loss: 3.3380 | LR: 4.86e-04 | 1303.9ms/step | 15.3 samples/s +Step 5350/44000 | Loss: 3.3054 | LR: 4.86e-04 | 1298.7ms/step | 15.4 samples/s +Step 5360/44000 | Loss: 3.3274 | LR: 4.86e-04 | 1297.3ms/step | 15.4 samples/s +Step 5370/44000 | Loss: 3.3265 | LR: 4.86e-04 | 1297.0ms/step | 15.4 samples/s +Step 5380/44000 | Loss: 3.3142 | LR: 4.86e-04 | 1297.2ms/step | 15.4 samples/s +Step 5390/44000 | Loss: 3.3365 | LR: 4.86e-04 | 1298.9ms/step | 15.4 samples/s +Step 5400/44000 | Loss: 3.3430 | LR: 4.86e-04 | 1296.8ms/step | 15.4 samples/s +Step 5410/44000 | Loss: 3.3286 | LR: 4.86e-04 | 1297.9ms/step | 15.4 samples/s +Step 5420/44000 | Loss: 3.3251 | LR: 4.86e-04 | 1296.5ms/step | 15.4 samples/s +Step 5430/44000 | Loss: 3.3071 | LR: 4.86e-04 | 1295.9ms/step | 15.4 samples/s +Step 5440/44000 | Loss: 3.3186 | LR: 4.86e-04 | 1298.0ms/step | 15.4 samples/s +Step 5450/44000 | Loss: 3.3308 | LR: 4.86e-04 | 1295.9ms/step | 15.4 samples/s +Step 5460/44000 | Loss: 3.3472 | LR: 4.86e-04 | 1296.6ms/step | 15.4 samples/s +Step 5470/44000 | Loss: 3.3274 | LR: 4.85e-04 | 1295.4ms/step | 15.4 samples/s +Step 5480/44000 | Loss: 3.3078 | LR: 4.85e-04 | 1297.5ms/step | 15.4 samples/s +Step 5490/44000 | Loss: 3.2967 | LR: 4.85e-04 | 1297.5ms/step | 15.4 samples/s +Step 5500/44000 | Loss: 3.3194 | LR: 4.85e-04 | 1296.9ms/step | 15.4 samples/s +Validation at step 5500: Loss = 3.3245, Perplexity = 27.79 +Step 5510/44000 | Loss: 3.3153 | LR: 4.85e-04 | 3487.7ms/step | 5.7 samples/s +Step 5520/44000 | Loss: 3.3172 | LR: 4.85e-04 | 1298.7ms/step | 15.4 samples/s +Step 5530/44000 | Loss: 3.2869 | LR: 4.85e-04 | 1299.3ms/step | 15.4 samples/s +Step 5540/44000 | Loss: 3.3225 | LR: 4.85e-04 | 1301.9ms/step | 15.4 samples/s +Step 5550/44000 | Loss: 3.3040 | LR: 4.85e-04 | 1302.9ms/step | 15.4 samples/s +Step 5560/44000 | Loss: 3.3122 | LR: 4.85e-04 | 1301.4ms/step | 15.4 samples/s +Step 5570/44000 | Loss: 3.3069 | LR: 4.85e-04 | 1298.7ms/step | 15.4 samples/s +Step 5580/44000 | Loss: 3.2953 | LR: 4.85e-04 | 1299.1ms/step | 15.4 samples/s +Step 5590/44000 | Loss: 3.2891 | LR: 4.85e-04 | 1300.4ms/step | 15.4 samples/s +Step 5600/44000 | Loss: 3.2915 | LR: 4.85e-04 | 1302.7ms/step | 15.4 samples/s +Step 5610/44000 | Loss: 3.3211 | LR: 4.85e-04 | 1308.2ms/step | 15.3 samples/s +Step 5620/44000 | Loss: 3.3024 | LR: 4.85e-04 | 1303.8ms/step | 15.3 samples/s +Step 5630/44000 | Loss: 3.3227 | LR: 4.84e-04 | 1303.3ms/step | 15.3 samples/s +Step 5640/44000 | Loss: 3.2964 | LR: 4.84e-04 | 1303.5ms/step | 15.3 samples/s +Step 5650/44000 | Loss: 3.2923 | LR: 4.84e-04 | 1302.2ms/step | 15.4 samples/s +Step 5660/44000 | Loss: 3.2800 | LR: 4.84e-04 | 1300.4ms/step | 15.4 samples/s +Step 5670/44000 | Loss: 3.2992 | LR: 4.84e-04 | 1300.3ms/step | 15.4 samples/s +Step 5680/44000 | Loss: 3.2815 | LR: 4.84e-04 | 1303.1ms/step | 15.3 samples/s +Step 5690/44000 | Loss: 3.3290 | LR: 4.84e-04 | 1302.4ms/step | 15.4 samples/s +Step 5700/44000 | Loss: 3.3177 | LR: 4.84e-04 | 1301.3ms/step | 15.4 samples/s +Step 5710/44000 | Loss: 3.2663 | LR: 4.84e-04 | 1298.8ms/step | 15.4 samples/s +Step 5720/44000 | Loss: 3.2665 | LR: 4.84e-04 | 1299.5ms/step | 15.4 samples/s +Step 5730/44000 | Loss: 3.2853 | LR: 4.84e-04 | 1299.5ms/step | 15.4 samples/s +Step 5740/44000 | Loss: 3.2893 | LR: 4.84e-04 | 1301.3ms/step | 15.4 samples/s +Step 5750/44000 | Loss: 3.2922 | LR: 4.84e-04 | 1301.6ms/step | 15.4 samples/s +Validation at step 5750: Loss = 3.3120, Perplexity = 27.44 +Step 5760/44000 | Loss: 3.3163 | LR: 4.84e-04 | 3483.9ms/step | 5.7 samples/s +Step 5770/44000 | Loss: 3.3207 | LR: 4.84e-04 | 1300.6ms/step | 15.4 samples/s +Step 5780/44000 | Loss: 3.2992 | LR: 4.83e-04 | 1298.3ms/step | 15.4 samples/s +Step 5790/44000 | Loss: 3.2820 | LR: 4.83e-04 | 1305.1ms/step | 15.3 samples/s +Step 5800/44000 | Loss: 3.3078 | LR: 4.83e-04 | 1307.4ms/step | 15.3 samples/s +Step 5810/44000 | Loss: 3.3207 | LR: 4.83e-04 | 1307.1ms/step | 15.3 samples/s +Step 5820/44000 | Loss: 3.3037 | LR: 4.83e-04 | 1307.1ms/step | 15.3 samples/s +Step 5830/44000 | Loss: 3.2999 | LR: 4.83e-04 | 1308.7ms/step | 15.3 samples/s +Step 5840/44000 | Loss: 3.2933 | LR: 4.83e-04 | 1306.5ms/step | 15.3 samples/s +Step 5850/44000 | Loss: 3.2743 | LR: 4.83e-04 | 1303.6ms/step | 15.3 samples/s +Step 5860/44000 | Loss: 3.3036 | LR: 4.83e-04 | 1300.9ms/step | 15.4 samples/s +Step 5870/44000 | Loss: 3.3102 | LR: 4.83e-04 | 1297.3ms/step | 15.4 samples/s +Step 5880/44000 | Loss: 3.2881 | LR: 4.83e-04 | 1295.6ms/step | 15.4 samples/s +Step 5890/44000 | Loss: 3.3051 | LR: 4.83e-04 | 1295.6ms/step | 15.4 samples/s +Step 5900/44000 | Loss: 3.2936 | LR: 4.83e-04 | 1296.7ms/step | 15.4 samples/s +Step 5910/44000 | Loss: 3.3002 | LR: 4.83e-04 | 1297.9ms/step | 15.4 samples/s +Step 5920/44000 | Loss: 3.3088 | LR: 4.83e-04 | 1298.4ms/step | 15.4 samples/s +Step 5930/44000 | Loss: 3.2606 | LR: 4.83e-04 | 1298.0ms/step | 15.4 samples/s +Step 5940/44000 | Loss: 3.3102 | LR: 4.82e-04 | 1297.8ms/step | 15.4 samples/s +Step 5950/44000 | Loss: 3.2722 | LR: 4.82e-04 | 1297.1ms/step | 15.4 samples/s +Step 5960/44000 | Loss: 3.2625 | LR: 4.82e-04 | 1302.2ms/step | 15.4 samples/s +Step 5970/44000 | Loss: 3.2692 | LR: 4.82e-04 | 1300.5ms/step | 15.4 samples/s +Step 5980/44000 | Loss: 3.2644 | LR: 4.82e-04 | 1302.5ms/step | 15.4 samples/s +Step 5990/44000 | Loss: 3.2641 | LR: 4.82e-04 | 1301.3ms/step | 15.4 samples/s +Step 6000/44000 | Loss: 3.2907 | LR: 4.82e-04 | 1300.4ms/step | 15.4 samples/s +Validation at step 6000: Loss = 3.2999, Perplexity = 27.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-6000 +Step 6010/44000 | Loss: 3.3117 | LR: 4.82e-04 | 3793.7ms/step | 5.3 samples/s +Step 6020/44000 | Loss: 3.2751 | LR: 4.82e-04 | 1302.9ms/step | 15.4 samples/s +Step 6030/44000 | Loss: 3.2789 | LR: 4.82e-04 | 1305.9ms/step | 15.3 samples/s +Step 6040/44000 | Loss: 3.2820 | LR: 4.82e-04 | 1302.7ms/step | 15.4 samples/s +Step 6050/44000 | Loss: 3.2862 | LR: 4.82e-04 | 1302.2ms/step | 15.4 samples/s +Step 6060/44000 | Loss: 3.3085 | LR: 4.82e-04 | 1300.7ms/step | 15.4 samples/s +Step 6070/44000 | Loss: 3.2887 | LR: 4.82e-04 | 1298.8ms/step | 15.4 samples/s +Step 6080/44000 | Loss: 3.2808 | LR: 4.81e-04 | 1303.8ms/step | 15.3 samples/s +Step 6090/44000 | Loss: 3.3093 | LR: 4.81e-04 | 1303.2ms/step | 15.3 samples/s +Step 6100/44000 | Loss: 3.2636 | LR: 4.81e-04 | 1302.1ms/step | 15.4 samples/s +Step 6110/44000 | Loss: 3.2779 | LR: 4.81e-04 | 1299.3ms/step | 15.4 samples/s +Step 6120/44000 | Loss: 3.2593 | LR: 4.81e-04 | 1299.8ms/step | 15.4 samples/s +Step 6130/44000 | Loss: 3.2770 | LR: 4.81e-04 | 1300.8ms/step | 15.4 samples/s +Step 6140/44000 | Loss: 3.2636 | LR: 4.81e-04 | 1300.5ms/step | 15.4 samples/s +Step 6150/44000 | Loss: 3.2717 | LR: 4.81e-04 | 1300.7ms/step | 15.4 samples/s +Step 6160/44000 | Loss: 3.2690 | LR: 4.81e-04 | 1298.0ms/step | 15.4 samples/s +Step 6170/44000 | Loss: 3.2861 | LR: 4.81e-04 | 1299.2ms/step | 15.4 samples/s +Step 6180/44000 | Loss: 3.2830 | LR: 4.81e-04 | 1298.9ms/step | 15.4 samples/s +Step 6190/44000 | Loss: 3.2663 | LR: 4.81e-04 | 1300.0ms/step | 15.4 samples/s +Step 6200/44000 | Loss: 3.2822 | LR: 4.81e-04 | 1298.5ms/step | 15.4 samples/s +Step 6210/44000 | Loss: 3.2784 | LR: 4.81e-04 | 1300.8ms/step | 15.4 samples/s +Step 6220/44000 | Loss: 3.3153 | LR: 4.81e-04 | 1301.7ms/step | 15.4 samples/s +Step 6230/44000 | Loss: 3.2771 | LR: 4.80e-04 | 1299.6ms/step | 15.4 samples/s +Step 6240/44000 | Loss: 3.2967 | LR: 4.80e-04 | 1297.2ms/step | 15.4 samples/s +Step 6250/44000 | Loss: 3.2925 | LR: 4.80e-04 | 1297.6ms/step | 15.4 samples/s +Validation at step 6250: Loss = 3.2972, Perplexity = 27.04 +Step 6260/44000 | Loss: 3.2864 | LR: 4.80e-04 | 3481.5ms/step | 5.7 samples/s +Step 6270/44000 | Loss: 3.3111 | LR: 4.80e-04 | 1302.3ms/step | 15.4 samples/s +Step 6280/44000 | Loss: 3.2615 | LR: 4.80e-04 | 1301.5ms/step | 15.4 samples/s +Step 6290/44000 | Loss: 3.2915 | LR: 4.80e-04 | 1301.2ms/step | 15.4 samples/s +Step 6300/44000 | Loss: 3.2679 | LR: 4.80e-04 | 1301.6ms/step | 15.4 samples/s +Step 6310/44000 | Loss: 3.2704 | LR: 4.80e-04 | 1308.0ms/step | 15.3 samples/s +Step 6320/44000 | Loss: 3.2663 | LR: 4.80e-04 | 1313.1ms/step | 15.2 samples/s +Step 6330/44000 | Loss: 3.2730 | LR: 4.80e-04 | 1304.7ms/step | 15.3 samples/s +Step 6340/44000 | Loss: 3.2834 | LR: 4.80e-04 | 1302.9ms/step | 15.4 samples/s +Step 6350/44000 | Loss: 3.2438 | LR: 4.80e-04 | 1303.1ms/step | 15.3 samples/s +Step 6360/44000 | Loss: 3.3000 | LR: 4.80e-04 | 1303.1ms/step | 15.3 samples/s +Step 6370/44000 | Loss: 3.2687 | LR: 4.79e-04 | 1301.8ms/step | 15.4 samples/s +Step 6380/44000 | Loss: 3.2558 | LR: 4.79e-04 | 1302.6ms/step | 15.4 samples/s +Step 6390/44000 | Loss: 3.2494 | LR: 4.79e-04 | 1302.2ms/step | 15.4 samples/s +Step 6400/44000 | Loss: 3.2798 | LR: 4.79e-04 | 1301.4ms/step | 15.4 samples/s +Step 6410/44000 | Loss: 3.2505 | LR: 4.79e-04 | 1300.0ms/step | 15.4 samples/s +Step 6420/44000 | Loss: 3.2819 | LR: 4.79e-04 | 1299.5ms/step | 15.4 samples/s +Step 6430/44000 | Loss: 3.2434 | LR: 4.79e-04 | 1300.2ms/step | 15.4 samples/s +Step 6440/44000 | Loss: 3.2602 | LR: 4.79e-04 | 1299.9ms/step | 15.4 samples/s +Step 6450/44000 | Loss: 3.2570 | LR: 4.79e-04 | 1300.1ms/step | 15.4 samples/s +Step 6460/44000 | Loss: 3.2687 | LR: 4.79e-04 | 1300.0ms/step | 15.4 samples/s +Step 6470/44000 | Loss: 3.2459 | LR: 4.79e-04 | 1301.1ms/step | 15.4 samples/s +Step 6480/44000 | Loss: 3.2604 | LR: 4.79e-04 | 1301.9ms/step | 15.4 samples/s +Step 6490/44000 | Loss: 3.2535 | LR: 4.79e-04 | 1302.2ms/step | 15.4 samples/s +Step 6500/44000 | Loss: 3.2514 | LR: 4.79e-04 | 1302.8ms/step | 15.4 samples/s +Validation at step 6500: Loss = 3.2787, Perplexity = 26.54 +Step 6510/44000 | Loss: 3.2765 | LR: 4.78e-04 | 3480.2ms/step | 5.7 samples/s +Step 6520/44000 | Loss: 3.2832 | LR: 4.78e-04 | 1298.9ms/step | 15.4 samples/s +Step 6530/44000 | Loss: 3.2801 | LR: 4.78e-04 | 1300.1ms/step | 15.4 samples/s +Step 6540/44000 | Loss: 3.2763 | LR: 4.78e-04 | 1300.4ms/step | 15.4 samples/s +Step 6550/44000 | Loss: 3.2767 | LR: 4.78e-04 | 1300.4ms/step | 15.4 samples/s +Step 6560/44000 | Loss: 3.2561 | LR: 4.78e-04 | 1299.8ms/step | 15.4 samples/s +Step 6570/44000 | Loss: 3.2599 | LR: 4.78e-04 | 1296.2ms/step | 15.4 samples/s +Step 6580/44000 | Loss: 3.2997 | LR: 4.78e-04 | 1299.6ms/step | 15.4 samples/s +Step 6590/44000 | Loss: 3.2645 | LR: 4.78e-04 | 1300.2ms/step | 15.4 samples/s +Step 6600/44000 | Loss: 3.2539 | LR: 4.78e-04 | 1300.2ms/step | 15.4 samples/s +Step 6610/44000 | Loss: 3.2204 | LR: 4.78e-04 | 1300.0ms/step | 15.4 samples/s +Step 6620/44000 | Loss: 3.2516 | LR: 4.78e-04 | 1300.0ms/step | 15.4 samples/s +Step 6630/44000 | Loss: 3.2555 | LR: 4.78e-04 | 1299.4ms/step | 15.4 samples/s +Step 6640/44000 | Loss: 3.2454 | LR: 4.77e-04 | 1300.5ms/step | 15.4 samples/s +Step 6650/44000 | Loss: 3.2713 | LR: 4.77e-04 | 1300.0ms/step | 15.4 samples/s +Step 6660/44000 | Loss: 3.2968 | LR: 4.77e-04 | 1299.7ms/step | 15.4 samples/s +Step 6670/44000 | Loss: 3.2437 | LR: 4.77e-04 | 1299.7ms/step | 15.4 samples/s +Step 6680/44000 | Loss: 3.2672 | LR: 4.77e-04 | 1297.8ms/step | 15.4 samples/s +Step 6690/44000 | Loss: 3.2691 | LR: 4.77e-04 | 1297.1ms/step | 15.4 samples/s +Step 6700/44000 | Loss: 3.2700 | LR: 4.77e-04 | 1300.4ms/step | 15.4 samples/s +Step 6710/44000 | Loss: 3.2668 | LR: 4.77e-04 | 1308.4ms/step | 15.3 samples/s +Step 6720/44000 | Loss: 3.2318 | LR: 4.77e-04 | 1307.7ms/step | 15.3 samples/s +Step 6730/44000 | Loss: 3.2644 | LR: 4.77e-04 | 1302.1ms/step | 15.4 samples/s +Step 6740/44000 | Loss: 3.2415 | LR: 4.77e-04 | 1301.1ms/step | 15.4 samples/s +Step 6750/44000 | Loss: 3.2483 | LR: 4.77e-04 | 1299.8ms/step | 15.4 samples/s +Validation at step 6750: Loss = 3.2685, Perplexity = 26.27 +Step 6760/44000 | Loss: 3.2730 | LR: 4.77e-04 | 3492.2ms/step | 5.7 samples/s +Step 6770/44000 | Loss: 3.2717 | LR: 4.76e-04 | 1300.6ms/step | 15.4 samples/s +Step 6780/44000 | Loss: 3.2667 | LR: 4.76e-04 | 1301.6ms/step | 15.4 samples/s +Step 6790/44000 | Loss: 3.2598 | LR: 4.76e-04 | 1301.4ms/step | 15.4 samples/s +Step 6800/44000 | Loss: 3.2140 | LR: 4.76e-04 | 1305.1ms/step | 15.3 samples/s +Step 6810/44000 | Loss: 3.2757 | LR: 4.76e-04 | 1303.3ms/step | 15.3 samples/s +Step 6820/44000 | Loss: 3.2422 | LR: 4.76e-04 | 1306.0ms/step | 15.3 samples/s +Step 6830/44000 | Loss: 3.2568 | LR: 4.76e-04 | 1309.0ms/step | 15.3 samples/s +Step 6840/44000 | Loss: 3.2607 | LR: 4.76e-04 | 1306.0ms/step | 15.3 samples/s +Step 6850/44000 | Loss: 3.2626 | LR: 4.76e-04 | 1302.5ms/step | 15.4 samples/s +Step 6860/44000 | Loss: 3.2606 | LR: 4.76e-04 | 1302.0ms/step | 15.4 samples/s +Step 6870/44000 | Loss: 3.2369 | LR: 4.76e-04 | 1300.7ms/step | 15.4 samples/s +Step 6880/44000 | Loss: 3.2110 | LR: 4.76e-04 | 1300.6ms/step | 15.4 samples/s +Step 6890/44000 | Loss: 3.2672 | LR: 4.76e-04 | 1299.9ms/step | 15.4 samples/s +Step 6900/44000 | Loss: 3.2784 | LR: 4.75e-04 | 1301.2ms/step | 15.4 samples/s +Step 6910/44000 | Loss: 3.2408 | LR: 4.75e-04 | 1301.9ms/step | 15.4 samples/s +Step 6920/44000 | Loss: 3.2592 | LR: 4.75e-04 | 1300.0ms/step | 15.4 samples/s +Step 6930/44000 | Loss: 3.2415 | LR: 4.75e-04 | 1299.4ms/step | 15.4 samples/s +Step 6940/44000 | Loss: 3.2673 | LR: 4.75e-04 | 1298.5ms/step | 15.4 samples/s +Step 6950/44000 | Loss: 3.2667 | LR: 4.75e-04 | 1299.8ms/step | 15.4 samples/s +Step 6960/44000 | Loss: 3.2388 | LR: 4.75e-04 | 1300.4ms/step | 15.4 samples/s +Step 6970/44000 | Loss: 3.2631 | LR: 4.75e-04 | 1300.1ms/step | 15.4 samples/s +Step 6980/44000 | Loss: 3.2294 | LR: 4.75e-04 | 1301.3ms/step | 15.4 samples/s +Step 6990/44000 | Loss: 3.2356 | LR: 4.75e-04 | 1300.3ms/step | 15.4 samples/s +Step 7000/44000 | Loss: 3.2428 | LR: 4.75e-04 | 1298.4ms/step | 15.4 samples/s +Validation at step 7000: Loss = 3.2589, Perplexity = 26.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-7000 +Step 7010/44000 | Loss: 3.2340 | LR: 4.75e-04 | 3791.1ms/step | 5.3 samples/s +Step 7020/44000 | Loss: 3.2363 | LR: 4.75e-04 | 1298.0ms/step | 15.4 samples/s +Step 7030/44000 | Loss: 3.2592 | LR: 4.74e-04 | 1301.9ms/step | 15.4 samples/s +Step 7040/44000 | Loss: 3.2500 | LR: 4.74e-04 | 1305.4ms/step | 15.3 samples/s +Step 7050/44000 | Loss: 3.2754 | LR: 4.74e-04 | 1301.4ms/step | 15.4 samples/s +Step 7060/44000 | Loss: 3.2563 | LR: 4.74e-04 | 1297.7ms/step | 15.4 samples/s +Step 7070/44000 | Loss: 3.2441 | LR: 4.74e-04 | 1297.5ms/step | 15.4 samples/s +Step 7080/44000 | Loss: 3.2359 | LR: 4.74e-04 | 1298.9ms/step | 15.4 samples/s +Step 7090/44000 | Loss: 3.2151 | LR: 4.74e-04 | 1299.7ms/step | 15.4 samples/s +Step 7100/44000 | Loss: 3.2414 | LR: 4.74e-04 | 1300.5ms/step | 15.4 samples/s +Step 7110/44000 | Loss: 3.2158 | LR: 4.74e-04 | 1300.6ms/step | 15.4 samples/s +Step 7120/44000 | Loss: 3.2351 | LR: 4.74e-04 | 1300.3ms/step | 15.4 samples/s +Step 7130/44000 | Loss: 3.2474 | LR: 4.74e-04 | 1301.2ms/step | 15.4 samples/s +Step 7140/44000 | Loss: 3.2675 | LR: 4.74e-04 | 1301.4ms/step | 15.4 samples/s +Step 7150/44000 | Loss: 3.2483 | LR: 4.73e-04 | 1300.6ms/step | 15.4 samples/s +Step 7160/44000 | Loss: 3.2585 | LR: 4.73e-04 | 1301.0ms/step | 15.4 samples/s +Step 7170/44000 | Loss: 3.2598 | LR: 4.73e-04 | 1299.7ms/step | 15.4 samples/s +Step 7180/44000 | Loss: 3.2431 | LR: 4.73e-04 | 1301.0ms/step | 15.4 samples/s +Step 7190/44000 | Loss: 3.2042 | LR: 4.73e-04 | 1300.2ms/step | 15.4 samples/s +Step 7200/44000 | Loss: 3.2577 | LR: 4.73e-04 | 1301.1ms/step | 15.4 samples/s +Step 7210/44000 | Loss: 3.2776 | LR: 4.73e-04 | 1301.7ms/step | 15.4 samples/s +Step 7220/44000 | Loss: 3.2597 | LR: 4.73e-04 | 1302.3ms/step | 15.4 samples/s +Step 7230/44000 | Loss: 3.2575 | LR: 4.73e-04 | 1302.9ms/step | 15.4 samples/s +Step 7240/44000 | Loss: 3.2469 | LR: 4.73e-04 | 1300.2ms/step | 15.4 samples/s +Step 7250/44000 | Loss: 3.2507 | LR: 4.73e-04 | 1300.0ms/step | 15.4 samples/s +Validation at step 7250: Loss = 3.2493, Perplexity = 25.77 +Step 7260/44000 | Loss: 3.2318 | LR: 4.73e-04 | 3491.0ms/step | 5.7 samples/s +Step 7270/44000 | Loss: 3.2416 | LR: 4.72e-04 | 1299.8ms/step | 15.4 samples/s +Step 7280/44000 | Loss: 3.2224 | LR: 4.72e-04 | 1298.0ms/step | 15.4 samples/s +Step 7290/44000 | Loss: 3.2640 | LR: 4.72e-04 | 1298.7ms/step | 15.4 samples/s +Step 7300/44000 | Loss: 3.2448 | LR: 4.72e-04 | 1299.1ms/step | 15.4 samples/s +Step 7310/44000 | Loss: 3.2615 | LR: 4.72e-04 | 1299.4ms/step | 15.4 samples/s +Step 7320/44000 | Loss: 3.2628 | LR: 4.72e-04 | 1298.8ms/step | 15.4 samples/s +Step 7330/44000 | Loss: 3.2554 | LR: 4.72e-04 | 1299.1ms/step | 15.4 samples/s +Step 7340/44000 | Loss: 3.2644 | LR: 4.72e-04 | 1298.0ms/step | 15.4 samples/s +Step 7350/44000 | Loss: 3.2131 | LR: 4.72e-04 | 1298.0ms/step | 15.4 samples/s +Step 7360/44000 | Loss: 3.2209 | LR: 4.72e-04 | 1298.6ms/step | 15.4 samples/s +Step 7370/44000 | Loss: 3.2124 | LR: 4.72e-04 | 1297.7ms/step | 15.4 samples/s +Step 7380/44000 | Loss: 3.2428 | LR: 4.72e-04 | 1298.7ms/step | 15.4 samples/s +Step 7390/44000 | Loss: 3.2471 | LR: 4.71e-04 | 1299.5ms/step | 15.4 samples/s +Step 7400/44000 | Loss: 3.2376 | LR: 4.71e-04 | 1300.6ms/step | 15.4 samples/s +Step 7410/44000 | Loss: 3.2601 | LR: 4.71e-04 | 1298.4ms/step | 15.4 samples/s +Step 7420/44000 | Loss: 3.2648 | LR: 4.71e-04 | 1298.0ms/step | 15.4 samples/s +Step 7430/44000 | Loss: 3.2428 | LR: 4.71e-04 | 1296.8ms/step | 15.4 samples/s +Step 7440/44000 | Loss: 3.2448 | LR: 4.71e-04 | 1300.4ms/step | 15.4 samples/s +Step 7450/44000 | Loss: 3.2594 | LR: 4.71e-04 | 1300.1ms/step | 15.4 samples/s +Step 7460/44000 | Loss: 3.2037 | LR: 4.71e-04 | 1301.6ms/step | 15.4 samples/s +Step 7470/44000 | Loss: 3.2448 | LR: 4.71e-04 | 1304.6ms/step | 15.3 samples/s +Step 7480/44000 | Loss: 3.2007 | LR: 4.71e-04 | 1303.6ms/step | 15.3 samples/s +Step 7490/44000 | Loss: 3.2418 | LR: 4.71e-04 | 1301.0ms/step | 15.4 samples/s +Step 7500/44000 | Loss: 3.2045 | LR: 4.71e-04 | 1300.0ms/step | 15.4 samples/s +Validation at step 7500: Loss = 3.2415, Perplexity = 25.57 +Step 7510/44000 | Loss: 3.2176 | LR: 4.70e-04 | 3479.4ms/step | 5.7 samples/s +Step 7520/44000 | Loss: 3.2300 | LR: 4.70e-04 | 1300.6ms/step | 15.4 samples/s +Step 7530/44000 | Loss: 3.2612 | LR: 4.70e-04 | 1298.7ms/step | 15.4 samples/s +Step 7540/44000 | Loss: 3.2164 | LR: 4.70e-04 | 1298.9ms/step | 15.4 samples/s +Step 7550/44000 | Loss: 3.2225 | LR: 4.70e-04 | 1298.7ms/step | 15.4 samples/s +Step 7560/44000 | Loss: 3.2252 | LR: 4.70e-04 | 1298.6ms/step | 15.4 samples/s +Step 7570/44000 | Loss: 3.2559 | LR: 4.70e-04 | 1299.0ms/step | 15.4 samples/s +Step 7580/44000 | Loss: 3.2113 | LR: 4.70e-04 | 1300.1ms/step | 15.4 samples/s +Step 7590/44000 | Loss: 3.2554 | LR: 4.70e-04 | 1308.5ms/step | 15.3 samples/s +Step 7600/44000 | Loss: 3.2357 | LR: 4.70e-04 | 1299.4ms/step | 15.4 samples/s +Step 7610/44000 | Loss: 3.2538 | LR: 4.70e-04 | 1299.1ms/step | 15.4 samples/s +Step 7620/44000 | Loss: 3.2054 | LR: 4.70e-04 | 1298.3ms/step | 15.4 samples/s +Step 7630/44000 | Loss: 3.2621 | LR: 4.69e-04 | 1298.4ms/step | 15.4 samples/s +Step 7640/44000 | Loss: 3.2204 | LR: 4.69e-04 | 1301.4ms/step | 15.4 samples/s +Step 7650/44000 | Loss: 3.2320 | LR: 4.69e-04 | 1297.5ms/step | 15.4 samples/s +Step 7660/44000 | Loss: 3.2428 | LR: 4.69e-04 | 1298.2ms/step | 15.4 samples/s +Step 7670/44000 | Loss: 3.2219 | LR: 4.69e-04 | 1297.8ms/step | 15.4 samples/s +Step 7680/44000 | Loss: 3.2411 | LR: 4.69e-04 | 1298.0ms/step | 15.4 samples/s +Step 7690/44000 | Loss: 3.2038 | LR: 4.69e-04 | 1295.2ms/step | 15.4 samples/s +Step 7700/44000 | Loss: 3.2278 | LR: 4.69e-04 | 1294.5ms/step | 15.4 samples/s +Step 7710/44000 | Loss: 3.2525 | LR: 4.69e-04 | 1303.7ms/step | 15.3 samples/s +Step 7720/44000 | Loss: 3.2510 | LR: 4.69e-04 | 1303.2ms/step | 15.3 samples/s +Step 7730/44000 | Loss: 3.2463 | LR: 4.69e-04 | 1301.6ms/step | 15.4 samples/s +Step 7740/44000 | Loss: 3.2157 | LR: 4.68e-04 | 1300.3ms/step | 15.4 samples/s +Step 7750/44000 | Loss: 3.1942 | LR: 4.68e-04 | 1298.0ms/step | 15.4 samples/s +Validation at step 7750: Loss = 3.2317, Perplexity = 25.32 +Step 7760/44000 | Loss: 3.2047 | LR: 4.68e-04 | 3481.8ms/step | 5.7 samples/s +Step 7770/44000 | Loss: 3.2079 | LR: 4.68e-04 | 1299.2ms/step | 15.4 samples/s +Step 7780/44000 | Loss: 3.2041 | LR: 4.68e-04 | 1300.9ms/step | 15.4 samples/s +Step 7790/44000 | Loss: 3.2267 | LR: 4.68e-04 | 1300.8ms/step | 15.4 samples/s +Step 7800/44000 | Loss: 3.2172 | LR: 4.68e-04 | 1300.0ms/step | 15.4 samples/s +Step 7810/44000 | Loss: 3.2063 | LR: 4.68e-04 | 1299.0ms/step | 15.4 samples/s +Step 7820/44000 | Loss: 3.2208 | LR: 4.68e-04 | 1300.5ms/step | 15.4 samples/s +Step 7830/44000 | Loss: 3.2368 | LR: 4.68e-04 | 1301.0ms/step | 15.4 samples/s +Step 7840/44000 | Loss: 3.2159 | LR: 4.68e-04 | 1300.4ms/step | 15.4 samples/s +Step 7850/44000 | Loss: 3.1957 | LR: 4.67e-04 | 1302.4ms/step | 15.4 samples/s +Step 7860/44000 | Loss: 3.2141 | LR: 4.67e-04 | 1301.5ms/step | 15.4 samples/s +Step 7870/44000 | Loss: 3.2173 | LR: 4.67e-04 | 1298.3ms/step | 15.4 samples/s +Step 7880/44000 | Loss: 3.1767 | LR: 4.67e-04 | 1298.6ms/step | 15.4 samples/s +Step 7890/44000 | Loss: 3.1961 | LR: 4.67e-04 | 1297.5ms/step | 15.4 samples/s +Step 7900/44000 | Loss: 3.2443 | LR: 4.67e-04 | 1298.7ms/step | 15.4 samples/s +Step 7910/44000 | Loss: 3.1914 | LR: 4.67e-04 | 1298.6ms/step | 15.4 samples/s +Step 7920/44000 | Loss: 3.2295 | LR: 4.67e-04 | 1297.5ms/step | 15.4 samples/s +Step 7930/44000 | Loss: 3.2065 | LR: 4.67e-04 | 1297.5ms/step | 15.4 samples/s +Step 7940/44000 | Loss: 3.2237 | LR: 4.67e-04 | 1298.4ms/step | 15.4 samples/s +Step 7950/44000 | Loss: 3.2175 | LR: 4.67e-04 | 1298.7ms/step | 15.4 samples/s +Step 7960/44000 | Loss: 3.2116 | LR: 4.66e-04 | 1298.6ms/step | 15.4 samples/s +Step 7970/44000 | Loss: 3.2061 | LR: 4.66e-04 | 1298.0ms/step | 15.4 samples/s +Step 7980/44000 | Loss: 3.2308 | LR: 4.66e-04 | 1298.5ms/step | 15.4 samples/s +Step 7990/44000 | Loss: 3.2285 | LR: 4.66e-04 | 1297.1ms/step | 15.4 samples/s +Step 8000/44000 | Loss: 3.2003 | LR: 4.66e-04 | 1298.5ms/step | 15.4 samples/s +Validation at step 8000: Loss = 3.2236, Perplexity = 25.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-8000 +Step 8010/44000 | Loss: 3.1698 | LR: 4.66e-04 | 3773.3ms/step | 5.3 samples/s +Step 8020/44000 | Loss: 3.1928 | LR: 4.66e-04 | 1294.5ms/step | 15.4 samples/s +Step 8030/44000 | Loss: 3.1925 | LR: 4.66e-04 | 1294.9ms/step | 15.4 samples/s +Step 8040/44000 | Loss: 3.2190 | LR: 4.66e-04 | 1296.0ms/step | 15.4 samples/s +Step 8050/44000 | Loss: 3.2010 | LR: 4.66e-04 | 1297.7ms/step | 15.4 samples/s +Step 8060/44000 | Loss: 3.2346 | LR: 4.66e-04 | 1301.4ms/step | 15.4 samples/s +Step 8070/44000 | Loss: 3.1988 | LR: 4.65e-04 | 1303.6ms/step | 15.3 samples/s +Step 8080/44000 | Loss: 3.1900 | LR: 4.65e-04 | 1300.9ms/step | 15.4 samples/s +Step 8090/44000 | Loss: 3.1928 | LR: 4.65e-04 | 1301.9ms/step | 15.4 samples/s +Step 8100/44000 | Loss: 3.2152 | LR: 4.65e-04 | 1303.4ms/step | 15.3 samples/s +Step 8110/44000 | Loss: 3.2100 | LR: 4.65e-04 | 1301.5ms/step | 15.4 samples/s +Step 8120/44000 | Loss: 3.2020 | LR: 4.65e-04 | 1302.6ms/step | 15.4 samples/s +Step 8130/44000 | Loss: 3.2025 | LR: 4.65e-04 | 1305.8ms/step | 15.3 samples/s +Step 8140/44000 | Loss: 3.2358 | LR: 4.65e-04 | 1303.0ms/step | 15.3 samples/s +Step 8150/44000 | Loss: 3.2002 | LR: 4.65e-04 | 1302.8ms/step | 15.4 samples/s +Step 8160/44000 | Loss: 3.1928 | LR: 4.65e-04 | 1303.5ms/step | 15.3 samples/s +Step 8170/44000 | Loss: 3.2181 | LR: 4.65e-04 | 1301.9ms/step | 15.4 samples/s +Step 8180/44000 | Loss: 3.2055 | LR: 4.64e-04 | 1303.0ms/step | 15.3 samples/s +Step 8190/44000 | Loss: 3.1989 | LR: 4.64e-04 | 1307.5ms/step | 15.3 samples/s +Step 8200/44000 | Loss: 3.2247 | LR: 4.64e-04 | 1306.8ms/step | 15.3 samples/s +Step 8210/44000 | Loss: 3.2233 | LR: 4.64e-04 | 1306.0ms/step | 15.3 samples/s +Step 8220/44000 | Loss: 3.2304 | LR: 4.64e-04 | 1305.3ms/step | 15.3 samples/s +Step 8230/44000 | Loss: 3.1902 | LR: 4.64e-04 | 1305.5ms/step | 15.3 samples/s +Step 8240/44000 | Loss: 3.2026 | LR: 4.64e-04 | 1308.3ms/step | 15.3 samples/s +Step 8250/44000 | Loss: 3.1992 | LR: 4.64e-04 | 1302.1ms/step | 15.4 samples/s +Validation at step 8250: Loss = 3.2159, Perplexity = 24.93 +Step 8260/44000 | Loss: 3.1850 | LR: 4.64e-04 | 3488.5ms/step | 5.7 samples/s +Step 8270/44000 | Loss: 3.2283 | LR: 4.64e-04 | 1303.0ms/step | 15.3 samples/s +Step 8280/44000 | Loss: 3.2075 | LR: 4.64e-04 | 1303.0ms/step | 15.3 samples/s +Step 8290/44000 | Loss: 3.2094 | LR: 4.63e-04 | 1303.9ms/step | 15.3 samples/s +Step 8300/44000 | Loss: 3.1642 | LR: 4.63e-04 | 1301.7ms/step | 15.4 samples/s +Step 8310/44000 | Loss: 3.2168 | LR: 4.63e-04 | 1307.0ms/step | 15.3 samples/s +Step 8320/44000 | Loss: 3.1923 | LR: 4.63e-04 | 1306.4ms/step | 15.3 samples/s +Step 8330/44000 | Loss: 3.1985 | LR: 4.63e-04 | 1302.6ms/step | 15.4 samples/s +Step 8340/44000 | Loss: 3.2453 | LR: 4.63e-04 | 1302.6ms/step | 15.4 samples/s +Step 8350/44000 | Loss: 3.1919 | LR: 4.63e-04 | 1300.9ms/step | 15.4 samples/s +Step 8360/44000 | Loss: 3.2186 | LR: 4.63e-04 | 1300.2ms/step | 15.4 samples/s +Step 8370/44000 | Loss: 3.2029 | LR: 4.63e-04 | 1307.2ms/step | 15.3 samples/s +Step 8380/44000 | Loss: 3.2417 | LR: 4.63e-04 | 1308.4ms/step | 15.3 samples/s +Step 8390/44000 | Loss: 3.1760 | LR: 4.62e-04 | 1309.7ms/step | 15.3 samples/s +Step 8400/44000 | Loss: 3.1913 | LR: 4.62e-04 | 1310.8ms/step | 15.3 samples/s +Step 8410/44000 | Loss: 3.1887 | LR: 4.62e-04 | 1302.4ms/step | 15.4 samples/s +Step 8420/44000 | Loss: 3.2294 | LR: 4.62e-04 | 1305.5ms/step | 15.3 samples/s +Step 8430/44000 | Loss: 3.1934 | LR: 4.62e-04 | 1306.1ms/step | 15.3 samples/s +Step 8440/44000 | Loss: 3.2208 | LR: 4.62e-04 | 1307.0ms/step | 15.3 samples/s +Step 8450/44000 | Loss: 3.1714 | LR: 4.62e-04 | 1307.5ms/step | 15.3 samples/s +Step 8460/44000 | Loss: 3.2196 | LR: 4.62e-04 | 1308.5ms/step | 15.3 samples/s +Step 8470/44000 | Loss: 3.1670 | LR: 4.62e-04 | 1309.4ms/step | 15.3 samples/s +Step 8480/44000 | Loss: 3.1934 | LR: 4.62e-04 | 1310.9ms/step | 15.3 samples/s +Step 8490/44000 | Loss: 3.2144 | LR: 4.62e-04 | 1308.0ms/step | 15.3 samples/s +Step 8500/44000 | Loss: 3.2218 | LR: 4.61e-04 | 1307.1ms/step | 15.3 samples/s +Validation at step 8500: Loss = 3.2089, Perplexity = 24.75 +Step 8510/44000 | Loss: 3.2196 | LR: 4.61e-04 | 3494.4ms/step | 5.7 samples/s +Step 8520/44000 | Loss: 3.1921 | LR: 4.61e-04 | 1303.9ms/step | 15.3 samples/s +Step 8530/44000 | Loss: 3.2183 | LR: 4.61e-04 | 1304.1ms/step | 15.3 samples/s +Step 8540/44000 | Loss: 3.1766 | LR: 4.61e-04 | 1303.4ms/step | 15.3 samples/s +Step 8550/44000 | Loss: 3.1968 | LR: 4.61e-04 | 1307.0ms/step | 15.3 samples/s +Step 8560/44000 | Loss: 3.2041 | LR: 4.61e-04 | 1305.6ms/step | 15.3 samples/s +Step 8570/44000 | Loss: 3.1717 | LR: 4.61e-04 | 1308.3ms/step | 15.3 samples/s +Step 8580/44000 | Loss: 3.2011 | LR: 4.61e-04 | 1308.7ms/step | 15.3 samples/s +Step 8590/44000 | Loss: 3.2004 | LR: 4.61e-04 | 1306.6ms/step | 15.3 samples/s +Step 8600/44000 | Loss: 3.1903 | LR: 4.60e-04 | 1306.0ms/step | 15.3 samples/s +Step 8610/44000 | Loss: 3.1689 | LR: 4.60e-04 | 1303.7ms/step | 15.3 samples/s +Step 8620/44000 | Loss: 3.1978 | LR: 4.60e-04 | 1299.6ms/step | 15.4 samples/s +Step 8630/44000 | Loss: 3.1880 | LR: 4.60e-04 | 1299.2ms/step | 15.4 samples/s +Step 8640/44000 | Loss: 3.1944 | LR: 4.60e-04 | 1302.7ms/step | 15.4 samples/s +Step 8650/44000 | Loss: 3.1993 | LR: 4.60e-04 | 1299.5ms/step | 15.4 samples/s +Step 8660/44000 | Loss: 3.1611 | LR: 4.60e-04 | 1300.6ms/step | 15.4 samples/s +Step 8670/44000 | Loss: 3.2022 | LR: 4.60e-04 | 1300.8ms/step | 15.4 samples/s +Step 8680/44000 | Loss: 3.2015 | LR: 4.60e-04 | 1298.4ms/step | 15.4 samples/s +Step 8690/44000 | Loss: 3.1803 | LR: 4.60e-04 | 1298.1ms/step | 15.4 samples/s +Step 8700/44000 | Loss: 3.1580 | LR: 4.59e-04 | 1300.3ms/step | 15.4 samples/s +Step 8710/44000 | Loss: 3.1905 | LR: 4.59e-04 | 1299.0ms/step | 15.4 samples/s +Step 8720/44000 | Loss: 3.1939 | LR: 4.59e-04 | 1297.6ms/step | 15.4 samples/s +Step 8730/44000 | Loss: 3.1970 | LR: 4.59e-04 | 1298.0ms/step | 15.4 samples/s +Step 8740/44000 | Loss: 3.1889 | LR: 4.59e-04 | 1297.4ms/step | 15.4 samples/s +Step 8750/44000 | Loss: 3.1629 | LR: 4.59e-04 | 1299.0ms/step | 15.4 samples/s +Validation at step 8750: Loss = 3.2028, Perplexity = 24.60 +Step 8760/44000 | Loss: 3.1772 | LR: 4.59e-04 | 3485.5ms/step | 5.7 samples/s +Step 8770/44000 | Loss: 3.2170 | LR: 4.59e-04 | 1304.1ms/step | 15.3 samples/s +Step 8780/44000 | Loss: 3.1817 | LR: 4.59e-04 | 1300.9ms/step | 15.4 samples/s +Step 8790/44000 | Loss: 3.1707 | LR: 4.59e-04 | 1301.1ms/step | 15.4 samples/s +Step 8800/44000 | Loss: 3.1644 | LR: 4.58e-04 | 1301.0ms/step | 15.4 samples/s +Step 8810/44000 | Loss: 3.1868 | LR: 4.58e-04 | 1299.7ms/step | 15.4 samples/s +Step 8820/44000 | Loss: 3.1706 | LR: 4.58e-04 | 1299.3ms/step | 15.4 samples/s +Step 8830/44000 | Loss: 3.1755 | LR: 4.58e-04 | 1298.1ms/step | 15.4 samples/s +Step 8840/44000 | Loss: 3.1858 | LR: 4.58e-04 | 1301.8ms/step | 15.4 samples/s +Step 8850/44000 | Loss: 3.1854 | LR: 4.58e-04 | 1300.2ms/step | 15.4 samples/s +Step 8860/44000 | Loss: 3.2044 | LR: 4.58e-04 | 1301.4ms/step | 15.4 samples/s +Step 8870/44000 | Loss: 3.1864 | LR: 4.58e-04 | 1305.3ms/step | 15.3 samples/s +Step 8880/44000 | Loss: 3.1649 | LR: 4.58e-04 | 1299.5ms/step | 15.4 samples/s +Step 8890/44000 | Loss: 3.2107 | LR: 4.58e-04 | 1298.9ms/step | 15.4 samples/s +Step 8900/44000 | Loss: 3.1728 | LR: 4.57e-04 | 1300.4ms/step | 15.4 samples/s +Step 8910/44000 | Loss: 3.2090 | LR: 4.57e-04 | 1300.7ms/step | 15.4 samples/s +Step 8920/44000 | Loss: 3.1885 | LR: 4.57e-04 | 1300.9ms/step | 15.4 samples/s +Step 8930/44000 | Loss: 3.1862 | LR: 4.57e-04 | 1298.4ms/step | 15.4 samples/s +Step 8940/44000 | Loss: 3.1776 | LR: 4.57e-04 | 1297.5ms/step | 15.4 samples/s +Step 8950/44000 | Loss: 3.1850 | LR: 4.57e-04 | 1298.1ms/step | 15.4 samples/s +Step 8960/44000 | Loss: 3.1675 | LR: 4.57e-04 | 1298.0ms/step | 15.4 samples/s +Step 8970/44000 | Loss: 3.1900 | LR: 4.57e-04 | 1300.0ms/step | 15.4 samples/s +Step 8980/44000 | Loss: 3.1889 | LR: 4.57e-04 | 1299.4ms/step | 15.4 samples/s +Step 8990/44000 | Loss: 3.1747 | LR: 4.57e-04 | 1301.3ms/step | 15.4 samples/s +Step 9000/44000 | Loss: 3.1760 | LR: 4.56e-04 | 1301.8ms/step | 15.4 samples/s +Validation at step 9000: Loss = 3.1950, Perplexity = 24.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-9000 +Step 9010/44000 | Loss: 3.1755 | LR: 4.56e-04 | 3772.8ms/step | 5.3 samples/s +Step 9020/44000 | Loss: 3.1934 | LR: 4.56e-04 | 1302.3ms/step | 15.4 samples/s +Step 9030/44000 | Loss: 3.1671 | LR: 4.56e-04 | 1302.7ms/step | 15.4 samples/s +Step 9040/44000 | Loss: 3.1714 | LR: 4.56e-04 | 1305.4ms/step | 15.3 samples/s +Step 9050/44000 | Loss: 3.1719 | LR: 4.56e-04 | 1307.8ms/step | 15.3 samples/s +Step 9060/44000 | Loss: 3.2100 | LR: 4.56e-04 | 1308.5ms/step | 15.3 samples/s +Step 9070/44000 | Loss: 3.2040 | LR: 4.56e-04 | 1303.8ms/step | 15.3 samples/s +Step 9080/44000 | Loss: 3.2267 | LR: 4.56e-04 | 1302.5ms/step | 15.4 samples/s +Step 9090/44000 | Loss: 3.1557 | LR: 4.56e-04 | 1305.3ms/step | 15.3 samples/s +Step 9100/44000 | Loss: 3.1608 | LR: 4.55e-04 | 1308.5ms/step | 15.3 samples/s +Step 9110/44000 | Loss: 3.1574 | LR: 4.55e-04 | 1306.7ms/step | 15.3 samples/s +Step 9120/44000 | Loss: 3.1602 | LR: 4.55e-04 | 1307.8ms/step | 15.3 samples/s +Step 9130/44000 | Loss: 3.2054 | LR: 4.55e-04 | 1311.7ms/step | 15.2 samples/s +Step 9140/44000 | Loss: 3.1543 | LR: 4.55e-04 | 1304.4ms/step | 15.3 samples/s +Step 9150/44000 | Loss: 3.1828 | LR: 4.55e-04 | 1300.5ms/step | 15.4 samples/s +Step 9160/44000 | Loss: 3.1571 | LR: 4.55e-04 | 1303.1ms/step | 15.3 samples/s +Step 9170/44000 | Loss: 3.1394 | LR: 4.55e-04 | 1302.8ms/step | 15.4 samples/s +Step 9180/44000 | Loss: 3.1868 | LR: 4.55e-04 | 1303.5ms/step | 15.3 samples/s +Step 9190/44000 | Loss: 3.1623 | LR: 4.54e-04 | 1305.6ms/step | 15.3 samples/s +Step 9200/44000 | Loss: 3.1891 | LR: 4.54e-04 | 1301.1ms/step | 15.4 samples/s +Step 9210/44000 | Loss: 3.1837 | LR: 4.54e-04 | 1301.9ms/step | 15.4 samples/s +Step 9220/44000 | Loss: 3.1734 | LR: 4.54e-04 | 1305.3ms/step | 15.3 samples/s +Step 9230/44000 | Loss: 3.1750 | LR: 4.54e-04 | 1307.2ms/step | 15.3 samples/s +Step 9240/44000 | Loss: 3.2245 | LR: 4.54e-04 | 1311.5ms/step | 15.2 samples/s +Step 9250/44000 | Loss: 3.1791 | LR: 4.54e-04 | 1308.6ms/step | 15.3 samples/s +Validation at step 9250: Loss = 3.1889, Perplexity = 24.26 +Step 9260/44000 | Loss: 3.1841 | LR: 4.54e-04 | 3491.6ms/step | 5.7 samples/s +Step 9270/44000 | Loss: 3.1819 | LR: 4.54e-04 | 1304.9ms/step | 15.3 samples/s +Step 9280/44000 | Loss: 3.1812 | LR: 4.54e-04 | 1302.9ms/step | 15.4 samples/s +Step 9290/44000 | Loss: 3.1928 | LR: 4.53e-04 | 1301.8ms/step | 15.4 samples/s +Step 9300/44000 | Loss: 3.1526 | LR: 4.53e-04 | 1303.1ms/step | 15.3 samples/s +Step 9310/44000 | Loss: 3.1725 | LR: 4.53e-04 | 1298.1ms/step | 15.4 samples/s +Step 9320/44000 | Loss: 3.1572 | LR: 4.53e-04 | 1296.6ms/step | 15.4 samples/s +Step 9330/44000 | Loss: 3.1776 | LR: 4.53e-04 | 1297.8ms/step | 15.4 samples/s +Step 9340/44000 | Loss: 3.1596 | LR: 4.53e-04 | 1297.4ms/step | 15.4 samples/s +Step 9350/44000 | Loss: 3.1507 | LR: 4.53e-04 | 1295.4ms/step | 15.4 samples/s +Step 9360/44000 | Loss: 3.1766 | LR: 4.53e-04 | 1297.1ms/step | 15.4 samples/s +Step 9370/44000 | Loss: 3.1788 | LR: 4.53e-04 | 1296.7ms/step | 15.4 samples/s +Step 9380/44000 | Loss: 3.1767 | LR: 4.52e-04 | 1297.2ms/step | 15.4 samples/s +Step 9390/44000 | Loss: 3.1873 | LR: 4.52e-04 | 1296.7ms/step | 15.4 samples/s +Step 9400/44000 | Loss: 3.1665 | LR: 4.52e-04 | 1297.8ms/step | 15.4 samples/s +Step 9410/44000 | Loss: 3.1704 | LR: 4.52e-04 | 1297.1ms/step | 15.4 samples/s +Step 9420/44000 | Loss: 3.1819 | LR: 4.52e-04 | 1297.2ms/step | 15.4 samples/s +Step 9430/44000 | Loss: 3.1669 | LR: 4.52e-04 | 1295.9ms/step | 15.4 samples/s +Step 9440/44000 | Loss: 3.1444 | LR: 4.52e-04 | 1296.9ms/step | 15.4 samples/s +Step 9450/44000 | Loss: 3.1677 | LR: 4.52e-04 | 1298.1ms/step | 15.4 samples/s +Step 9460/44000 | Loss: 3.1806 | LR: 4.52e-04 | 1298.3ms/step | 15.4 samples/s +Step 9470/44000 | Loss: 3.1613 | LR: 4.52e-04 | 1299.9ms/step | 15.4 samples/s +Step 9480/44000 | Loss: 3.1577 | LR: 4.51e-04 | 1299.5ms/step | 15.4 samples/s +Step 9490/44000 | Loss: 3.1725 | LR: 4.51e-04 | 1297.9ms/step | 15.4 samples/s +Step 9500/44000 | Loss: 3.1745 | LR: 4.51e-04 | 1298.5ms/step | 15.4 samples/s +Validation at step 9500: Loss = 3.1824, Perplexity = 24.11 +Step 9510/44000 | Loss: 3.2050 | LR: 4.51e-04 | 3490.4ms/step | 5.7 samples/s +Step 9520/44000 | Loss: 3.1576 | LR: 4.51e-04 | 1310.2ms/step | 15.3 samples/s +Step 9530/44000 | Loss: 3.1975 | LR: 4.51e-04 | 1310.9ms/step | 15.3 samples/s +Step 9540/44000 | Loss: 3.1965 | LR: 4.51e-04 | 1309.2ms/step | 15.3 samples/s +Step 9550/44000 | Loss: 3.1819 | LR: 4.51e-04 | 1310.7ms/step | 15.3 samples/s +Step 9560/44000 | Loss: 3.1617 | LR: 4.51e-04 | 1313.8ms/step | 15.2 samples/s +Step 9570/44000 | Loss: 3.1582 | LR: 4.50e-04 | 1304.1ms/step | 15.3 samples/s +Step 9580/44000 | Loss: 3.1885 | LR: 4.50e-04 | 1301.6ms/step | 15.4 samples/s +Step 9590/44000 | Loss: 3.1530 | LR: 4.50e-04 | 1301.7ms/step | 15.4 samples/s +Step 9600/44000 | Loss: 3.1554 | LR: 4.50e-04 | 1299.8ms/step | 15.4 samples/s +Step 9610/44000 | Loss: 3.1574 | LR: 4.50e-04 | 1299.7ms/step | 15.4 samples/s +Step 9620/44000 | Loss: 3.1978 | LR: 4.50e-04 | 1299.2ms/step | 15.4 samples/s +Step 9630/44000 | Loss: 3.1742 | LR: 4.50e-04 | 1299.8ms/step | 15.4 samples/s +Step 9640/44000 | Loss: 3.1700 | LR: 4.50e-04 | 1301.6ms/step | 15.4 samples/s +Step 9650/44000 | Loss: 3.1731 | LR: 4.50e-04 | 1301.3ms/step | 15.4 samples/s +Step 9660/44000 | Loss: 3.1467 | LR: 4.49e-04 | 1301.9ms/step | 15.4 samples/s +Step 9670/44000 | Loss: 3.1707 | LR: 4.49e-04 | 1298.3ms/step | 15.4 samples/s +Step 9680/44000 | Loss: 3.1755 | LR: 4.49e-04 | 1297.6ms/step | 15.4 samples/s +Step 9690/44000 | Loss: 3.1726 | LR: 4.49e-04 | 1297.9ms/step | 15.4 samples/s +Step 9700/44000 | Loss: 3.1760 | LR: 4.49e-04 | 1296.6ms/step | 15.4 samples/s +Step 9710/44000 | Loss: 3.1738 | LR: 4.49e-04 | 1297.3ms/step | 15.4 samples/s +Step 9720/44000 | Loss: 3.1608 | LR: 4.49e-04 | 1300.9ms/step | 15.4 samples/s +Step 9730/44000 | Loss: 3.1766 | LR: 4.49e-04 | 1299.9ms/step | 15.4 samples/s +Step 9740/44000 | Loss: 3.1636 | LR: 4.49e-04 | 1307.1ms/step | 15.3 samples/s +Step 9750/44000 | Loss: 3.1586 | LR: 4.48e-04 | 1302.6ms/step | 15.4 samples/s +Validation at step 9750: Loss = 3.1765, Perplexity = 23.96 +Step 9760/44000 | Loss: 3.1358 | LR: 4.48e-04 | 3476.8ms/step | 5.8 samples/s +Step 9770/44000 | Loss: 3.1390 | LR: 4.48e-04 | 1294.7ms/step | 15.4 samples/s +Step 9780/44000 | Loss: 3.1647 | LR: 4.48e-04 | 1296.7ms/step | 15.4 samples/s +Step 9790/44000 | Loss: 3.1811 | LR: 4.48e-04 | 1297.1ms/step | 15.4 samples/s +Step 9800/44000 | Loss: 3.1636 | LR: 4.48e-04 | 1297.6ms/step | 15.4 samples/s +Step 9810/44000 | Loss: 3.1822 | LR: 4.48e-04 | 1296.7ms/step | 15.4 samples/s +Step 9820/44000 | Loss: 3.1811 | LR: 4.48e-04 | 1297.1ms/step | 15.4 samples/s +Step 9830/44000 | Loss: 3.2093 | LR: 4.48e-04 | 1298.4ms/step | 15.4 samples/s +Step 9840/44000 | Loss: 3.1558 | LR: 4.47e-04 | 1298.3ms/step | 15.4 samples/s +Step 9850/44000 | Loss: 3.1749 | LR: 4.47e-04 | 1296.5ms/step | 15.4 samples/s +Step 9860/44000 | Loss: 3.1368 | LR: 4.47e-04 | 1300.5ms/step | 15.4 samples/s +Step 9870/44000 | Loss: 3.1600 | LR: 4.47e-04 | 1298.5ms/step | 15.4 samples/s +Step 9880/44000 | Loss: 3.1633 | LR: 4.47e-04 | 1296.8ms/step | 15.4 samples/s +Step 9890/44000 | Loss: 3.1445 | LR: 4.47e-04 | 1298.4ms/step | 15.4 samples/s +Step 9900/44000 | Loss: 3.1686 | LR: 4.47e-04 | 1297.6ms/step | 15.4 samples/s +Step 9910/44000 | Loss: 3.1842 | LR: 4.47e-04 | 1297.3ms/step | 15.4 samples/s +Step 9920/44000 | Loss: 3.1469 | LR: 4.47e-04 | 1296.5ms/step | 15.4 samples/s +Step 9930/44000 | Loss: 3.1420 | LR: 4.46e-04 | 1295.9ms/step | 15.4 samples/s +Step 9940/44000 | Loss: 3.1643 | LR: 4.46e-04 | 1297.4ms/step | 15.4 samples/s +Step 9950/44000 | Loss: 3.1476 | LR: 4.46e-04 | 1296.8ms/step | 15.4 samples/s +Step 9960/44000 | Loss: 3.1616 | LR: 4.46e-04 | 1299.5ms/step | 15.4 samples/s +Step 9970/44000 | Loss: 3.2042 | LR: 4.46e-04 | 1299.1ms/step | 15.4 samples/s +Step 9980/44000 | Loss: 3.1546 | LR: 4.46e-04 | 1299.9ms/step | 15.4 samples/s +Step 9990/44000 | Loss: 3.1643 | LR: 4.46e-04 | 1299.5ms/step | 15.4 samples/s +Step 10000/44000 | Loss: 3.1584 | LR: 4.46e-04 | 1300.6ms/step | 15.4 samples/s +Validation at step 10000: Loss = 3.1706, Perplexity = 23.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-10000 +Step 10010/44000 | Loss: 3.1539 | LR: 4.46e-04 | 3779.6ms/step | 5.3 samples/s +Step 10020/44000 | Loss: 3.1831 | LR: 4.45e-04 | 1296.0ms/step | 15.4 samples/s +Step 10030/44000 | Loss: 3.1647 | LR: 4.45e-04 | 1295.0ms/step | 15.4 samples/s +Step 10040/44000 | Loss: 3.1326 | LR: 4.45e-04 | 1295.0ms/step | 15.4 samples/s +Step 10050/44000 | Loss: 3.1429 | LR: 4.45e-04 | 1294.8ms/step | 15.4 samples/s +Step 10060/44000 | Loss: 3.1414 | LR: 4.45e-04 | 1299.3ms/step | 15.4 samples/s +Step 10070/44000 | Loss: 3.1444 | LR: 4.45e-04 | 1299.0ms/step | 15.4 samples/s +Step 10080/44000 | Loss: 3.1539 | LR: 4.45e-04 | 1300.7ms/step | 15.4 samples/s +Step 10090/44000 | Loss: 3.1516 | LR: 4.45e-04 | 1301.7ms/step | 15.4 samples/s +Step 10100/44000 | Loss: 3.1695 | LR: 4.45e-04 | 1300.8ms/step | 15.4 samples/s +Step 10110/44000 | Loss: 3.1782 | LR: 4.44e-04 | 1300.1ms/step | 15.4 samples/s +Step 10120/44000 | Loss: 3.1996 | LR: 4.44e-04 | 1301.1ms/step | 15.4 samples/s +Step 10130/44000 | Loss: 3.1468 | LR: 4.44e-04 | 1301.6ms/step | 15.4 samples/s +Step 10140/44000 | Loss: 3.2060 | LR: 4.44e-04 | 1299.0ms/step | 15.4 samples/s +Step 10150/44000 | Loss: 3.1633 | LR: 4.44e-04 | 1299.0ms/step | 15.4 samples/s +Step 10160/44000 | Loss: 3.1639 | LR: 4.44e-04 | 1296.5ms/step | 15.4 samples/s +Step 10170/44000 | Loss: 3.1351 | LR: 4.44e-04 | 1297.0ms/step | 15.4 samples/s +Step 10180/44000 | Loss: 3.1705 | LR: 4.44e-04 | 1298.5ms/step | 15.4 samples/s +Step 10190/44000 | Loss: 3.1403 | LR: 4.43e-04 | 1298.9ms/step | 15.4 samples/s +Step 10200/44000 | Loss: 3.1423 | LR: 4.43e-04 | 1300.6ms/step | 15.4 samples/s +Step 10210/44000 | Loss: 3.1592 | LR: 4.43e-04 | 1300.9ms/step | 15.4 samples/s +Step 10220/44000 | Loss: 3.1452 | LR: 4.43e-04 | 1302.6ms/step | 15.4 samples/s +Step 10230/44000 | Loss: 3.1674 | LR: 4.43e-04 | 1301.3ms/step | 15.4 samples/s +Step 10240/44000 | Loss: 3.1451 | LR: 4.43e-04 | 1301.7ms/step | 15.4 samples/s +Step 10250/44000 | Loss: 3.1665 | LR: 4.43e-04 | 1302.2ms/step | 15.4 samples/s +Validation at step 10250: Loss = 3.1652, Perplexity = 23.69 +Step 10260/44000 | Loss: 3.1246 | LR: 4.43e-04 | 3493.1ms/step | 5.7 samples/s +Step 10270/44000 | Loss: 3.1134 | LR: 4.43e-04 | 1302.3ms/step | 15.4 samples/s +Step 10280/44000 | Loss: 3.1150 | LR: 4.42e-04 | 1304.4ms/step | 15.3 samples/s +Step 10290/44000 | Loss: 3.1100 | LR: 4.42e-04 | 1302.3ms/step | 15.4 samples/s +Step 10300/44000 | Loss: 3.1417 | LR: 4.42e-04 | 1301.5ms/step | 15.4 samples/s +Step 10310/44000 | Loss: 3.1675 | LR: 4.42e-04 | 1301.6ms/step | 15.4 samples/s +Step 10320/44000 | Loss: 3.1330 | LR: 4.42e-04 | 1301.0ms/step | 15.4 samples/s +Step 10330/44000 | Loss: 3.1751 | LR: 4.42e-04 | 1300.7ms/step | 15.4 samples/s +Step 10340/44000 | Loss: 3.1494 | LR: 4.42e-04 | 1299.9ms/step | 15.4 samples/s +Step 10350/44000 | Loss: 3.1616 | LR: 4.42e-04 | 1302.8ms/step | 15.4 samples/s +Step 10360/44000 | Loss: 3.1325 | LR: 4.42e-04 | 1304.0ms/step | 15.3 samples/s +Step 10370/44000 | Loss: 3.1616 | LR: 4.41e-04 | 1303.1ms/step | 15.3 samples/s +Step 10380/44000 | Loss: 3.1722 | LR: 4.41e-04 | 1302.7ms/step | 15.4 samples/s +Step 10390/44000 | Loss: 3.1423 | LR: 4.41e-04 | 1301.7ms/step | 15.4 samples/s +Step 10400/44000 | Loss: 3.1740 | LR: 4.41e-04 | 1300.8ms/step | 15.4 samples/s +Step 10410/44000 | Loss: 3.1804 | LR: 4.41e-04 | 1302.4ms/step | 15.4 samples/s +Step 10420/44000 | Loss: 3.1738 | LR: 4.41e-04 | 1300.9ms/step | 15.4 samples/s +Step 10430/44000 | Loss: 3.1658 | LR: 4.41e-04 | 1301.8ms/step | 15.4 samples/s +Step 10440/44000 | Loss: 3.1349 | LR: 4.41e-04 | 1303.8ms/step | 15.3 samples/s +Step 10450/44000 | Loss: 3.1459 | LR: 4.40e-04 | 1299.1ms/step | 15.4 samples/s +Step 10460/44000 | Loss: 3.1415 | LR: 4.40e-04 | 1298.0ms/step | 15.4 samples/s +Step 10470/44000 | Loss: 3.1708 | LR: 4.40e-04 | 1298.2ms/step | 15.4 samples/s +Step 10480/44000 | Loss: 3.1403 | LR: 4.40e-04 | 1299.0ms/step | 15.4 samples/s +Step 10490/44000 | Loss: 3.1218 | LR: 4.40e-04 | 1299.4ms/step | 15.4 samples/s +Step 10500/44000 | Loss: 3.1744 | LR: 4.40e-04 | 1298.4ms/step | 15.4 samples/s +Validation at step 10500: Loss = 3.1613, Perplexity = 23.60 +Step 10510/44000 | Loss: 3.1839 | LR: 4.40e-04 | 3476.2ms/step | 5.8 samples/s +Step 10520/44000 | Loss: 3.1247 | LR: 4.40e-04 | 1298.2ms/step | 15.4 samples/s +Step 10530/44000 | Loss: 3.1413 | LR: 4.40e-04 | 1297.2ms/step | 15.4 samples/s +Step 10540/44000 | Loss: 3.1526 | LR: 4.39e-04 | 1297.7ms/step | 15.4 samples/s +Step 10550/44000 | Loss: 3.1428 | LR: 4.39e-04 | 1298.7ms/step | 15.4 samples/s +Step 10560/44000 | Loss: 3.1299 | LR: 4.39e-04 | 1300.2ms/step | 15.4 samples/s +Step 10570/44000 | Loss: 3.1454 | LR: 4.39e-04 | 1299.3ms/step | 15.4 samples/s +Step 10580/44000 | Loss: 3.1622 | LR: 4.39e-04 | 1298.1ms/step | 15.4 samples/s +Step 10590/44000 | Loss: 3.1114 | LR: 4.39e-04 | 1296.8ms/step | 15.4 samples/s +Step 10600/44000 | Loss: 3.1325 | LR: 4.39e-04 | 1301.5ms/step | 15.4 samples/s +Step 10610/44000 | Loss: 3.1221 | LR: 4.39e-04 | 1300.8ms/step | 15.4 samples/s +Step 10620/44000 | Loss: 3.1674 | LR: 4.38e-04 | 1301.7ms/step | 15.4 samples/s +Step 10630/44000 | Loss: 3.1565 | LR: 4.38e-04 | 1300.9ms/step | 15.4 samples/s +Step 10640/44000 | Loss: 3.1360 | LR: 4.38e-04 | 1298.8ms/step | 15.4 samples/s +Step 10650/44000 | Loss: 3.1519 | LR: 4.38e-04 | 1298.7ms/step | 15.4 samples/s +Step 10660/44000 | Loss: 3.1383 | LR: 4.38e-04 | 1299.6ms/step | 15.4 samples/s +Step 10670/44000 | Loss: 3.1421 | LR: 4.38e-04 | 1298.0ms/step | 15.4 samples/s +Step 10680/44000 | Loss: 3.1595 | LR: 4.38e-04 | 1297.1ms/step | 15.4 samples/s +Step 10690/44000 | Loss: 3.1108 | LR: 4.38e-04 | 1298.4ms/step | 15.4 samples/s +Step 10700/44000 | Loss: 3.0828 | LR: 4.37e-04 | 1299.5ms/step | 15.4 samples/s +Step 10710/44000 | Loss: 3.1177 | LR: 4.37e-04 | 1300.7ms/step | 15.4 samples/s +Step 10720/44000 | Loss: 3.1062 | LR: 4.37e-04 | 1299.8ms/step | 15.4 samples/s +Step 10730/44000 | Loss: 3.1175 | LR: 4.37e-04 | 1302.0ms/step | 15.4 samples/s +Step 10740/44000 | Loss: 3.1296 | LR: 4.37e-04 | 1300.6ms/step | 15.4 samples/s +Step 10750/44000 | Loss: 3.1208 | LR: 4.37e-04 | 1301.5ms/step | 15.4 samples/s +Validation at step 10750: Loss = 3.1548, Perplexity = 23.45 +Step 10760/44000 | Loss: 3.1578 | LR: 4.37e-04 | 3491.9ms/step | 5.7 samples/s +Step 10770/44000 | Loss: 3.1592 | LR: 4.37e-04 | 1302.9ms/step | 15.4 samples/s +Step 10780/44000 | Loss: 3.1336 | LR: 4.37e-04 | 1302.0ms/step | 15.4 samples/s +Step 10790/44000 | Loss: 3.1432 | LR: 4.36e-04 | 1300.9ms/step | 15.4 samples/s +Step 10800/44000 | Loss: 3.1270 | LR: 4.36e-04 | 1299.9ms/step | 15.4 samples/s +Step 10810/44000 | Loss: 3.1441 | LR: 4.36e-04 | 1301.2ms/step | 15.4 samples/s +Step 10820/44000 | Loss: 3.1522 | LR: 4.36e-04 | 1305.7ms/step | 15.3 samples/s +Step 10830/44000 | Loss: 3.1344 | LR: 4.36e-04 | 1300.9ms/step | 15.4 samples/s +Step 10840/44000 | Loss: 3.1607 | LR: 4.36e-04 | 1298.8ms/step | 15.4 samples/s +Step 10850/44000 | Loss: 3.1549 | LR: 4.36e-04 | 1299.6ms/step | 15.4 samples/s +Step 10860/44000 | Loss: 3.1740 | LR: 4.36e-04 | 1299.6ms/step | 15.4 samples/s +Step 10870/44000 | Loss: 3.1379 | LR: 4.35e-04 | 1303.2ms/step | 15.3 samples/s +Step 10880/44000 | Loss: 3.1348 | LR: 4.35e-04 | 1305.5ms/step | 15.3 samples/s +Step 10890/44000 | Loss: 3.1387 | LR: 4.35e-04 | 1316.1ms/step | 15.2 samples/s +Step 10900/44000 | Loss: 3.1432 | LR: 4.35e-04 | 1302.8ms/step | 15.4 samples/s +Step 10910/44000 | Loss: 3.1200 | LR: 4.35e-04 | 1300.8ms/step | 15.4 samples/s +Step 10920/44000 | Loss: 3.1055 | LR: 4.35e-04 | 1302.1ms/step | 15.4 samples/s +Step 10930/44000 | Loss: 3.1733 | LR: 4.35e-04 | 1300.1ms/step | 15.4 samples/s +Step 10940/44000 | Loss: 3.1420 | LR: 4.35e-04 | 1300.2ms/step | 15.4 samples/s +Step 10950/44000 | Loss: 3.1221 | LR: 4.34e-04 | 1311.8ms/step | 15.2 samples/s +Step 10960/44000 | Loss: 3.1514 | LR: 4.34e-04 | 1303.5ms/step | 15.3 samples/s +Step 10970/44000 | Loss: 3.1344 | LR: 4.34e-04 | 1301.8ms/step | 15.4 samples/s +Step 10980/44000 | Loss: 3.1267 | LR: 4.34e-04 | 1304.0ms/step | 15.3 samples/s +Step 10990/44000 | Loss: 3.1397 | LR: 4.34e-04 | 1304.0ms/step | 15.3 samples/s +Step 11000/44000 | Loss: 3.1328 | LR: 4.34e-04 | 1304.8ms/step | 15.3 samples/s +Validation at step 11000: Loss = 3.1498, Perplexity = 23.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-11000 +Step 11010/44000 | Loss: 3.1486 | LR: 4.34e-04 | 3795.5ms/step | 5.3 samples/s +Step 11020/44000 | Loss: 3.1537 | LR: 4.34e-04 | 1300.5ms/step | 15.4 samples/s +Step 11030/44000 | Loss: 3.1024 | LR: 4.33e-04 | 1301.9ms/step | 15.4 samples/s +Step 11040/44000 | Loss: 3.1457 | LR: 4.33e-04 | 1306.1ms/step | 15.3 samples/s +Step 11050/44000 | Loss: 3.1263 | LR: 4.33e-04 | 1306.5ms/step | 15.3 samples/s +Step 11060/44000 | Loss: 3.1405 | LR: 4.33e-04 | 1298.9ms/step | 15.4 samples/s +Step 11070/44000 | Loss: 3.1352 | LR: 4.33e-04 | 1302.8ms/step | 15.4 samples/s +Step 11080/44000 | Loss: 3.0996 | LR: 4.33e-04 | 1299.7ms/step | 15.4 samples/s +Step 11090/44000 | Loss: 3.1584 | LR: 4.33e-04 | 1301.4ms/step | 15.4 samples/s +Step 11100/44000 | Loss: 3.1331 | LR: 4.33e-04 | 1302.7ms/step | 15.4 samples/s +Step 11110/44000 | Loss: 3.0975 | LR: 4.32e-04 | 1303.3ms/step | 15.3 samples/s +Step 11120/44000 | Loss: 3.1259 | LR: 4.32e-04 | 1303.0ms/step | 15.3 samples/s +Step 11130/44000 | Loss: 3.1228 | LR: 4.32e-04 | 1302.0ms/step | 15.4 samples/s +Step 11140/44000 | Loss: 3.1253 | LR: 4.32e-04 | 1302.0ms/step | 15.4 samples/s +Step 11150/44000 | Loss: 3.1473 | LR: 4.32e-04 | 1304.0ms/step | 15.3 samples/s +Step 11160/44000 | Loss: 3.1294 | LR: 4.32e-04 | 1303.9ms/step | 15.3 samples/s +Step 11170/44000 | Loss: 3.1264 | LR: 4.32e-04 | 1301.4ms/step | 15.4 samples/s +Step 11180/44000 | Loss: 3.1346 | LR: 4.32e-04 | 1301.5ms/step | 15.4 samples/s +Step 11190/44000 | Loss: 3.1313 | LR: 4.31e-04 | 1301.2ms/step | 15.4 samples/s +Step 11200/44000 | Loss: 3.1316 | LR: 4.31e-04 | 1303.6ms/step | 15.3 samples/s +Step 11210/44000 | Loss: 3.1357 | LR: 4.31e-04 | 1303.1ms/step | 15.3 samples/s +Step 11220/44000 | Loss: 3.1102 | LR: 4.31e-04 | 1301.6ms/step | 15.4 samples/s +Step 11230/44000 | Loss: 3.1346 | LR: 4.31e-04 | 1301.2ms/step | 15.4 samples/s +Step 11240/44000 | Loss: 3.1220 | LR: 4.31e-04 | 1300.8ms/step | 15.4 samples/s +Step 11250/44000 | Loss: 3.1294 | LR: 4.31e-04 | 1300.2ms/step | 15.4 samples/s +Validation at step 11250: Loss = 3.1453, Perplexity = 23.23 +Step 11260/44000 | Loss: 3.1043 | LR: 4.31e-04 | 3485.4ms/step | 5.7 samples/s +Step 11270/44000 | Loss: 3.1470 | LR: 4.30e-04 | 1302.9ms/step | 15.4 samples/s +Step 11280/44000 | Loss: 3.1212 | LR: 4.30e-04 | 1302.9ms/step | 15.4 samples/s +Step 11290/44000 | Loss: 3.1334 | LR: 4.30e-04 | 1300.2ms/step | 15.4 samples/s +Step 11300/44000 | Loss: 3.1512 | LR: 4.30e-04 | 1300.9ms/step | 15.4 samples/s +Step 11310/44000 | Loss: 3.1173 | LR: 4.30e-04 | 1300.9ms/step | 15.4 samples/s +Step 11320/44000 | Loss: 3.1303 | LR: 4.30e-04 | 1297.8ms/step | 15.4 samples/s +Step 11330/44000 | Loss: 3.1308 | LR: 4.30e-04 | 1298.4ms/step | 15.4 samples/s +Step 11340/44000 | Loss: 3.1029 | LR: 4.30e-04 | 1298.0ms/step | 15.4 samples/s +Step 11350/44000 | Loss: 3.1293 | LR: 4.29e-04 | 1299.7ms/step | 15.4 samples/s +Step 11360/44000 | Loss: 3.1124 | LR: 4.29e-04 | 1299.8ms/step | 15.4 samples/s +Step 11370/44000 | Loss: 3.1418 | LR: 4.29e-04 | 1301.7ms/step | 15.4 samples/s +Step 11380/44000 | Loss: 3.1419 | LR: 4.29e-04 | 1300.1ms/step | 15.4 samples/s +Step 11390/44000 | Loss: 3.1585 | LR: 4.29e-04 | 1300.7ms/step | 15.4 samples/s +Step 11400/44000 | Loss: 3.1217 | LR: 4.29e-04 | 1302.1ms/step | 15.4 samples/s +Step 11410/44000 | Loss: 3.0926 | LR: 4.29e-04 | 1301.2ms/step | 15.4 samples/s +Step 11420/44000 | Loss: 3.1418 | LR: 4.29e-04 | 1301.8ms/step | 15.4 samples/s +Step 11430/44000 | Loss: 3.1227 | LR: 4.28e-04 | 1302.2ms/step | 15.4 samples/s +Step 11440/44000 | Loss: 3.1468 | LR: 4.28e-04 | 1302.2ms/step | 15.4 samples/s +Step 11450/44000 | Loss: 3.1229 | LR: 4.28e-04 | 1301.6ms/step | 15.4 samples/s +Step 11460/44000 | Loss: 3.1573 | LR: 4.28e-04 | 1300.2ms/step | 15.4 samples/s +Step 11470/44000 | Loss: 3.1315 | LR: 4.28e-04 | 1301.1ms/step | 15.4 samples/s +Step 11480/44000 | Loss: 3.1200 | LR: 4.28e-04 | 1303.2ms/step | 15.3 samples/s +Step 11490/44000 | Loss: 3.1235 | LR: 4.28e-04 | 1301.6ms/step | 15.4 samples/s +Step 11500/44000 | Loss: 3.1245 | LR: 4.28e-04 | 1300.9ms/step | 15.4 samples/s +Validation at step 11500: Loss = 3.1393, Perplexity = 23.09 +Step 11510/44000 | Loss: 3.1394 | LR: 4.27e-04 | 3489.9ms/step | 5.7 samples/s +Step 11520/44000 | Loss: 3.0984 | LR: 4.27e-04 | 1304.4ms/step | 15.3 samples/s +Step 11530/44000 | Loss: 3.1279 | LR: 4.27e-04 | 1301.2ms/step | 15.4 samples/s +Step 11540/44000 | Loss: 3.1279 | LR: 4.27e-04 | 1298.6ms/step | 15.4 samples/s +Step 11550/44000 | Loss: 3.1111 | LR: 4.27e-04 | 1299.2ms/step | 15.4 samples/s +Step 11560/44000 | Loss: 3.1282 | LR: 4.27e-04 | 1300.0ms/step | 15.4 samples/s +Step 11570/44000 | Loss: 3.1284 | LR: 4.27e-04 | 1300.5ms/step | 15.4 samples/s +Step 11580/44000 | Loss: 3.1269 | LR: 4.27e-04 | 1298.3ms/step | 15.4 samples/s +Step 11590/44000 | Loss: 3.1357 | LR: 4.26e-04 | 1296.9ms/step | 15.4 samples/s +Step 11600/44000 | Loss: 3.1213 | LR: 4.26e-04 | 1298.6ms/step | 15.4 samples/s +Step 11610/44000 | Loss: 3.1294 | LR: 4.26e-04 | 1297.2ms/step | 15.4 samples/s +Step 11620/44000 | Loss: 3.1229 | LR: 4.26e-04 | 1301.8ms/step | 15.4 samples/s +Step 11630/44000 | Loss: 3.1030 | LR: 4.26e-04 | 1303.0ms/step | 15.3 samples/s +Step 11640/44000 | Loss: 3.1282 | LR: 4.26e-04 | 1306.3ms/step | 15.3 samples/s +Step 11650/44000 | Loss: 3.1280 | LR: 4.26e-04 | 1314.4ms/step | 15.2 samples/s +Step 11660/44000 | Loss: 3.0983 | LR: 4.26e-04 | 1303.5ms/step | 15.3 samples/s +Step 11670/44000 | Loss: 3.1209 | LR: 4.25e-04 | 1303.1ms/step | 15.3 samples/s +Step 11680/44000 | Loss: 3.0977 | LR: 4.25e-04 | 1301.3ms/step | 15.4 samples/s +Step 11690/44000 | Loss: 3.1532 | LR: 4.25e-04 | 1302.9ms/step | 15.3 samples/s +Step 11700/44000 | Loss: 3.0917 | LR: 4.25e-04 | 1303.7ms/step | 15.3 samples/s +Step 11710/44000 | Loss: 3.1423 | LR: 4.25e-04 | 1300.3ms/step | 15.4 samples/s +Step 11720/44000 | Loss: 3.1330 | LR: 4.25e-04 | 1306.9ms/step | 15.3 samples/s +Step 11730/44000 | Loss: 3.1324 | LR: 4.25e-04 | 1307.4ms/step | 15.3 samples/s +Step 11740/44000 | Loss: 3.1576 | LR: 4.24e-04 | 1306.2ms/step | 15.3 samples/s +Step 11750/44000 | Loss: 3.1194 | LR: 4.24e-04 | 1303.9ms/step | 15.3 samples/s +Validation at step 11750: Loss = 3.1359, Perplexity = 23.01 +Step 11760/44000 | Loss: 3.1533 | LR: 4.24e-04 | 3500.9ms/step | 5.7 samples/s +Step 11770/44000 | Loss: 3.1470 | LR: 4.24e-04 | 1301.8ms/step | 15.4 samples/s +Step 11780/44000 | Loss: 3.1538 | LR: 4.24e-04 | 1302.0ms/step | 15.4 samples/s +Step 11790/44000 | Loss: 3.1163 | LR: 4.24e-04 | 1302.0ms/step | 15.4 samples/s +Step 11800/44000 | Loss: 3.1089 | LR: 4.24e-04 | 1300.4ms/step | 15.4 samples/s +Step 11810/44000 | Loss: 3.1020 | LR: 4.24e-04 | 1300.5ms/step | 15.4 samples/s +Step 11820/44000 | Loss: 3.1307 | LR: 4.23e-04 | 1302.3ms/step | 15.4 samples/s +Step 11830/44000 | Loss: 3.1540 | LR: 4.23e-04 | 1302.3ms/step | 15.4 samples/s +Step 11840/44000 | Loss: 3.0927 | LR: 4.23e-04 | 1299.8ms/step | 15.4 samples/s +Step 11850/44000 | Loss: 3.1347 | LR: 4.23e-04 | 1301.3ms/step | 15.4 samples/s +Step 11860/44000 | Loss: 3.1104 | LR: 4.23e-04 | 1300.4ms/step | 15.4 samples/s +Step 11870/44000 | Loss: 3.1360 | LR: 4.23e-04 | 1301.0ms/step | 15.4 samples/s +Step 11880/44000 | Loss: 3.1195 | LR: 4.23e-04 | 1299.5ms/step | 15.4 samples/s +Step 11890/44000 | Loss: 3.0941 | LR: 4.23e-04 | 1301.1ms/step | 15.4 samples/s +Step 11900/44000 | Loss: 3.1523 | LR: 4.22e-04 | 1305.9ms/step | 15.3 samples/s +Step 11910/44000 | Loss: 3.1232 | LR: 4.22e-04 | 1300.8ms/step | 15.4 samples/s +Step 11920/44000 | Loss: 3.1202 | LR: 4.22e-04 | 1301.1ms/step | 15.4 samples/s +Step 11930/44000 | Loss: 3.1247 | LR: 4.22e-04 | 1300.4ms/step | 15.4 samples/s +Step 11940/44000 | Loss: 3.1116 | LR: 4.22e-04 | 1304.8ms/step | 15.3 samples/s +Step 11950/44000 | Loss: 3.0894 | LR: 4.22e-04 | 1308.2ms/step | 15.3 samples/s +Step 11960/44000 | Loss: 3.1085 | LR: 4.22e-04 | 1300.5ms/step | 15.4 samples/s +Step 11970/44000 | Loss: 3.1375 | LR: 4.21e-04 | 1299.2ms/step | 15.4 samples/s +Step 11980/44000 | Loss: 3.1110 | LR: 4.21e-04 | 1301.4ms/step | 15.4 samples/s +Step 11990/44000 | Loss: 3.1099 | LR: 4.21e-04 | 1300.1ms/step | 15.4 samples/s +Step 12000/44000 | Loss: 3.1258 | LR: 4.21e-04 | 1300.9ms/step | 15.4 samples/s +Validation at step 12000: Loss = 3.1313, Perplexity = 22.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-12000 +Step 12010/44000 | Loss: 3.1109 | LR: 4.21e-04 | 3785.6ms/step | 5.3 samples/s +Step 12020/44000 | Loss: 3.0921 | LR: 4.21e-04 | 1300.0ms/step | 15.4 samples/s +Step 12030/44000 | Loss: 3.1291 | LR: 4.21e-04 | 1301.8ms/step | 15.4 samples/s +Step 12040/44000 | Loss: 3.1309 | LR: 4.21e-04 | 1300.6ms/step | 15.4 samples/s +Step 12050/44000 | Loss: 3.1168 | LR: 4.20e-04 | 1299.8ms/step | 15.4 samples/s +Step 12060/44000 | Loss: 3.1097 | LR: 4.20e-04 | 1299.9ms/step | 15.4 samples/s +Step 12070/44000 | Loss: 3.1497 | LR: 4.20e-04 | 1299.7ms/step | 15.4 samples/s +Step 12080/44000 | Loss: 3.1239 | LR: 4.20e-04 | 1299.5ms/step | 15.4 samples/s +Step 12090/44000 | Loss: 3.1363 | LR: 4.20e-04 | 1299.1ms/step | 15.4 samples/s +Step 12100/44000 | Loss: 3.1274 | LR: 4.20e-04 | 1298.7ms/step | 15.4 samples/s +Step 12110/44000 | Loss: 3.1252 | LR: 4.20e-04 | 1300.4ms/step | 15.4 samples/s +Step 12120/44000 | Loss: 3.1279 | LR: 4.19e-04 | 1298.3ms/step | 15.4 samples/s +Step 12130/44000 | Loss: 3.1420 | LR: 4.19e-04 | 1299.7ms/step | 15.4 samples/s +Step 12140/44000 | Loss: 3.0918 | LR: 4.19e-04 | 1301.9ms/step | 15.4 samples/s +Step 12150/44000 | Loss: 3.1068 | LR: 4.19e-04 | 1301.7ms/step | 15.4 samples/s +Step 12160/44000 | Loss: 3.1375 | LR: 4.19e-04 | 1308.3ms/step | 15.3 samples/s +Step 12170/44000 | Loss: 3.1234 | LR: 4.19e-04 | 1316.6ms/step | 15.2 samples/s +Step 12180/44000 | Loss: 3.1009 | LR: 4.19e-04 | 1303.4ms/step | 15.3 samples/s +Step 12190/44000 | Loss: 3.1159 | LR: 4.19e-04 | 1299.8ms/step | 15.4 samples/s +Step 12200/44000 | Loss: 3.0829 | LR: 4.18e-04 | 1302.4ms/step | 15.4 samples/s +Step 12210/44000 | Loss: 3.1142 | LR: 4.18e-04 | 1300.4ms/step | 15.4 samples/s +Step 12220/44000 | Loss: 3.0896 | LR: 4.18e-04 | 1301.8ms/step | 15.4 samples/s +Step 12230/44000 | Loss: 3.0896 | LR: 4.18e-04 | 1302.4ms/step | 15.4 samples/s +Step 12240/44000 | Loss: 3.1446 | LR: 4.18e-04 | 1303.9ms/step | 15.3 samples/s +Step 12250/44000 | Loss: 3.1250 | LR: 4.18e-04 | 1303.8ms/step | 15.3 samples/s +Validation at step 12250: Loss = 3.1287, Perplexity = 22.84 +Step 12260/44000 | Loss: 3.1010 | LR: 4.18e-04 | 3477.3ms/step | 5.8 samples/s +Step 12270/44000 | Loss: 3.1270 | LR: 4.17e-04 | 1298.1ms/step | 15.4 samples/s +Step 12280/44000 | Loss: 3.1060 | LR: 4.17e-04 | 1297.4ms/step | 15.4 samples/s +Step 12290/44000 | Loss: 3.1229 | LR: 4.17e-04 | 1298.1ms/step | 15.4 samples/s +Step 12300/44000 | Loss: 3.1240 | LR: 4.17e-04 | 1297.8ms/step | 15.4 samples/s +Step 12310/44000 | Loss: 3.1285 | LR: 4.17e-04 | 1298.3ms/step | 15.4 samples/s +Step 12320/44000 | Loss: 3.1087 | LR: 4.17e-04 | 1299.5ms/step | 15.4 samples/s +Step 12330/44000 | Loss: 3.1358 | LR: 4.17e-04 | 1300.1ms/step | 15.4 samples/s +Step 12340/44000 | Loss: 3.1173 | LR: 4.17e-04 | 1301.7ms/step | 15.4 samples/s +Step 12350/44000 | Loss: 3.1337 | LR: 4.16e-04 | 1300.5ms/step | 15.4 samples/s +Step 12360/44000 | Loss: 3.1163 | LR: 4.16e-04 | 1300.1ms/step | 15.4 samples/s +Step 12370/44000 | Loss: 3.1113 | LR: 4.16e-04 | 1297.8ms/step | 15.4 samples/s +Step 12380/44000 | Loss: 3.0818 | LR: 4.16e-04 | 1298.7ms/step | 15.4 samples/s +Step 12390/44000 | Loss: 3.1175 | LR: 4.16e-04 | 1299.8ms/step | 15.4 samples/s +Step 12400/44000 | Loss: 3.0836 | LR: 4.16e-04 | 1301.6ms/step | 15.4 samples/s +Step 12410/44000 | Loss: 3.1075 | LR: 4.16e-04 | 1302.1ms/step | 15.4 samples/s +Step 12420/44000 | Loss: 3.1185 | LR: 4.15e-04 | 1299.9ms/step | 15.4 samples/s +Step 12430/44000 | Loss: 3.1009 | LR: 4.15e-04 | 1301.0ms/step | 15.4 samples/s +Step 12440/44000 | Loss: 3.1162 | LR: 4.15e-04 | 1299.7ms/step | 15.4 samples/s +Step 12450/44000 | Loss: 3.1104 | LR: 4.15e-04 | 1301.2ms/step | 15.4 samples/s +Step 12460/44000 | Loss: 3.1185 | LR: 4.15e-04 | 1301.0ms/step | 15.4 samples/s +Step 12470/44000 | Loss: 3.1100 | LR: 4.15e-04 | 1298.3ms/step | 15.4 samples/s +Step 12480/44000 | Loss: 3.1026 | LR: 4.15e-04 | 1297.0ms/step | 15.4 samples/s +Step 12490/44000 | Loss: 3.0839 | LR: 4.14e-04 | 1302.0ms/step | 15.4 samples/s +Step 12500/44000 | Loss: 3.0777 | LR: 4.14e-04 | 1301.2ms/step | 15.4 samples/s +Validation at step 12500: Loss = 3.1223, Perplexity = 22.70 +Step 12510/44000 | Loss: 3.1336 | LR: 4.14e-04 | 3477.5ms/step | 5.8 samples/s +Step 12520/44000 | Loss: 3.1156 | LR: 4.14e-04 | 1301.4ms/step | 15.4 samples/s +Step 12530/44000 | Loss: 3.1096 | LR: 4.14e-04 | 1300.6ms/step | 15.4 samples/s +Step 12540/44000 | Loss: 3.1103 | LR: 4.14e-04 | 1299.7ms/step | 15.4 samples/s +Step 12550/44000 | Loss: 3.1243 | LR: 4.14e-04 | 1299.5ms/step | 15.4 samples/s +Step 12560/44000 | Loss: 3.0899 | LR: 4.14e-04 | 1300.6ms/step | 15.4 samples/s +Step 12570/44000 | Loss: 3.1011 | LR: 4.13e-04 | 1299.8ms/step | 15.4 samples/s +Step 12580/44000 | Loss: 3.1462 | LR: 4.13e-04 | 1300.1ms/step | 15.4 samples/s +Step 12590/44000 | Loss: 3.1036 | LR: 4.13e-04 | 1302.4ms/step | 15.4 samples/s +Step 12600/44000 | Loss: 3.1084 | LR: 4.13e-04 | 1308.6ms/step | 15.3 samples/s +Step 12610/44000 | Loss: 3.1226 | LR: 4.13e-04 | 1298.7ms/step | 15.4 samples/s +Step 12620/44000 | Loss: 3.1543 | LR: 4.13e-04 | 1306.7ms/step | 15.3 samples/s +Step 12630/44000 | Loss: 3.1165 | LR: 4.13e-04 | 1303.3ms/step | 15.3 samples/s +Step 12640/44000 | Loss: 3.1071 | LR: 4.12e-04 | 1299.7ms/step | 15.4 samples/s +Step 12650/44000 | Loss: 3.1181 | LR: 4.12e-04 | 1298.7ms/step | 15.4 samples/s +Step 12660/44000 | Loss: 3.1056 | LR: 4.12e-04 | 1298.7ms/step | 15.4 samples/s +Step 12670/44000 | Loss: 3.1191 | LR: 4.12e-04 | 1298.0ms/step | 15.4 samples/s +Step 12680/44000 | Loss: 3.1229 | LR: 4.12e-04 | 1301.5ms/step | 15.4 samples/s +Step 12690/44000 | Loss: 3.1113 | LR: 4.12e-04 | 1301.4ms/step | 15.4 samples/s +Step 12700/44000 | Loss: 3.1058 | LR: 4.12e-04 | 1298.4ms/step | 15.4 samples/s +Step 12710/44000 | Loss: 3.0724 | LR: 4.11e-04 | 1298.3ms/step | 15.4 samples/s +Step 12720/44000 | Loss: 3.0714 | LR: 4.11e-04 | 1296.7ms/step | 15.4 samples/s +Step 12730/44000 | Loss: 3.0977 | LR: 4.11e-04 | 1299.8ms/step | 15.4 samples/s +Step 12740/44000 | Loss: 3.1154 | LR: 4.11e-04 | 1296.8ms/step | 15.4 samples/s +Step 12750/44000 | Loss: 3.0772 | LR: 4.11e-04 | 1297.6ms/step | 15.4 samples/s +Validation at step 12750: Loss = 3.1173, Perplexity = 22.59 +Step 12760/44000 | Loss: 3.1049 | LR: 4.11e-04 | 3505.3ms/step | 5.7 samples/s +Step 12770/44000 | Loss: 3.1121 | LR: 4.11e-04 | 1304.5ms/step | 15.3 samples/s +Step 12780/44000 | Loss: 3.0936 | LR: 4.10e-04 | 1299.9ms/step | 15.4 samples/s +Step 12790/44000 | Loss: 3.0980 | LR: 4.10e-04 | 1299.8ms/step | 15.4 samples/s +Step 12800/44000 | Loss: 3.1183 | LR: 4.10e-04 | 1298.7ms/step | 15.4 samples/s +Step 12810/44000 | Loss: 3.1189 | LR: 4.10e-04 | 1300.7ms/step | 15.4 samples/s +Step 12820/44000 | Loss: 3.0953 | LR: 4.10e-04 | 1309.4ms/step | 15.3 samples/s +Step 12830/44000 | Loss: 3.1064 | LR: 4.10e-04 | 1303.3ms/step | 15.3 samples/s +Step 12840/44000 | Loss: 3.0852 | LR: 4.10e-04 | 1306.0ms/step | 15.3 samples/s +Step 12850/44000 | Loss: 3.1052 | LR: 4.09e-04 | 1303.0ms/step | 15.3 samples/s +Step 12860/44000 | Loss: 3.0819 | LR: 4.09e-04 | 1302.0ms/step | 15.4 samples/s +Step 12870/44000 | Loss: 3.1123 | LR: 4.09e-04 | 1301.0ms/step | 15.4 samples/s +Step 12880/44000 | Loss: 3.0728 | LR: 4.09e-04 | 1303.4ms/step | 15.3 samples/s +Step 12890/44000 | Loss: 3.1270 | LR: 4.09e-04 | 1302.7ms/step | 15.4 samples/s +Step 12900/44000 | Loss: 3.0877 | LR: 4.09e-04 | 1300.7ms/step | 15.4 samples/s +Step 12910/44000 | Loss: 3.1036 | LR: 4.09e-04 | 1301.3ms/step | 15.4 samples/s +Step 12920/44000 | Loss: 3.1038 | LR: 4.09e-04 | 1301.7ms/step | 15.4 samples/s +Step 12930/44000 | Loss: 3.1068 | LR: 4.08e-04 | 1299.9ms/step | 15.4 samples/s +Step 12940/44000 | Loss: 3.1217 | LR: 4.08e-04 | 1302.1ms/step | 15.4 samples/s +Step 12950/44000 | Loss: 3.1057 | LR: 4.08e-04 | 1301.1ms/step | 15.4 samples/s +Step 12960/44000 | Loss: 3.1122 | LR: 4.08e-04 | 1305.4ms/step | 15.3 samples/s +Step 12970/44000 | Loss: 3.1204 | LR: 4.08e-04 | 1312.5ms/step | 15.2 samples/s +Step 12980/44000 | Loss: 3.1144 | LR: 4.08e-04 | 1304.6ms/step | 15.3 samples/s +Step 12990/44000 | Loss: 3.1071 | LR: 4.08e-04 | 1301.5ms/step | 15.4 samples/s +Step 13000/44000 | Loss: 3.0928 | LR: 4.07e-04 | 1304.2ms/step | 15.3 samples/s +Validation at step 13000: Loss = 3.1142, Perplexity = 22.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-13000 +Step 13010/44000 | Loss: 3.1019 | LR: 4.07e-04 | 3796.6ms/step | 5.3 samples/s +Step 13020/44000 | Loss: 3.1021 | LR: 4.07e-04 | 1301.7ms/step | 15.4 samples/s +Step 13030/44000 | Loss: 3.1087 | LR: 4.07e-04 | 1302.7ms/step | 15.4 samples/s +Step 13040/44000 | Loss: 3.0831 | LR: 4.07e-04 | 1304.6ms/step | 15.3 samples/s +Step 13050/44000 | Loss: 3.0681 | LR: 4.07e-04 | 1306.0ms/step | 15.3 samples/s +Step 13060/44000 | Loss: 3.1138 | LR: 4.07e-04 | 1307.3ms/step | 15.3 samples/s +Step 13070/44000 | Loss: 3.1063 | LR: 4.06e-04 | 1302.2ms/step | 15.4 samples/s +Step 13080/44000 | Loss: 3.1231 | LR: 4.06e-04 | 1306.4ms/step | 15.3 samples/s +Step 13090/44000 | Loss: 3.0920 | LR: 4.06e-04 | 1304.1ms/step | 15.3 samples/s +Step 13100/44000 | Loss: 3.0940 | LR: 4.06e-04 | 1302.5ms/step | 15.4 samples/s +Step 13110/44000 | Loss: 3.1066 | LR: 4.06e-04 | 1303.8ms/step | 15.3 samples/s +Step 13120/44000 | Loss: 3.1163 | LR: 4.06e-04 | 1303.3ms/step | 15.3 samples/s +Step 13130/44000 | Loss: 3.1164 | LR: 4.06e-04 | 1302.4ms/step | 15.4 samples/s +Step 13140/44000 | Loss: 3.1173 | LR: 4.05e-04 | 1304.5ms/step | 15.3 samples/s +Step 13150/44000 | Loss: 3.1007 | LR: 4.05e-04 | 1308.7ms/step | 15.3 samples/s +Step 13160/44000 | Loss: 3.0921 | LR: 4.05e-04 | 1302.2ms/step | 15.4 samples/s +Step 13170/44000 | Loss: 3.0974 | LR: 4.05e-04 | 1300.6ms/step | 15.4 samples/s +Step 13180/44000 | Loss: 3.1188 | LR: 4.05e-04 | 1300.0ms/step | 15.4 samples/s +Step 13190/44000 | Loss: 3.0939 | LR: 4.05e-04 | 1301.6ms/step | 15.4 samples/s +Step 13200/44000 | Loss: 3.1261 | LR: 4.05e-04 | 1301.4ms/step | 15.4 samples/s +Step 13210/44000 | Loss: 3.0815 | LR: 4.04e-04 | 1303.0ms/step | 15.3 samples/s +Step 13220/44000 | Loss: 3.0831 | LR: 4.04e-04 | 1302.3ms/step | 15.4 samples/s +Step 13230/44000 | Loss: 3.1139 | LR: 4.04e-04 | 1300.3ms/step | 15.4 samples/s +Step 13240/44000 | Loss: 3.0941 | LR: 4.04e-04 | 1299.4ms/step | 15.4 samples/s +Step 13250/44000 | Loss: 3.1124 | LR: 4.04e-04 | 1299.0ms/step | 15.4 samples/s +Validation at step 13250: Loss = 3.1099, Perplexity = 22.42 +Step 13260/44000 | Loss: 3.1037 | LR: 4.04e-04 | 3476.8ms/step | 5.8 samples/s +Step 13270/44000 | Loss: 3.0933 | LR: 4.04e-04 | 1296.6ms/step | 15.4 samples/s +Step 13280/44000 | Loss: 3.1197 | LR: 4.03e-04 | 1296.5ms/step | 15.4 samples/s +Step 13290/44000 | Loss: 3.1050 | LR: 4.03e-04 | 1298.1ms/step | 15.4 samples/s +Step 13300/44000 | Loss: 3.0667 | LR: 4.03e-04 | 1297.3ms/step | 15.4 samples/s +Step 13310/44000 | Loss: 3.0988 | LR: 4.03e-04 | 1298.4ms/step | 15.4 samples/s +Step 13320/44000 | Loss: 3.0807 | LR: 4.03e-04 | 1296.6ms/step | 15.4 samples/s +Step 13330/44000 | Loss: 3.0877 | LR: 4.03e-04 | 1296.7ms/step | 15.4 samples/s +Step 13340/44000 | Loss: 3.0965 | LR: 4.03e-04 | 1297.1ms/step | 15.4 samples/s +Step 13350/44000 | Loss: 3.0826 | LR: 4.02e-04 | 1300.9ms/step | 15.4 samples/s +Step 13360/44000 | Loss: 3.0971 | LR: 4.02e-04 | 1303.6ms/step | 15.3 samples/s +Step 13370/44000 | Loss: 3.1265 | LR: 4.02e-04 | 1302.3ms/step | 15.4 samples/s +Step 13380/44000 | Loss: 3.1192 | LR: 4.02e-04 | 1304.3ms/step | 15.3 samples/s +Step 13390/44000 | Loss: 3.0868 | LR: 4.02e-04 | 1305.4ms/step | 15.3 samples/s +Step 13400/44000 | Loss: 3.0887 | LR: 4.02e-04 | 1299.3ms/step | 15.4 samples/s +Step 13410/44000 | Loss: 3.0823 | LR: 4.02e-04 | 1299.6ms/step | 15.4 samples/s +Step 13420/44000 | Loss: 3.0522 | LR: 4.01e-04 | 1298.9ms/step | 15.4 samples/s +Step 13430/44000 | Loss: 3.1081 | LR: 4.01e-04 | 1299.4ms/step | 15.4 samples/s +Step 13440/44000 | Loss: 3.0990 | LR: 4.01e-04 | 1298.9ms/step | 15.4 samples/s +Step 13450/44000 | Loss: 3.1070 | LR: 4.01e-04 | 1299.6ms/step | 15.4 samples/s +Step 13460/44000 | Loss: 3.0845 | LR: 4.01e-04 | 1298.8ms/step | 15.4 samples/s +Step 13470/44000 | Loss: 3.0986 | LR: 4.01e-04 | 1299.8ms/step | 15.4 samples/s +Step 13480/44000 | Loss: 3.0734 | LR: 4.01e-04 | 1299.5ms/step | 15.4 samples/s +Step 13490/44000 | Loss: 3.0992 | LR: 4.00e-04 | 1301.5ms/step | 15.4 samples/s +Step 13500/44000 | Loss: 3.1012 | LR: 4.00e-04 | 1301.0ms/step | 15.4 samples/s +Validation at step 13500: Loss = 3.1058, Perplexity = 22.33 +Step 13510/44000 | Loss: 3.0880 | LR: 4.00e-04 | 3479.7ms/step | 5.7 samples/s +Step 13520/44000 | Loss: 3.0851 | LR: 4.00e-04 | 1302.5ms/step | 15.4 samples/s +Step 13530/44000 | Loss: 3.0825 | LR: 4.00e-04 | 1300.9ms/step | 15.4 samples/s +Step 13540/44000 | Loss: 3.0734 | LR: 4.00e-04 | 1299.2ms/step | 15.4 samples/s +Step 13550/44000 | Loss: 3.0904 | LR: 4.00e-04 | 1299.4ms/step | 15.4 samples/s +Step 13560/44000 | Loss: 3.0821 | LR: 3.99e-04 | 1296.6ms/step | 15.4 samples/s +Step 13570/44000 | Loss: 3.1196 | LR: 3.99e-04 | 1296.0ms/step | 15.4 samples/s +Step 13580/44000 | Loss: 3.0988 | LR: 3.99e-04 | 1299.2ms/step | 15.4 samples/s +Step 13590/44000 | Loss: 3.0766 | LR: 3.99e-04 | 1301.8ms/step | 15.4 samples/s +Step 13600/44000 | Loss: 3.0883 | LR: 3.99e-04 | 1300.9ms/step | 15.4 samples/s +Step 13610/44000 | Loss: 3.0780 | LR: 3.99e-04 | 1300.4ms/step | 15.4 samples/s +Step 13620/44000 | Loss: 3.1100 | LR: 3.98e-04 | 1302.7ms/step | 15.4 samples/s +Step 13630/44000 | Loss: 3.0998 | LR: 3.98e-04 | 1301.4ms/step | 15.4 samples/s +Step 13640/44000 | Loss: 3.0773 | LR: 3.98e-04 | 1300.9ms/step | 15.4 samples/s +Step 13650/44000 | Loss: 3.0950 | LR: 3.98e-04 | 1300.0ms/step | 15.4 samples/s +Step 13660/44000 | Loss: 3.0799 | LR: 3.98e-04 | 1300.5ms/step | 15.4 samples/s +Step 13670/44000 | Loss: 3.0675 | LR: 3.98e-04 | 1301.6ms/step | 15.4 samples/s +Step 13680/44000 | Loss: 3.0818 | LR: 3.98e-04 | 1299.7ms/step | 15.4 samples/s +Step 13690/44000 | Loss: 3.0987 | LR: 3.97e-04 | 1297.6ms/step | 15.4 samples/s +Step 13700/44000 | Loss: 3.0850 | LR: 3.97e-04 | 1296.1ms/step | 15.4 samples/s +Step 13710/44000 | Loss: 3.0956 | LR: 3.97e-04 | 1295.2ms/step | 15.4 samples/s +Step 13720/44000 | Loss: 3.0768 | LR: 3.97e-04 | 1298.5ms/step | 15.4 samples/s +Step 13730/44000 | Loss: 3.0940 | LR: 3.97e-04 | 1302.2ms/step | 15.4 samples/s +Step 13740/44000 | Loss: 3.0998 | LR: 3.97e-04 | 1306.9ms/step | 15.3 samples/s +Step 13750/44000 | Loss: 3.1335 | LR: 3.97e-04 | 1307.7ms/step | 15.3 samples/s +Validation at step 13750: Loss = 3.1046, Perplexity = 22.30 +Step 13760/44000 | Loss: 3.1226 | LR: 3.96e-04 | 3493.7ms/step | 5.7 samples/s +Step 13770/44000 | Loss: 3.0919 | LR: 3.96e-04 | 1302.2ms/step | 15.4 samples/s +Step 13780/44000 | Loss: 3.0834 | LR: 3.96e-04 | 1296.6ms/step | 15.4 samples/s +Step 13790/44000 | Loss: 3.1200 | LR: 3.96e-04 | 1298.0ms/step | 15.4 samples/s +Step 13800/44000 | Loss: 3.0787 | LR: 3.96e-04 | 1298.8ms/step | 15.4 samples/s +Step 13810/44000 | Loss: 3.0722 | LR: 3.96e-04 | 1300.0ms/step | 15.4 samples/s +Step 13820/44000 | Loss: 3.0884 | LR: 3.96e-04 | 1312.1ms/step | 15.2 samples/s +Step 13830/44000 | Loss: 3.0715 | LR: 3.95e-04 | 1302.1ms/step | 15.4 samples/s +Step 13840/44000 | Loss: 3.0622 | LR: 3.95e-04 | 1302.6ms/step | 15.4 samples/s +Step 13850/44000 | Loss: 3.0820 | LR: 3.95e-04 | 1306.6ms/step | 15.3 samples/s +Step 13860/44000 | Loss: 3.0897 | LR: 3.95e-04 | 1303.5ms/step | 15.3 samples/s +Step 13870/44000 | Loss: 3.0891 | LR: 3.95e-04 | 1302.8ms/step | 15.4 samples/s +Step 13880/44000 | Loss: 3.1075 | LR: 3.95e-04 | 1303.6ms/step | 15.3 samples/s +Step 13890/44000 | Loss: 3.0749 | LR: 3.95e-04 | 1304.2ms/step | 15.3 samples/s +Step 13900/44000 | Loss: 3.1234 | LR: 3.94e-04 | 1305.3ms/step | 15.3 samples/s +Step 13910/44000 | Loss: 3.0960 | LR: 3.94e-04 | 1307.4ms/step | 15.3 samples/s +Step 13920/44000 | Loss: 3.0770 | LR: 3.94e-04 | 1307.5ms/step | 15.3 samples/s +Step 13930/44000 | Loss: 3.0888 | LR: 3.94e-04 | 1303.5ms/step | 15.3 samples/s +Step 13940/44000 | Loss: 3.0700 | LR: 3.94e-04 | 1300.8ms/step | 15.4 samples/s +Step 13950/44000 | Loss: 3.0711 | LR: 3.94e-04 | 1300.0ms/step | 15.4 samples/s +Step 13960/44000 | Loss: 3.0909 | LR: 3.93e-04 | 1297.7ms/step | 15.4 samples/s +Step 13970/44000 | Loss: 3.0744 | LR: 3.93e-04 | 1303.3ms/step | 15.3 samples/s +Step 13980/44000 | Loss: 3.0738 | LR: 3.93e-04 | 1301.0ms/step | 15.4 samples/s +Step 13990/44000 | Loss: 3.0756 | LR: 3.93e-04 | 1298.4ms/step | 15.4 samples/s +Step 14000/44000 | Loss: 3.0948 | LR: 3.93e-04 | 1297.8ms/step | 15.4 samples/s +Validation at step 14000: Loss = 3.0976, Perplexity = 22.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-14000 +Step 14010/44000 | Loss: 3.0966 | LR: 3.93e-04 | 3786.0ms/step | 5.3 samples/s +Step 14020/44000 | Loss: 3.0705 | LR: 3.93e-04 | 1307.2ms/step | 15.3 samples/s +Step 14030/44000 | Loss: 3.0402 | LR: 3.92e-04 | 1304.4ms/step | 15.3 samples/s +Step 14040/44000 | Loss: 3.0572 | LR: 3.92e-04 | 1312.0ms/step | 15.2 samples/s +Step 14050/44000 | Loss: 3.0481 | LR: 3.92e-04 | 1307.1ms/step | 15.3 samples/s +Step 14060/44000 | Loss: 3.1051 | LR: 3.92e-04 | 1304.6ms/step | 15.3 samples/s +Step 14070/44000 | Loss: 3.0975 | LR: 3.92e-04 | 1303.9ms/step | 15.3 samples/s +Step 14080/44000 | Loss: 3.0867 | LR: 3.92e-04 | 1307.1ms/step | 15.3 samples/s +Step 14090/44000 | Loss: 3.0703 | LR: 3.92e-04 | 1307.9ms/step | 15.3 samples/s +Step 14100/44000 | Loss: 3.0990 | LR: 3.91e-04 | 1306.9ms/step | 15.3 samples/s +Step 14110/44000 | Loss: 3.0941 | LR: 3.91e-04 | 1305.5ms/step | 15.3 samples/s +Step 14120/44000 | Loss: 3.0852 | LR: 3.91e-04 | 1310.2ms/step | 15.3 samples/s +Step 14130/44000 | Loss: 3.1076 | LR: 3.91e-04 | 1303.9ms/step | 15.3 samples/s +Step 14140/44000 | Loss: 3.0957 | LR: 3.91e-04 | 1297.6ms/step | 15.4 samples/s +Step 14150/44000 | Loss: 3.0650 | LR: 3.91e-04 | 1297.8ms/step | 15.4 samples/s +Step 14160/44000 | Loss: 3.0768 | LR: 3.90e-04 | 1296.1ms/step | 15.4 samples/s +Step 14170/44000 | Loss: 3.1016 | LR: 3.90e-04 | 1297.1ms/step | 15.4 samples/s +Step 14180/44000 | Loss: 3.0890 | LR: 3.90e-04 | 1299.3ms/step | 15.4 samples/s +Step 14190/44000 | Loss: 3.0835 | LR: 3.90e-04 | 1300.7ms/step | 15.4 samples/s +Step 14200/44000 | Loss: 3.0943 | LR: 3.90e-04 | 1296.6ms/step | 15.4 samples/s +Step 14210/44000 | Loss: 3.0787 | LR: 3.90e-04 | 1295.2ms/step | 15.4 samples/s +Step 14220/44000 | Loss: 3.0795 | LR: 3.90e-04 | 1295.9ms/step | 15.4 samples/s +Step 14230/44000 | Loss: 3.0841 | LR: 3.89e-04 | 1297.0ms/step | 15.4 samples/s +Step 14240/44000 | Loss: 3.0651 | LR: 3.89e-04 | 1301.1ms/step | 15.4 samples/s +Step 14250/44000 | Loss: 3.0793 | LR: 3.89e-04 | 1298.0ms/step | 15.4 samples/s +Validation at step 14250: Loss = 3.0951, Perplexity = 22.09 +Step 14260/44000 | Loss: 3.1104 | LR: 3.89e-04 | 3502.3ms/step | 5.7 samples/s +Step 14270/44000 | Loss: 3.1040 | LR: 3.89e-04 | 1303.7ms/step | 15.3 samples/s +Step 14280/44000 | Loss: 3.1007 | LR: 3.89e-04 | 1307.4ms/step | 15.3 samples/s +Step 14290/44000 | Loss: 3.0869 | LR: 3.89e-04 | 1303.0ms/step | 15.3 samples/s +Step 14300/44000 | Loss: 3.0862 | LR: 3.88e-04 | 1300.9ms/step | 15.4 samples/s +Step 14310/44000 | Loss: 3.0850 | LR: 3.88e-04 | 1300.3ms/step | 15.4 samples/s +Step 14320/44000 | Loss: 3.0827 | LR: 3.88e-04 | 1302.5ms/step | 15.4 samples/s +Step 14330/44000 | Loss: 3.0967 | LR: 3.88e-04 | 1305.4ms/step | 15.3 samples/s +Step 14340/44000 | Loss: 3.0955 | LR: 3.88e-04 | 1310.8ms/step | 15.3 samples/s +Step 14350/44000 | Loss: 3.0769 | LR: 3.88e-04 | 1312.8ms/step | 15.2 samples/s +Step 14360/44000 | Loss: 3.0938 | LR: 3.87e-04 | 1303.6ms/step | 15.3 samples/s +Step 14370/44000 | Loss: 3.0914 | LR: 3.87e-04 | 1300.2ms/step | 15.4 samples/s +Step 14380/44000 | Loss: 3.0728 | LR: 3.87e-04 | 1302.9ms/step | 15.4 samples/s +Step 14390/44000 | Loss: 3.0768 | LR: 3.87e-04 | 1303.0ms/step | 15.3 samples/s +Step 14400/44000 | Loss: 3.0798 | LR: 3.87e-04 | 1298.2ms/step | 15.4 samples/s +Step 14410/44000 | Loss: 3.0599 | LR: 3.87e-04 | 1298.6ms/step | 15.4 samples/s +Step 14420/44000 | Loss: 3.0830 | LR: 3.87e-04 | 1299.3ms/step | 15.4 samples/s +Step 14430/44000 | Loss: 3.0500 | LR: 3.86e-04 | 1300.4ms/step | 15.4 samples/s +Step 14440/44000 | Loss: 3.0906 | LR: 3.86e-04 | 1302.2ms/step | 15.4 samples/s +Step 14450/44000 | Loss: 3.0826 | LR: 3.86e-04 | 1299.8ms/step | 15.4 samples/s +Step 14460/44000 | Loss: 3.0939 | LR: 3.86e-04 | 1301.0ms/step | 15.4 samples/s +Step 14470/44000 | Loss: 3.0902 | LR: 3.86e-04 | 1301.0ms/step | 15.4 samples/s +Step 14480/44000 | Loss: 3.0547 | LR: 3.86e-04 | 1306.5ms/step | 15.3 samples/s +Step 14490/44000 | Loss: 3.0679 | LR: 3.85e-04 | 1301.4ms/step | 15.4 samples/s +Step 14500/44000 | Loss: 3.0955 | LR: 3.85e-04 | 1302.2ms/step | 15.4 samples/s +Validation at step 14500: Loss = 3.0902, Perplexity = 21.98 +Step 14510/44000 | Loss: 3.0642 | LR: 3.85e-04 | 3481.6ms/step | 5.7 samples/s +Step 14520/44000 | Loss: 3.0964 | LR: 3.85e-04 | 1302.1ms/step | 15.4 samples/s +Step 14530/44000 | Loss: 3.0752 | LR: 3.85e-04 | 1302.2ms/step | 15.4 samples/s +Step 14540/44000 | Loss: 3.0856 | LR: 3.85e-04 | 1303.0ms/step | 15.3 samples/s +Step 14550/44000 | Loss: 3.0827 | LR: 3.85e-04 | 1302.2ms/step | 15.4 samples/s +Step 14560/44000 | Loss: 3.0918 | LR: 3.84e-04 | 1302.6ms/step | 15.4 samples/s diff --git a/epfl_final/logs/l8k365m_resampled_resume.log b/epfl_final/logs/l8k365m_resampled_resume.log new file mode 100644 index 0000000000000000000000000000000000000000..3e331326af05221edbafdeef7864e282c7844e72 --- /dev/null +++ b/epfl_final/logs/l8k365m_resampled_resume.log @@ -0,0 +1,3424 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-14000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m resampled 8k nrb31", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "resampled" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32", + "sparse_mask_seed": 0, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-14000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-14000... +Overriding checkpoint LR with config LR: 0.0005 +Resumed from checkpoint at step 14000 (epoch 0, micro_step 140000) +Resuming training from step 14000 +Resuming wandb run 304r0ilz +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 304r0ilz +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_171131-304r0ilz +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m resampled 8k nrb31 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/304r0ilz +Starting training... +Validation at step 14000: Loss = 3.0982, Perplexity = 22.16 +wandb: WARNING Tried to log to step 14000 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 14001: sparse_stochastic (configured sparse attention) +Step 14010/44000 | Loss: 3.0775 | LR: 3.93e-04 | 26114.8ms/step | 0.8 samples/s +wandb: WARNING Tried to log to step 14010 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14020/44000 | Loss: 3.0893 | LR: 3.93e-04 | 1277.2ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14020 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14030/44000 | Loss: 3.0612 | LR: 3.92e-04 | 1280.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14030 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14040/44000 | Loss: 3.1108 | LR: 3.92e-04 | 1280.4ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14040 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14050/44000 | Loss: 3.0747 | LR: 3.92e-04 | 1283.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14050 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14060/44000 | Loss: 3.1223 | LR: 3.92e-04 | 1282.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14060 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14070/44000 | Loss: 3.1111 | LR: 3.92e-04 | 1283.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14070 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14080/44000 | Loss: 3.0900 | LR: 3.92e-04 | 1281.7ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14080 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14090/44000 | Loss: 3.0558 | LR: 3.92e-04 | 1283.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14090 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14100/44000 | Loss: 3.0724 | LR: 3.91e-04 | 1282.6ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14100 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14110/44000 | Loss: 3.0466 | LR: 3.91e-04 | 1283.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14110 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14120/44000 | Loss: 3.0666 | LR: 3.91e-04 | 1283.4ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14120 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14130/44000 | Loss: 3.1002 | LR: 3.91e-04 | 1282.7ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14130 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14140/44000 | Loss: 3.0683 | LR: 3.91e-04 | 1282.4ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14140 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14150/44000 | Loss: 3.0718 | LR: 3.91e-04 | 1282.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14150 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14160/44000 | Loss: 3.0681 | LR: 3.90e-04 | 1279.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14160 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14170/44000 | Loss: 3.1110 | LR: 3.90e-04 | 1277.9ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14170 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14180/44000 | Loss: 3.0632 | LR: 3.90e-04 | 1277.4ms/step | 15.7 samples/s +wandb: WARNING Tried to log to step 14180 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14190/44000 | Loss: 3.0980 | LR: 3.90e-04 | 1280.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14190 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14200/44000 | Loss: 3.1060 | LR: 3.90e-04 | 1283.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14200 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14210/44000 | Loss: 3.0749 | LR: 3.90e-04 | 1283.2ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14210 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14220/44000 | Loss: 3.0619 | LR: 3.90e-04 | 1286.6ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14220 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14230/44000 | Loss: 3.0820 | LR: 3.89e-04 | 1279.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14230 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14240/44000 | Loss: 3.0566 | LR: 3.89e-04 | 1278.5ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14240 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14250/44000 | Loss: 3.0775 | LR: 3.89e-04 | 1279.5ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14250 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 14250: Loss = 3.0946, Perplexity = 22.08 +Step 14260/44000 | Loss: 3.0537 | LR: 3.89e-04 | 3462.5ms/step | 5.8 samples/s +wandb: WARNING Tried to log to step 14260 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14270/44000 | Loss: 3.0690 | LR: 3.89e-04 | 1286.3ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14270 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14280/44000 | Loss: 3.0745 | LR: 3.89e-04 | 1285.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14280 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14290/44000 | Loss: 3.0612 | LR: 3.89e-04 | 1286.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14290 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14300/44000 | Loss: 3.0600 | LR: 3.88e-04 | 1286.1ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14300 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14310/44000 | Loss: 3.1197 | LR: 3.88e-04 | 1287.2ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14310 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14320/44000 | Loss: 3.0818 | LR: 3.88e-04 | 1290.8ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14320 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14330/44000 | Loss: 3.1019 | LR: 3.88e-04 | 1288.2ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14330 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14340/44000 | Loss: 3.0701 | LR: 3.88e-04 | 1287.8ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14340 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14350/44000 | Loss: 3.0968 | LR: 3.88e-04 | 1287.7ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14350 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14360/44000 | Loss: 3.0755 | LR: 3.87e-04 | 1287.4ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14360 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14370/44000 | Loss: 3.0711 | LR: 3.87e-04 | 1287.5ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14370 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14380/44000 | Loss: 3.0487 | LR: 3.87e-04 | 1288.1ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14380 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14390/44000 | Loss: 3.0742 | LR: 3.87e-04 | 1287.4ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14390 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14400/44000 | Loss: 3.0443 | LR: 3.87e-04 | 1290.9ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14400 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14410/44000 | Loss: 3.0676 | LR: 3.87e-04 | 1290.3ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14410 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14420/44000 | Loss: 3.0566 | LR: 3.87e-04 | 1288.3ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14420 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14430/44000 | Loss: 3.0700 | LR: 3.86e-04 | 1290.1ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14430 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14440/44000 | Loss: 3.0592 | LR: 3.86e-04 | 1290.0ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14440 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14450/44000 | Loss: 3.0773 | LR: 3.86e-04 | 1288.3ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14450 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14460/44000 | Loss: 3.0523 | LR: 3.86e-04 | 1288.6ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14460 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14470/44000 | Loss: 3.0965 | LR: 3.86e-04 | 1289.2ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14470 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14480/44000 | Loss: 3.0755 | LR: 3.86e-04 | 1286.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14480 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14490/44000 | Loss: 3.0757 | LR: 3.85e-04 | 1287.4ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 14490 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14500/44000 | Loss: 3.0877 | LR: 3.85e-04 | 1285.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14500 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 14500: Loss = 3.0919, Perplexity = 22.02 +Step 14510/44000 | Loss: 3.0872 | LR: 3.85e-04 | 3993.7ms/step | 5.0 samples/s +wandb: WARNING Tried to log to step 14510 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14520/44000 | Loss: 3.0614 | LR: 3.85e-04 | 1282.0ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14520 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14530/44000 | Loss: 3.0895 | LR: 3.85e-04 | 1282.8ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14530 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14540/44000 | Loss: 3.0803 | LR: 3.85e-04 | 1283.9ms/step | 15.6 samples/s +wandb: WARNING Tried to log to step 14540 that is less than the current step 14541. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 14550/44000 | Loss: 3.0526 | LR: 3.85e-04 | 1284.0ms/step | 15.6 samples/s +Step 14560/44000 | Loss: 3.0991 | LR: 3.84e-04 | 1288.1ms/step | 15.5 samples/s +Step 14570/44000 | Loss: 3.0647 | LR: 3.84e-04 | 1288.0ms/step | 15.5 samples/s +Step 14580/44000 | Loss: 3.0695 | LR: 3.84e-04 | 1286.0ms/step | 15.6 samples/s +Step 14590/44000 | Loss: 3.0725 | LR: 3.84e-04 | 1288.7ms/step | 15.5 samples/s +Step 14600/44000 | Loss: 3.0405 | LR: 3.84e-04 | 1287.4ms/step | 15.5 samples/s +Step 14610/44000 | Loss: 3.0716 | LR: 3.84e-04 | 1286.5ms/step | 15.5 samples/s +Step 14620/44000 | Loss: 3.1079 | LR: 3.83e-04 | 1285.4ms/step | 15.6 samples/s +Step 14630/44000 | Loss: 3.0588 | LR: 3.83e-04 | 1285.5ms/step | 15.6 samples/s +Step 14640/44000 | Loss: 3.0566 | LR: 3.83e-04 | 1285.9ms/step | 15.6 samples/s +Step 14650/44000 | Loss: 3.0762 | LR: 3.83e-04 | 1285.2ms/step | 15.6 samples/s +Step 14660/44000 | Loss: 3.0742 | LR: 3.83e-04 | 1286.9ms/step | 15.5 samples/s +Step 14670/44000 | Loss: 3.0618 | LR: 3.83e-04 | 1289.1ms/step | 15.5 samples/s +Step 14680/44000 | Loss: 3.0849 | LR: 3.83e-04 | 1287.7ms/step | 15.5 samples/s +Step 14690/44000 | Loss: 3.0907 | LR: 3.82e-04 | 1286.5ms/step | 15.5 samples/s +Step 14700/44000 | Loss: 3.0592 | LR: 3.82e-04 | 1288.4ms/step | 15.5 samples/s +Step 14710/44000 | Loss: 3.0899 | LR: 3.82e-04 | 1291.0ms/step | 15.5 samples/s +Step 14720/44000 | Loss: 3.0877 | LR: 3.82e-04 | 1287.1ms/step | 15.5 samples/s +Step 14730/44000 | Loss: 3.0782 | LR: 3.82e-04 | 1286.6ms/step | 15.5 samples/s +Step 14740/44000 | Loss: 3.0824 | LR: 3.82e-04 | 1281.2ms/step | 15.6 samples/s +Step 14750/44000 | Loss: 3.0397 | LR: 3.81e-04 | 1281.7ms/step | 15.6 samples/s +Validation at step 14750: Loss = 3.0874, Perplexity = 21.92 +Step 14760/44000 | Loss: 3.0855 | LR: 3.81e-04 | 3441.3ms/step | 5.8 samples/s +Step 14770/44000 | Loss: 3.0514 | LR: 3.81e-04 | 1281.2ms/step | 15.6 samples/s +Step 14780/44000 | Loss: 3.0481 | LR: 3.81e-04 | 1282.4ms/step | 15.6 samples/s +Step 14790/44000 | Loss: 3.0660 | LR: 3.81e-04 | 1285.9ms/step | 15.6 samples/s +Step 14800/44000 | Loss: 3.0863 | LR: 3.81e-04 | 1285.9ms/step | 15.6 samples/s +Step 14810/44000 | Loss: 3.0470 | LR: 3.81e-04 | 1284.6ms/step | 15.6 samples/s +Step 14820/44000 | Loss: 3.0817 | LR: 3.80e-04 | 1283.8ms/step | 15.6 samples/s +Step 14830/44000 | Loss: 3.0643 | LR: 3.80e-04 | 1283.4ms/step | 15.6 samples/s +Step 14840/44000 | Loss: 3.0701 | LR: 3.80e-04 | 1284.0ms/step | 15.6 samples/s +Step 14850/44000 | Loss: 3.0700 | LR: 3.80e-04 | 1287.7ms/step | 15.5 samples/s +Step 14860/44000 | Loss: 3.0438 | LR: 3.80e-04 | 1286.5ms/step | 15.5 samples/s +Step 14870/44000 | Loss: 3.0442 | LR: 3.80e-04 | 1285.1ms/step | 15.6 samples/s +Step 14880/44000 | Loss: 3.0416 | LR: 3.79e-04 | 1287.0ms/step | 15.5 samples/s +Step 14890/44000 | Loss: 3.0769 | LR: 3.79e-04 | 1283.8ms/step | 15.6 samples/s +Step 14900/44000 | Loss: 3.0842 | LR: 3.79e-04 | 1283.6ms/step | 15.6 samples/s +Step 14910/44000 | Loss: 3.0481 | LR: 3.79e-04 | 1284.7ms/step | 15.6 samples/s +Step 14920/44000 | Loss: 3.0891 | LR: 3.79e-04 | 1286.9ms/step | 15.5 samples/s +Step 14930/44000 | Loss: 3.0690 | LR: 3.79e-04 | 1286.9ms/step | 15.5 samples/s +Step 14940/44000 | Loss: 3.0779 | LR: 3.79e-04 | 1286.8ms/step | 15.5 samples/s +Step 14950/44000 | Loss: 3.0496 | LR: 3.78e-04 | 1283.2ms/step | 15.6 samples/s +Step 14960/44000 | Loss: 3.0105 | LR: 3.78e-04 | 1284.4ms/step | 15.6 samples/s +Step 14970/44000 | Loss: 3.0607 | LR: 3.78e-04 | 1283.8ms/step | 15.6 samples/s +Step 14980/44000 | Loss: 3.0458 | LR: 3.78e-04 | 1284.5ms/step | 15.6 samples/s +Step 14990/44000 | Loss: 3.0787 | LR: 3.78e-04 | 1286.4ms/step | 15.5 samples/s +Step 15000/44000 | Loss: 3.1098 | LR: 3.78e-04 | 1287.7ms/step | 15.5 samples/s +Validation at step 15000: Loss = 3.0847, Perplexity = 21.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-15000 +Step 15010/44000 | Loss: 3.0846 | LR: 3.77e-04 | 3748.6ms/step | 5.3 samples/s +Step 15020/44000 | Loss: 3.0554 | LR: 3.77e-04 | 1282.8ms/step | 15.6 samples/s +Step 15030/44000 | Loss: 3.0743 | LR: 3.77e-04 | 1284.1ms/step | 15.6 samples/s +Step 15040/44000 | Loss: 3.0835 | LR: 3.77e-04 | 1283.6ms/step | 15.6 samples/s +Step 15050/44000 | Loss: 3.0595 | LR: 3.77e-04 | 1283.1ms/step | 15.6 samples/s +Step 15060/44000 | Loss: 3.0931 | LR: 3.77e-04 | 1284.7ms/step | 15.6 samples/s +Step 15070/44000 | Loss: 3.0562 | LR: 3.77e-04 | 1286.9ms/step | 15.5 samples/s +Step 15080/44000 | Loss: 3.0953 | LR: 3.76e-04 | 1279.2ms/step | 15.6 samples/s +Step 15090/44000 | Loss: 3.0703 | LR: 3.76e-04 | 1280.7ms/step | 15.6 samples/s +Step 15100/44000 | Loss: 3.0455 | LR: 3.76e-04 | 1285.0ms/step | 15.6 samples/s +Step 15110/44000 | Loss: 3.0876 | LR: 3.76e-04 | 1284.4ms/step | 15.6 samples/s +Step 15120/44000 | Loss: 3.0913 | LR: 3.76e-04 | 1284.4ms/step | 15.6 samples/s +Step 15130/44000 | Loss: 3.0640 | LR: 3.76e-04 | 1287.1ms/step | 15.5 samples/s +Step 15140/44000 | Loss: 3.0750 | LR: 3.75e-04 | 1285.8ms/step | 15.6 samples/s +Step 15150/44000 | Loss: 3.0729 | LR: 3.75e-04 | 1284.1ms/step | 15.6 samples/s +Step 15160/44000 | Loss: 3.0957 | LR: 3.75e-04 | 1287.1ms/step | 15.5 samples/s +Step 15170/44000 | Loss: 3.0674 | LR: 3.75e-04 | 1285.6ms/step | 15.6 samples/s +Step 15180/44000 | Loss: 3.1000 | LR: 3.75e-04 | 1286.6ms/step | 15.5 samples/s +Step 15190/44000 | Loss: 3.0633 | LR: 3.75e-04 | 1287.8ms/step | 15.5 samples/s +Step 15200/44000 | Loss: 3.0999 | LR: 3.74e-04 | 1289.2ms/step | 15.5 samples/s +Step 15210/44000 | Loss: 3.0640 | LR: 3.74e-04 | 1284.9ms/step | 15.6 samples/s +Step 15220/44000 | Loss: 3.0555 | LR: 3.74e-04 | 1290.3ms/step | 15.5 samples/s +Step 15230/44000 | Loss: 3.0414 | LR: 3.74e-04 | 1283.3ms/step | 15.6 samples/s +Step 15240/44000 | Loss: 3.0228 | LR: 3.74e-04 | 1285.8ms/step | 15.6 samples/s +Step 15250/44000 | Loss: 3.0785 | LR: 3.74e-04 | 1285.4ms/step | 15.6 samples/s +Validation at step 15250: Loss = 3.0807, Perplexity = 21.77 +Step 15260/44000 | Loss: 3.0537 | LR: 3.74e-04 | 3447.8ms/step | 5.8 samples/s +Step 15270/44000 | Loss: 3.0515 | LR: 3.73e-04 | 1283.9ms/step | 15.6 samples/s +Step 15280/44000 | Loss: 3.0560 | LR: 3.73e-04 | 1280.0ms/step | 15.6 samples/s +Step 15290/44000 | Loss: 3.0645 | LR: 3.73e-04 | 1284.3ms/step | 15.6 samples/s +Step 15300/44000 | Loss: 3.0697 | LR: 3.73e-04 | 1284.7ms/step | 15.6 samples/s +Step 15310/44000 | Loss: 3.0819 | LR: 3.73e-04 | 1279.9ms/step | 15.6 samples/s +Step 15320/44000 | Loss: 3.0709 | LR: 3.73e-04 | 1281.7ms/step | 15.6 samples/s +Step 15330/44000 | Loss: 3.0547 | LR: 3.72e-04 | 1285.6ms/step | 15.6 samples/s +Step 15340/44000 | Loss: 3.0602 | LR: 3.72e-04 | 1285.9ms/step | 15.6 samples/s +Step 15350/44000 | Loss: 3.0516 | LR: 3.72e-04 | 1286.2ms/step | 15.5 samples/s +Step 15360/44000 | Loss: 3.0558 | LR: 3.72e-04 | 1288.7ms/step | 15.5 samples/s +Step 15370/44000 | Loss: 3.0607 | LR: 3.72e-04 | 1281.4ms/step | 15.6 samples/s +Step 15380/44000 | Loss: 3.0627 | LR: 3.72e-04 | 1284.1ms/step | 15.6 samples/s +Step 15390/44000 | Loss: 3.0312 | LR: 3.71e-04 | 1287.0ms/step | 15.5 samples/s +Step 15400/44000 | Loss: 3.0425 | LR: 3.71e-04 | 1283.7ms/step | 15.6 samples/s +Step 15410/44000 | Loss: 3.0364 | LR: 3.71e-04 | 1282.4ms/step | 15.6 samples/s +Step 15420/44000 | Loss: 3.0659 | LR: 3.71e-04 | 1280.7ms/step | 15.6 samples/s +Step 15430/44000 | Loss: 3.0515 | LR: 3.71e-04 | 1281.7ms/step | 15.6 samples/s +Step 15440/44000 | Loss: 3.0902 | LR: 3.71e-04 | 1280.0ms/step | 15.6 samples/s +Step 15450/44000 | Loss: 3.0680 | LR: 3.71e-04 | 1278.7ms/step | 15.6 samples/s +Step 15460/44000 | Loss: 3.0653 | LR: 3.70e-04 | 1282.7ms/step | 15.6 samples/s +Step 15470/44000 | Loss: 3.0608 | LR: 3.70e-04 | 1282.0ms/step | 15.6 samples/s +Step 15480/44000 | Loss: 3.0586 | LR: 3.70e-04 | 1284.1ms/step | 15.6 samples/s +Step 15490/44000 | Loss: 3.0530 | LR: 3.70e-04 | 1285.1ms/step | 15.6 samples/s +Step 15500/44000 | Loss: 3.0944 | LR: 3.70e-04 | 1282.5ms/step | 15.6 samples/s +Validation at step 15500: Loss = 3.0774, Perplexity = 21.70 +Step 15510/44000 | Loss: 3.0622 | LR: 3.70e-04 | 3440.0ms/step | 5.8 samples/s +Step 15520/44000 | Loss: 3.0849 | LR: 3.69e-04 | 1285.1ms/step | 15.6 samples/s +Step 15530/44000 | Loss: 3.0478 | LR: 3.69e-04 | 1282.3ms/step | 15.6 samples/s +Step 15540/44000 | Loss: 3.0362 | LR: 3.69e-04 | 1282.5ms/step | 15.6 samples/s +Step 15550/44000 | Loss: 3.0389 | LR: 3.69e-04 | 1279.8ms/step | 15.6 samples/s +Step 15560/44000 | Loss: 3.0544 | LR: 3.69e-04 | 1277.2ms/step | 15.7 samples/s +Step 15570/44000 | Loss: 3.0624 | LR: 3.69e-04 | 1278.1ms/step | 15.6 samples/s +Step 15580/44000 | Loss: 3.0681 | LR: 3.68e-04 | 1276.0ms/step | 15.7 samples/s +Step 15590/44000 | Loss: 3.0766 | LR: 3.68e-04 | 1283.5ms/step | 15.6 samples/s +Step 15600/44000 | Loss: 3.0572 | LR: 3.68e-04 | 1278.0ms/step | 15.6 samples/s +Step 15610/44000 | Loss: 3.0709 | LR: 3.68e-04 | 1280.7ms/step | 15.6 samples/s +Step 15620/44000 | Loss: 3.0806 | LR: 3.68e-04 | 1282.2ms/step | 15.6 samples/s +Step 15630/44000 | Loss: 3.0431 | LR: 3.68e-04 | 1283.8ms/step | 15.6 samples/s +Step 15640/44000 | Loss: 3.0625 | LR: 3.67e-04 | 1281.6ms/step | 15.6 samples/s +Step 15650/44000 | Loss: 3.0647 | LR: 3.67e-04 | 1282.4ms/step | 15.6 samples/s +Step 15660/44000 | Loss: 3.0364 | LR: 3.67e-04 | 1286.2ms/step | 15.6 samples/s +Step 15670/44000 | Loss: 3.0510 | LR: 3.67e-04 | 1282.3ms/step | 15.6 samples/s +Step 15680/44000 | Loss: 3.1095 | LR: 3.67e-04 | 1283.8ms/step | 15.6 samples/s +Step 15690/44000 | Loss: 3.0810 | LR: 3.67e-04 | 1276.9ms/step | 15.7 samples/s +Step 15700/44000 | Loss: 3.0943 | LR: 3.67e-04 | 1277.6ms/step | 15.7 samples/s +Step 15710/44000 | Loss: 3.1030 | LR: 3.66e-04 | 1280.6ms/step | 15.6 samples/s +Step 15720/44000 | Loss: 3.0460 | LR: 3.66e-04 | 1283.5ms/step | 15.6 samples/s +Step 15730/44000 | Loss: 3.0561 | LR: 3.66e-04 | 1282.4ms/step | 15.6 samples/s +Step 15740/44000 | Loss: 3.0514 | LR: 3.66e-04 | 1295.9ms/step | 15.4 samples/s +Step 15750/44000 | Loss: 3.0477 | LR: 3.66e-04 | 1281.8ms/step | 15.6 samples/s +Validation at step 15750: Loss = 3.0744, Perplexity = 21.64 +Step 15760/44000 | Loss: 3.0497 | LR: 3.66e-04 | 3437.9ms/step | 5.8 samples/s +Step 15770/44000 | Loss: 3.0416 | LR: 3.65e-04 | 1278.3ms/step | 15.6 samples/s +Step 15780/44000 | Loss: 3.0558 | LR: 3.65e-04 | 1283.8ms/step | 15.6 samples/s +Step 15790/44000 | Loss: 3.0609 | LR: 3.65e-04 | 1280.8ms/step | 15.6 samples/s +Step 15800/44000 | Loss: 3.0539 | LR: 3.65e-04 | 1279.1ms/step | 15.6 samples/s +Step 15810/44000 | Loss: 3.0400 | LR: 3.65e-04 | 1278.6ms/step | 15.6 samples/s +Step 15820/44000 | Loss: 3.0445 | LR: 3.65e-04 | 1276.0ms/step | 15.7 samples/s +Step 15830/44000 | Loss: 3.0695 | LR: 3.64e-04 | 1280.7ms/step | 15.6 samples/s +Step 15840/44000 | Loss: 3.0582 | LR: 3.64e-04 | 1280.4ms/step | 15.6 samples/s +Step 15850/44000 | Loss: 3.0425 | LR: 3.64e-04 | 1285.1ms/step | 15.6 samples/s +Step 15860/44000 | Loss: 3.0481 | LR: 3.64e-04 | 1281.3ms/step | 15.6 samples/s +Step 15870/44000 | Loss: 3.0574 | LR: 3.64e-04 | 1284.9ms/step | 15.6 samples/s +Step 15880/44000 | Loss: 3.0648 | LR: 3.64e-04 | 1283.0ms/step | 15.6 samples/s +Step 15890/44000 | Loss: 3.0299 | LR: 3.63e-04 | 1279.9ms/step | 15.6 samples/s +Step 15900/44000 | Loss: 3.0721 | LR: 3.63e-04 | 1281.2ms/step | 15.6 samples/s +Step 15910/44000 | Loss: 3.0694 | LR: 3.63e-04 | 1280.3ms/step | 15.6 samples/s +Step 15920/44000 | Loss: 3.0666 | LR: 3.63e-04 | 1280.3ms/step | 15.6 samples/s +Step 15930/44000 | Loss: 3.0234 | LR: 3.63e-04 | 1279.8ms/step | 15.6 samples/s +Step 15940/44000 | Loss: 3.0470 | LR: 3.63e-04 | 1279.2ms/step | 15.6 samples/s +Step 15950/44000 | Loss: 3.0276 | LR: 3.62e-04 | 1279.9ms/step | 15.6 samples/s +Step 15960/44000 | Loss: 3.0358 | LR: 3.62e-04 | 1280.2ms/step | 15.6 samples/s +Step 15970/44000 | Loss: 3.0485 | LR: 3.62e-04 | 1279.0ms/step | 15.6 samples/s +Step 15980/44000 | Loss: 3.0581 | LR: 3.62e-04 | 1277.0ms/step | 15.7 samples/s +Step 15990/44000 | Loss: 3.0637 | LR: 3.62e-04 | 1280.0ms/step | 15.6 samples/s +Step 16000/44000 | Loss: 3.0679 | LR: 3.62e-04 | 1282.2ms/step | 15.6 samples/s +Validation at step 16000: Loss = 3.0712, Perplexity = 21.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-16000 +Step 16010/44000 | Loss: 3.0193 | LR: 3.62e-04 | 3709.8ms/step | 5.4 samples/s +Step 16020/44000 | Loss: 3.0380 | LR: 3.61e-04 | 1277.6ms/step | 15.7 samples/s +Step 16030/44000 | Loss: 3.0548 | LR: 3.61e-04 | 1278.0ms/step | 15.6 samples/s +Step 16040/44000 | Loss: 3.0405 | LR: 3.61e-04 | 1277.6ms/step | 15.7 samples/s +Step 16050/44000 | Loss: 3.0747 | LR: 3.61e-04 | 1277.7ms/step | 15.7 samples/s +Step 16060/44000 | Loss: 3.0441 | LR: 3.61e-04 | 1282.8ms/step | 15.6 samples/s +Step 16070/44000 | Loss: 3.0665 | LR: 3.61e-04 | 1282.2ms/step | 15.6 samples/s +Step 16080/44000 | Loss: 3.0561 | LR: 3.60e-04 | 1283.4ms/step | 15.6 samples/s +Step 16090/44000 | Loss: 3.0225 | LR: 3.60e-04 | 1282.1ms/step | 15.6 samples/s +Step 16100/44000 | Loss: 3.0448 | LR: 3.60e-04 | 1281.6ms/step | 15.6 samples/s +Step 16110/44000 | Loss: 3.0600 | LR: 3.60e-04 | 1280.4ms/step | 15.6 samples/s +Step 16120/44000 | Loss: 3.0582 | LR: 3.60e-04 | 1281.8ms/step | 15.6 samples/s +Step 16130/44000 | Loss: 3.0415 | LR: 3.60e-04 | 1282.5ms/step | 15.6 samples/s +Step 16140/44000 | Loss: 3.0503 | LR: 3.59e-04 | 1281.9ms/step | 15.6 samples/s +Step 16150/44000 | Loss: 3.0290 | LR: 3.59e-04 | 1283.4ms/step | 15.6 samples/s +Step 16160/44000 | Loss: 3.0444 | LR: 3.59e-04 | 1281.1ms/step | 15.6 samples/s +Step 16170/44000 | Loss: 3.0681 | LR: 3.59e-04 | 1278.0ms/step | 15.6 samples/s +Step 16180/44000 | Loss: 3.0752 | LR: 3.59e-04 | 1280.6ms/step | 15.6 samples/s +Step 16190/44000 | Loss: 3.0654 | LR: 3.59e-04 | 1278.4ms/step | 15.6 samples/s +Step 16200/44000 | Loss: 3.0354 | LR: 3.58e-04 | 1282.1ms/step | 15.6 samples/s +Step 16210/44000 | Loss: 3.0350 | LR: 3.58e-04 | 1282.4ms/step | 15.6 samples/s +Step 16220/44000 | Loss: 3.0811 | LR: 3.58e-04 | 1281.4ms/step | 15.6 samples/s +Step 16230/44000 | Loss: 3.0196 | LR: 3.58e-04 | 1280.7ms/step | 15.6 samples/s +Step 16240/44000 | Loss: 3.0332 | LR: 3.58e-04 | 1281.1ms/step | 15.6 samples/s +Step 16250/44000 | Loss: 3.0754 | LR: 3.58e-04 | 1280.4ms/step | 15.6 samples/s +Validation at step 16250: Loss = 3.0674, Perplexity = 21.49 +Step 16260/44000 | Loss: 3.0331 | LR: 3.57e-04 | 3426.8ms/step | 5.8 samples/s +Step 16270/44000 | Loss: 3.0930 | LR: 3.57e-04 | 1279.0ms/step | 15.6 samples/s +Step 16280/44000 | Loss: 3.0520 | LR: 3.57e-04 | 1279.7ms/step | 15.6 samples/s +Step 16290/44000 | Loss: 3.0318 | LR: 3.57e-04 | 1278.0ms/step | 15.6 samples/s +Step 16300/44000 | Loss: 3.0790 | LR: 3.57e-04 | 1280.6ms/step | 15.6 samples/s +Step 16310/44000 | Loss: 3.0627 | LR: 3.57e-04 | 1279.7ms/step | 15.6 samples/s +Step 16320/44000 | Loss: 3.0209 | LR: 3.56e-04 | 1279.2ms/step | 15.6 samples/s +Step 16330/44000 | Loss: 3.0374 | LR: 3.56e-04 | 1276.7ms/step | 15.7 samples/s +Step 16340/44000 | Loss: 3.0630 | LR: 3.56e-04 | 1279.6ms/step | 15.6 samples/s +Step 16350/44000 | Loss: 3.0611 | LR: 3.56e-04 | 1280.4ms/step | 15.6 samples/s +Step 16360/44000 | Loss: 3.0670 | LR: 3.56e-04 | 1278.1ms/step | 15.6 samples/s +Step 16370/44000 | Loss: 3.0378 | LR: 3.56e-04 | 1278.7ms/step | 15.6 samples/s +Step 16380/44000 | Loss: 3.0530 | LR: 3.55e-04 | 1278.6ms/step | 15.6 samples/s +Step 16390/44000 | Loss: 3.0874 | LR: 3.55e-04 | 1280.0ms/step | 15.6 samples/s +Step 16400/44000 | Loss: 3.0333 | LR: 3.55e-04 | 1282.8ms/step | 15.6 samples/s +Step 16410/44000 | Loss: 3.0450 | LR: 3.55e-04 | 1282.1ms/step | 15.6 samples/s +Step 16420/44000 | Loss: 3.0400 | LR: 3.55e-04 | 1276.3ms/step | 15.7 samples/s +Step 16430/44000 | Loss: 3.0611 | LR: 3.55e-04 | 1277.9ms/step | 15.7 samples/s +Step 16440/44000 | Loss: 3.0463 | LR: 3.54e-04 | 1279.3ms/step | 15.6 samples/s +Step 16450/44000 | Loss: 3.0516 | LR: 3.54e-04 | 1280.1ms/step | 15.6 samples/s +Step 16460/44000 | Loss: 3.0634 | LR: 3.54e-04 | 1278.8ms/step | 15.6 samples/s +Step 16470/44000 | Loss: 3.0274 | LR: 3.54e-04 | 1279.2ms/step | 15.6 samples/s +Step 16480/44000 | Loss: 3.0324 | LR: 3.54e-04 | 1280.6ms/step | 15.6 samples/s +Step 16490/44000 | Loss: 3.0286 | LR: 3.54e-04 | 1281.6ms/step | 15.6 samples/s +Step 16500/44000 | Loss: 3.0418 | LR: 3.53e-04 | 1280.3ms/step | 15.6 samples/s +Validation at step 16500: Loss = 3.0646, Perplexity = 21.43 +Step 16510/44000 | Loss: 3.0636 | LR: 3.53e-04 | 3431.5ms/step | 5.8 samples/s +Step 16520/44000 | Loss: 3.0398 | LR: 3.53e-04 | 1282.1ms/step | 15.6 samples/s +Step 16530/44000 | Loss: 3.0154 | LR: 3.53e-04 | 1281.8ms/step | 15.6 samples/s +Step 16540/44000 | Loss: 3.0517 | LR: 3.53e-04 | 1282.2ms/step | 15.6 samples/s +Step 16550/44000 | Loss: 3.0066 | LR: 3.53e-04 | 1280.7ms/step | 15.6 samples/s +Step 16560/44000 | Loss: 3.0321 | LR: 3.52e-04 | 1279.0ms/step | 15.6 samples/s +Step 16570/44000 | Loss: 3.0650 | LR: 3.52e-04 | 1276.5ms/step | 15.7 samples/s +Step 16580/44000 | Loss: 3.0305 | LR: 3.52e-04 | 1274.6ms/step | 15.7 samples/s +Step 16590/44000 | Loss: 3.0421 | LR: 3.52e-04 | 1275.8ms/step | 15.7 samples/s +Step 16600/44000 | Loss: 3.0283 | LR: 3.52e-04 | 1283.9ms/step | 15.6 samples/s +Step 16610/44000 | Loss: 3.0483 | LR: 3.52e-04 | 1282.0ms/step | 15.6 samples/s +Step 16620/44000 | Loss: 3.0499 | LR: 3.51e-04 | 1278.6ms/step | 15.6 samples/s +Step 16630/44000 | Loss: 3.0276 | LR: 3.51e-04 | 1277.9ms/step | 15.7 samples/s +Step 16640/44000 | Loss: 3.0628 | LR: 3.51e-04 | 1283.3ms/step | 15.6 samples/s +Step 16650/44000 | Loss: 3.0275 | LR: 3.51e-04 | 1280.2ms/step | 15.6 samples/s +Step 16660/44000 | Loss: 3.0275 | LR: 3.51e-04 | 1278.9ms/step | 15.6 samples/s +Step 16670/44000 | Loss: 3.0534 | LR: 3.51e-04 | 1280.0ms/step | 15.6 samples/s +Step 16680/44000 | Loss: 3.0081 | LR: 3.50e-04 | 1287.8ms/step | 15.5 samples/s +Step 16690/44000 | Loss: 3.0354 | LR: 3.50e-04 | 1281.2ms/step | 15.6 samples/s +Step 16700/44000 | Loss: 3.0706 | LR: 3.50e-04 | 1278.9ms/step | 15.6 samples/s +Step 16710/44000 | Loss: 3.0092 | LR: 3.50e-04 | 1278.4ms/step | 15.6 samples/s +Step 16720/44000 | Loss: 3.0542 | LR: 3.50e-04 | 1283.3ms/step | 15.6 samples/s +Step 16730/44000 | Loss: 3.0212 | LR: 3.50e-04 | 1278.5ms/step | 15.6 samples/s +Step 16740/44000 | Loss: 3.0247 | LR: 3.49e-04 | 1278.7ms/step | 15.6 samples/s +Step 16750/44000 | Loss: 3.0202 | LR: 3.49e-04 | 1279.4ms/step | 15.6 samples/s +Validation at step 16750: Loss = 3.0616, Perplexity = 21.36 +Step 16760/44000 | Loss: 3.0640 | LR: 3.49e-04 | 3426.6ms/step | 5.8 samples/s +Step 16770/44000 | Loss: 3.0682 | LR: 3.49e-04 | 1279.2ms/step | 15.6 samples/s +Step 16780/44000 | Loss: 3.0574 | LR: 3.49e-04 | 1280.0ms/step | 15.6 samples/s +Step 16790/44000 | Loss: 3.0752 | LR: 3.49e-04 | 1280.4ms/step | 15.6 samples/s +Step 16800/44000 | Loss: 3.0499 | LR: 3.48e-04 | 1280.6ms/step | 15.6 samples/s +Step 16810/44000 | Loss: 3.0264 | LR: 3.48e-04 | 1277.7ms/step | 15.7 samples/s +Step 16820/44000 | Loss: 3.0742 | LR: 3.48e-04 | 1277.6ms/step | 15.7 samples/s +Step 16830/44000 | Loss: 3.0681 | LR: 3.48e-04 | 1280.7ms/step | 15.6 samples/s +Step 16840/44000 | Loss: 3.0391 | LR: 3.48e-04 | 1281.4ms/step | 15.6 samples/s +Step 16850/44000 | Loss: 3.0272 | LR: 3.48e-04 | 1279.7ms/step | 15.6 samples/s +Step 16860/44000 | Loss: 3.0664 | LR: 3.47e-04 | 1277.7ms/step | 15.7 samples/s +Step 16870/44000 | Loss: 3.0395 | LR: 3.47e-04 | 1280.9ms/step | 15.6 samples/s +Step 16880/44000 | Loss: 3.0612 | LR: 3.47e-04 | 1281.4ms/step | 15.6 samples/s +Step 16890/44000 | Loss: 3.0300 | LR: 3.47e-04 | 1282.3ms/step | 15.6 samples/s +Step 16900/44000 | Loss: 3.0549 | LR: 3.47e-04 | 1282.0ms/step | 15.6 samples/s +Step 16910/44000 | Loss: 3.0420 | LR: 3.47e-04 | 1282.5ms/step | 15.6 samples/s +Step 16920/44000 | Loss: 3.0547 | LR: 3.46e-04 | 1283.2ms/step | 15.6 samples/s +Step 16930/44000 | Loss: 3.0748 | LR: 3.46e-04 | 1280.7ms/step | 15.6 samples/s +Step 16940/44000 | Loss: 3.0368 | LR: 3.46e-04 | 1281.4ms/step | 15.6 samples/s +Step 16950/44000 | Loss: 3.0270 | LR: 3.46e-04 | 1279.9ms/step | 15.6 samples/s +Step 16960/44000 | Loss: 3.0397 | LR: 3.46e-04 | 1279.6ms/step | 15.6 samples/s +Step 16970/44000 | Loss: 3.0653 | LR: 3.46e-04 | 1278.5ms/step | 15.6 samples/s +Step 16980/44000 | Loss: 3.0741 | LR: 3.45e-04 | 1281.0ms/step | 15.6 samples/s +Step 16990/44000 | Loss: 3.0322 | LR: 3.45e-04 | 1281.3ms/step | 15.6 samples/s +Step 17000/44000 | Loss: 3.0425 | LR: 3.45e-04 | 1279.6ms/step | 15.6 samples/s +Validation at step 17000: Loss = 3.0591, Perplexity = 21.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-17000 +Step 17010/44000 | Loss: 3.0595 | LR: 3.45e-04 | 3690.1ms/step | 5.4 samples/s +Step 17020/44000 | Loss: 3.0373 | LR: 3.45e-04 | 1281.6ms/step | 15.6 samples/s +Step 17030/44000 | Loss: 3.0449 | LR: 3.45e-04 | 1280.6ms/step | 15.6 samples/s +Step 17040/44000 | Loss: 3.0797 | LR: 3.44e-04 | 1283.5ms/step | 15.6 samples/s +Step 17050/44000 | Loss: 3.0710 | LR: 3.44e-04 | 1289.4ms/step | 15.5 samples/s +Step 17060/44000 | Loss: 3.0247 | LR: 3.44e-04 | 1276.8ms/step | 15.7 samples/s +Step 17070/44000 | Loss: 3.0080 | LR: 3.44e-04 | 1283.8ms/step | 15.6 samples/s +Step 17080/44000 | Loss: 3.0558 | LR: 3.44e-04 | 1286.0ms/step | 15.6 samples/s +Step 17090/44000 | Loss: 3.0390 | LR: 3.44e-04 | 1281.4ms/step | 15.6 samples/s +Step 17100/44000 | Loss: 3.0447 | LR: 3.43e-04 | 1282.4ms/step | 15.6 samples/s +Step 17110/44000 | Loss: 3.0348 | LR: 3.43e-04 | 1282.3ms/step | 15.6 samples/s +Step 17120/44000 | Loss: 3.0427 | LR: 3.43e-04 | 1280.3ms/step | 15.6 samples/s +Step 17130/44000 | Loss: 3.0297 | LR: 3.43e-04 | 1280.1ms/step | 15.6 samples/s +Step 17140/44000 | Loss: 3.0582 | LR: 3.43e-04 | 1281.7ms/step | 15.6 samples/s +Step 17150/44000 | Loss: 3.0596 | LR: 3.43e-04 | 1284.1ms/step | 15.6 samples/s +Step 17160/44000 | Loss: 3.0235 | LR: 3.42e-04 | 1282.4ms/step | 15.6 samples/s +Step 17170/44000 | Loss: 3.0363 | LR: 3.42e-04 | 1282.4ms/step | 15.6 samples/s +Step 17180/44000 | Loss: 3.0193 | LR: 3.42e-04 | 1278.8ms/step | 15.6 samples/s +Step 17190/44000 | Loss: 3.0626 | LR: 3.42e-04 | 1281.0ms/step | 15.6 samples/s +Step 17200/44000 | Loss: 3.0355 | LR: 3.42e-04 | 1281.9ms/step | 15.6 samples/s +Step 17210/44000 | Loss: 3.0552 | LR: 3.42e-04 | 1280.1ms/step | 15.6 samples/s +Step 17220/44000 | Loss: 3.0435 | LR: 3.41e-04 | 1281.5ms/step | 15.6 samples/s +Step 17230/44000 | Loss: 3.0361 | LR: 3.41e-04 | 1280.8ms/step | 15.6 samples/s +Step 17240/44000 | Loss: 3.0378 | LR: 3.41e-04 | 1281.5ms/step | 15.6 samples/s +Step 17250/44000 | Loss: 3.0490 | LR: 3.41e-04 | 1282.5ms/step | 15.6 samples/s +Validation at step 17250: Loss = 3.0554, Perplexity = 21.23 +Step 17260/44000 | Loss: 3.0423 | LR: 3.41e-04 | 3426.0ms/step | 5.8 samples/s +Step 17270/44000 | Loss: 3.0530 | LR: 3.41e-04 | 1278.8ms/step | 15.6 samples/s +Step 17280/44000 | Loss: 3.0411 | LR: 3.40e-04 | 1279.6ms/step | 15.6 samples/s +Step 17290/44000 | Loss: 3.0398 | LR: 3.40e-04 | 1280.5ms/step | 15.6 samples/s +Step 17300/44000 | Loss: 3.0264 | LR: 3.40e-04 | 1281.1ms/step | 15.6 samples/s +Step 17310/44000 | Loss: 3.0340 | LR: 3.40e-04 | 1276.8ms/step | 15.7 samples/s +Step 17320/44000 | Loss: 3.0385 | LR: 3.40e-04 | 1279.5ms/step | 15.6 samples/s +Step 17330/44000 | Loss: 3.0427 | LR: 3.40e-04 | 1282.9ms/step | 15.6 samples/s +Step 17340/44000 | Loss: 3.0414 | LR: 3.39e-04 | 1281.8ms/step | 15.6 samples/s +Step 17350/44000 | Loss: 3.0400 | LR: 3.39e-04 | 1282.0ms/step | 15.6 samples/s +Step 17360/44000 | Loss: 3.0346 | LR: 3.39e-04 | 1276.1ms/step | 15.7 samples/s +Step 17370/44000 | Loss: 3.0108 | LR: 3.39e-04 | 1281.1ms/step | 15.6 samples/s +Step 17380/44000 | Loss: 3.0588 | LR: 3.39e-04 | 1279.0ms/step | 15.6 samples/s +Step 17390/44000 | Loss: 3.0321 | LR: 3.39e-04 | 1279.2ms/step | 15.6 samples/s +Step 17400/44000 | Loss: 3.0618 | LR: 3.38e-04 | 1278.3ms/step | 15.6 samples/s +Step 17410/44000 | Loss: 3.0421 | LR: 3.38e-04 | 1277.5ms/step | 15.7 samples/s +Step 17420/44000 | Loss: 3.0165 | LR: 3.38e-04 | 1279.9ms/step | 15.6 samples/s +Step 17430/44000 | Loss: 3.0460 | LR: 3.38e-04 | 1279.8ms/step | 15.6 samples/s +Step 17440/44000 | Loss: 3.0375 | LR: 3.38e-04 | 1280.2ms/step | 15.6 samples/s +Step 17450/44000 | Loss: 3.0128 | LR: 3.38e-04 | 1276.4ms/step | 15.7 samples/s +Step 17460/44000 | Loss: 3.0307 | LR: 3.37e-04 | 1275.9ms/step | 15.7 samples/s +Step 17470/44000 | Loss: 3.0338 | LR: 3.37e-04 | 1276.3ms/step | 15.7 samples/s +Step 17480/44000 | Loss: 3.0575 | LR: 3.37e-04 | 1279.3ms/step | 15.6 samples/s +Step 17490/44000 | Loss: 3.0123 | LR: 3.37e-04 | 1277.8ms/step | 15.7 samples/s +Step 17500/44000 | Loss: 3.0631 | LR: 3.37e-04 | 1280.6ms/step | 15.6 samples/s +Validation at step 17500: Loss = 3.0518, Perplexity = 21.15 +Step 17510/44000 | Loss: 3.0373 | LR: 3.37e-04 | 3433.1ms/step | 5.8 samples/s +Step 17520/44000 | Loss: 2.9988 | LR: 3.36e-04 | 1280.9ms/step | 15.6 samples/s +Step 17530/44000 | Loss: 3.0366 | LR: 3.36e-04 | 1280.3ms/step | 15.6 samples/s +Step 17540/44000 | Loss: 3.0623 | LR: 3.36e-04 | 1277.2ms/step | 15.7 samples/s +Step 17550/44000 | Loss: 3.0247 | LR: 3.36e-04 | 1280.0ms/step | 15.6 samples/s +Step 17560/44000 | Loss: 3.0249 | LR: 3.36e-04 | 1278.2ms/step | 15.6 samples/s +Step 17570/44000 | Loss: 3.0587 | LR: 3.35e-04 | 1278.3ms/step | 15.6 samples/s +Step 17580/44000 | Loss: 3.0347 | LR: 3.35e-04 | 1277.5ms/step | 15.7 samples/s +Step 17590/44000 | Loss: 3.0386 | LR: 3.35e-04 | 1278.4ms/step | 15.6 samples/s +Step 17600/44000 | Loss: 3.0473 | LR: 3.35e-04 | 1280.3ms/step | 15.6 samples/s +Step 17610/44000 | Loss: 3.0449 | LR: 3.35e-04 | 1278.8ms/step | 15.6 samples/s +Step 17620/44000 | Loss: 3.0951 | LR: 3.35e-04 | 1275.0ms/step | 15.7 samples/s +Step 17630/44000 | Loss: 3.0628 | LR: 3.34e-04 | 1278.1ms/step | 15.6 samples/s +Step 17640/44000 | Loss: 3.0509 | LR: 3.34e-04 | 1275.2ms/step | 15.7 samples/s +Step 17650/44000 | Loss: 3.0193 | LR: 3.34e-04 | 1276.1ms/step | 15.7 samples/s +Step 17660/44000 | Loss: 3.0488 | LR: 3.34e-04 | 1278.7ms/step | 15.6 samples/s +Step 17670/44000 | Loss: 3.0595 | LR: 3.34e-04 | 1279.6ms/step | 15.6 samples/s +Step 17680/44000 | Loss: 3.0078 | LR: 3.34e-04 | 1279.9ms/step | 15.6 samples/s +Step 17690/44000 | Loss: 3.0301 | LR: 3.33e-04 | 1280.5ms/step | 15.6 samples/s +Step 17700/44000 | Loss: 3.0551 | LR: 3.33e-04 | 1279.0ms/step | 15.6 samples/s +Step 17710/44000 | Loss: 3.0286 | LR: 3.33e-04 | 1278.2ms/step | 15.6 samples/s +Step 17720/44000 | Loss: 3.0216 | LR: 3.33e-04 | 1276.9ms/step | 15.7 samples/s +Step 17730/44000 | Loss: 3.0041 | LR: 3.33e-04 | 1277.8ms/step | 15.7 samples/s +Step 17740/44000 | Loss: 3.0173 | LR: 3.33e-04 | 1279.6ms/step | 15.6 samples/s +Step 17750/44000 | Loss: 3.0507 | LR: 3.32e-04 | 1280.1ms/step | 15.6 samples/s +Validation at step 17750: Loss = 3.0493, Perplexity = 21.10 +Step 17760/44000 | Loss: 3.0307 | LR: 3.32e-04 | 3428.9ms/step | 5.8 samples/s +Step 17770/44000 | Loss: 3.0496 | LR: 3.32e-04 | 1282.7ms/step | 15.6 samples/s +Step 17780/44000 | Loss: 3.0508 | LR: 3.32e-04 | 1280.3ms/step | 15.6 samples/s +Step 17790/44000 | Loss: 3.0298 | LR: 3.32e-04 | 1279.7ms/step | 15.6 samples/s +Step 17800/44000 | Loss: 3.0337 | LR: 3.32e-04 | 1280.4ms/step | 15.6 samples/s +Step 17810/44000 | Loss: 3.0342 | LR: 3.31e-04 | 1279.5ms/step | 15.6 samples/s +Step 17820/44000 | Loss: 3.0642 | LR: 3.31e-04 | 1281.2ms/step | 15.6 samples/s +Step 17830/44000 | Loss: 3.0254 | LR: 3.31e-04 | 1280.7ms/step | 15.6 samples/s +Step 17840/44000 | Loss: 3.0417 | LR: 3.31e-04 | 1280.8ms/step | 15.6 samples/s +Step 17850/44000 | Loss: 3.0298 | LR: 3.31e-04 | 1280.2ms/step | 15.6 samples/s +Step 17860/44000 | Loss: 3.0156 | LR: 3.31e-04 | 1282.9ms/step | 15.6 samples/s +Step 17870/44000 | Loss: 3.0312 | LR: 3.30e-04 | 1280.8ms/step | 15.6 samples/s +Step 17880/44000 | Loss: 3.0207 | LR: 3.30e-04 | 1282.1ms/step | 15.6 samples/s +Step 17890/44000 | Loss: 3.0035 | LR: 3.30e-04 | 1281.7ms/step | 15.6 samples/s +Step 17900/44000 | Loss: 3.0442 | LR: 3.30e-04 | 1281.0ms/step | 15.6 samples/s +Step 17910/44000 | Loss: 3.0514 | LR: 3.30e-04 | 1285.4ms/step | 15.6 samples/s +Step 17920/44000 | Loss: 3.0262 | LR: 3.29e-04 | 1285.6ms/step | 15.6 samples/s +Step 17930/44000 | Loss: 3.0290 | LR: 3.29e-04 | 1282.5ms/step | 15.6 samples/s +Step 17940/44000 | Loss: 3.0299 | LR: 3.29e-04 | 1279.4ms/step | 15.6 samples/s +Step 17950/44000 | Loss: 3.0271 | LR: 3.29e-04 | 1280.6ms/step | 15.6 samples/s +Step 17960/44000 | Loss: 3.0433 | LR: 3.29e-04 | 1281.5ms/step | 15.6 samples/s +Step 17970/44000 | Loss: 3.0469 | LR: 3.29e-04 | 1280.8ms/step | 15.6 samples/s +Step 17980/44000 | Loss: 3.0045 | LR: 3.28e-04 | 1285.0ms/step | 15.6 samples/s +Step 17990/44000 | Loss: 3.0390 | LR: 3.28e-04 | 1285.2ms/step | 15.6 samples/s +Step 18000/44000 | Loss: 3.0466 | LR: 3.28e-04 | 1284.4ms/step | 15.6 samples/s +Validation at step 18000: Loss = 3.0466, Perplexity = 21.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-18000 +Step 18010/44000 | Loss: 3.0251 | LR: 3.28e-04 | 3734.3ms/step | 5.4 samples/s +Step 18020/44000 | Loss: 3.0227 | LR: 3.28e-04 | 1284.7ms/step | 15.6 samples/s +Step 18030/44000 | Loss: 2.9997 | LR: 3.28e-04 | 1283.0ms/step | 15.6 samples/s +Step 18040/44000 | Loss: 3.0536 | LR: 3.27e-04 | 1282.6ms/step | 15.6 samples/s +Step 18050/44000 | Loss: 3.0196 | LR: 3.27e-04 | 1286.4ms/step | 15.5 samples/s +Step 18060/44000 | Loss: 3.0225 | LR: 3.27e-04 | 1285.8ms/step | 15.6 samples/s +Step 18070/44000 | Loss: 3.0081 | LR: 3.27e-04 | 1283.6ms/step | 15.6 samples/s +Step 18080/44000 | Loss: 3.0190 | LR: 3.27e-04 | 1281.4ms/step | 15.6 samples/s +Step 18090/44000 | Loss: 3.0325 | LR: 3.27e-04 | 1282.7ms/step | 15.6 samples/s +Step 18100/44000 | Loss: 3.0104 | LR: 3.26e-04 | 1281.9ms/step | 15.6 samples/s +Step 18110/44000 | Loss: 3.0477 | LR: 3.26e-04 | 1281.9ms/step | 15.6 samples/s +Step 18120/44000 | Loss: 3.0452 | LR: 3.26e-04 | 1280.7ms/step | 15.6 samples/s +Step 18130/44000 | Loss: 3.0330 | LR: 3.26e-04 | 1280.9ms/step | 15.6 samples/s +Step 18140/44000 | Loss: 3.0181 | LR: 3.26e-04 | 1280.0ms/step | 15.6 samples/s +Step 18150/44000 | Loss: 3.0060 | LR: 3.26e-04 | 1281.7ms/step | 15.6 samples/s +Step 18160/44000 | Loss: 3.0324 | LR: 3.25e-04 | 1280.0ms/step | 15.6 samples/s +Step 18170/44000 | Loss: 3.0332 | LR: 3.25e-04 | 1282.9ms/step | 15.6 samples/s +Step 18180/44000 | Loss: 3.0231 | LR: 3.25e-04 | 1283.7ms/step | 15.6 samples/s +Step 18190/44000 | Loss: 3.0479 | LR: 3.25e-04 | 1283.6ms/step | 15.6 samples/s +Step 18200/44000 | Loss: 3.0371 | LR: 3.25e-04 | 1281.2ms/step | 15.6 samples/s +Step 18210/44000 | Loss: 3.0300 | LR: 3.24e-04 | 1278.4ms/step | 15.6 samples/s +Step 18220/44000 | Loss: 3.0190 | LR: 3.24e-04 | 1282.0ms/step | 15.6 samples/s +Step 18230/44000 | Loss: 3.0226 | LR: 3.24e-04 | 1280.9ms/step | 15.6 samples/s +Step 18240/44000 | Loss: 3.0223 | LR: 3.24e-04 | 1282.3ms/step | 15.6 samples/s +Step 18250/44000 | Loss: 3.0171 | LR: 3.24e-04 | 1283.7ms/step | 15.6 samples/s +Validation at step 18250: Loss = 3.0434, Perplexity = 20.98 +Step 18260/44000 | Loss: 3.0315 | LR: 3.24e-04 | 3433.9ms/step | 5.8 samples/s +Step 18270/44000 | Loss: 3.0342 | LR: 3.23e-04 | 1284.0ms/step | 15.6 samples/s +Step 18280/44000 | Loss: 3.0432 | LR: 3.23e-04 | 1283.7ms/step | 15.6 samples/s +Step 18290/44000 | Loss: 3.0130 | LR: 3.23e-04 | 1284.7ms/step | 15.6 samples/s +Step 18300/44000 | Loss: 3.0224 | LR: 3.23e-04 | 1285.7ms/step | 15.6 samples/s +Step 18310/44000 | Loss: 3.0582 | LR: 3.23e-04 | 1283.0ms/step | 15.6 samples/s +Step 18320/44000 | Loss: 3.0095 | LR: 3.23e-04 | 1280.3ms/step | 15.6 samples/s +Step 18330/44000 | Loss: 3.0627 | LR: 3.22e-04 | 1280.3ms/step | 15.6 samples/s +Step 18340/44000 | Loss: 3.0403 | LR: 3.22e-04 | 1279.7ms/step | 15.6 samples/s +Step 18350/44000 | Loss: 3.0394 | LR: 3.22e-04 | 1279.4ms/step | 15.6 samples/s +Step 18360/44000 | Loss: 3.0590 | LR: 3.22e-04 | 1278.7ms/step | 15.6 samples/s +Step 18370/44000 | Loss: 3.0221 | LR: 3.22e-04 | 1282.9ms/step | 15.6 samples/s +Step 18380/44000 | Loss: 3.0166 | LR: 3.22e-04 | 1282.6ms/step | 15.6 samples/s +Step 18390/44000 | Loss: 3.0507 | LR: 3.21e-04 | 1282.9ms/step | 15.6 samples/s +Step 18400/44000 | Loss: 3.0448 | LR: 3.21e-04 | 1282.2ms/step | 15.6 samples/s +Step 18410/44000 | Loss: 3.0303 | LR: 3.21e-04 | 1278.4ms/step | 15.6 samples/s +Step 18420/44000 | Loss: 3.0038 | LR: 3.21e-04 | 1283.5ms/step | 15.6 samples/s +Step 18430/44000 | Loss: 3.0423 | LR: 3.21e-04 | 1283.7ms/step | 15.6 samples/s +Step 18440/44000 | Loss: 3.0201 | LR: 3.20e-04 | 1282.9ms/step | 15.6 samples/s +Step 18450/44000 | Loss: 3.0409 | LR: 3.20e-04 | 1282.4ms/step | 15.6 samples/s +Step 18460/44000 | Loss: 3.0561 | LR: 3.20e-04 | 1283.4ms/step | 15.6 samples/s +Step 18470/44000 | Loss: 3.0013 | LR: 3.20e-04 | 1279.3ms/step | 15.6 samples/s +Step 18480/44000 | Loss: 3.0354 | LR: 3.20e-04 | 1284.4ms/step | 15.6 samples/s +Step 18490/44000 | Loss: 3.0369 | LR: 3.20e-04 | 1282.0ms/step | 15.6 samples/s +Step 18500/44000 | Loss: 3.0321 | LR: 3.19e-04 | 1281.3ms/step | 15.6 samples/s +Validation at step 18500: Loss = 3.0421, Perplexity = 20.95 +Step 18510/44000 | Loss: 3.0072 | LR: 3.19e-04 | 3438.8ms/step | 5.8 samples/s +Step 18520/44000 | Loss: 3.0084 | LR: 3.19e-04 | 1283.4ms/step | 15.6 samples/s +Step 18530/44000 | Loss: 3.0034 | LR: 3.19e-04 | 1283.6ms/step | 15.6 samples/s +Step 18540/44000 | Loss: 3.0550 | LR: 3.19e-04 | 1283.5ms/step | 15.6 samples/s +Step 18550/44000 | Loss: 3.0270 | LR: 3.19e-04 | 1283.1ms/step | 15.6 samples/s +Step 18560/44000 | Loss: 2.9937 | LR: 3.18e-04 | 1279.1ms/step | 15.6 samples/s +Step 18570/44000 | Loss: 3.0300 | LR: 3.18e-04 | 1279.7ms/step | 15.6 samples/s +Step 18580/44000 | Loss: 2.9985 | LR: 3.18e-04 | 1278.2ms/step | 15.6 samples/s +Step 18590/44000 | Loss: 3.0175 | LR: 3.18e-04 | 1282.7ms/step | 15.6 samples/s +Step 18600/44000 | Loss: 3.0406 | LR: 3.18e-04 | 1283.2ms/step | 15.6 samples/s +Step 18610/44000 | Loss: 3.0520 | LR: 3.18e-04 | 1278.6ms/step | 15.6 samples/s +Step 18620/44000 | Loss: 3.0121 | LR: 3.17e-04 | 1280.0ms/step | 15.6 samples/s +Step 18630/44000 | Loss: 3.0159 | LR: 3.17e-04 | 1277.0ms/step | 15.7 samples/s +Step 18640/44000 | Loss: 3.0176 | LR: 3.17e-04 | 1283.1ms/step | 15.6 samples/s +Step 18650/44000 | Loss: 3.0047 | LR: 3.17e-04 | 1278.3ms/step | 15.6 samples/s +Step 18660/44000 | Loss: 2.9799 | LR: 3.17e-04 | 1277.6ms/step | 15.7 samples/s +Step 18670/44000 | Loss: 2.9999 | LR: 3.16e-04 | 1278.7ms/step | 15.6 samples/s +Step 18680/44000 | Loss: 3.0162 | LR: 3.16e-04 | 1278.5ms/step | 15.6 samples/s +Step 18690/44000 | Loss: 3.0216 | LR: 3.16e-04 | 1277.3ms/step | 15.7 samples/s +Step 18700/44000 | Loss: 2.9942 | LR: 3.16e-04 | 1275.3ms/step | 15.7 samples/s +Step 18710/44000 | Loss: 3.0287 | LR: 3.16e-04 | 1275.5ms/step | 15.7 samples/s +Step 18720/44000 | Loss: 3.0367 | LR: 3.16e-04 | 1285.4ms/step | 15.6 samples/s +Step 18730/44000 | Loss: 3.0381 | LR: 3.15e-04 | 1286.6ms/step | 15.5 samples/s +Step 18740/44000 | Loss: 3.0396 | LR: 3.15e-04 | 1284.0ms/step | 15.6 samples/s +Step 18750/44000 | Loss: 3.0440 | LR: 3.15e-04 | 1285.3ms/step | 15.6 samples/s +Validation at step 18750: Loss = 3.0371, Perplexity = 20.85 +Step 18760/44000 | Loss: 3.0237 | LR: 3.15e-04 | 3435.4ms/step | 5.8 samples/s +Step 18770/44000 | Loss: 3.0231 | LR: 3.15e-04 | 1280.3ms/step | 15.6 samples/s +Step 18780/44000 | Loss: 3.0164 | LR: 3.15e-04 | 1283.9ms/step | 15.6 samples/s +Step 18790/44000 | Loss: 3.0136 | LR: 3.14e-04 | 1286.0ms/step | 15.6 samples/s +Step 18800/44000 | Loss: 3.0357 | LR: 3.14e-04 | 1285.4ms/step | 15.6 samples/s +Step 18810/44000 | Loss: 3.0285 | LR: 3.14e-04 | 1281.4ms/step | 15.6 samples/s +Step 18820/44000 | Loss: 3.0196 | LR: 3.14e-04 | 1281.6ms/step | 15.6 samples/s +Step 18830/44000 | Loss: 3.0084 | LR: 3.14e-04 | 1283.3ms/step | 15.6 samples/s +Step 18840/44000 | Loss: 3.0347 | LR: 3.13e-04 | 1282.6ms/step | 15.6 samples/s +Step 18850/44000 | Loss: 2.9932 | LR: 3.13e-04 | 1283.5ms/step | 15.6 samples/s +Step 18860/44000 | Loss: 3.0225 | LR: 3.13e-04 | 1284.4ms/step | 15.6 samples/s +Step 18870/44000 | Loss: 3.0378 | LR: 3.13e-04 | 1281.1ms/step | 15.6 samples/s +Step 18880/44000 | Loss: 3.0059 | LR: 3.13e-04 | 1285.9ms/step | 15.6 samples/s +Step 18890/44000 | Loss: 3.0245 | LR: 3.13e-04 | 1283.6ms/step | 15.6 samples/s +Step 18900/44000 | Loss: 2.9847 | LR: 3.12e-04 | 1283.4ms/step | 15.6 samples/s +Step 18910/44000 | Loss: 3.0247 | LR: 3.12e-04 | 1283.6ms/step | 15.6 samples/s +Step 18920/44000 | Loss: 3.0279 | LR: 3.12e-04 | 1285.4ms/step | 15.6 samples/s +Step 18930/44000 | Loss: 3.0244 | LR: 3.12e-04 | 1286.3ms/step | 15.5 samples/s +Step 18940/44000 | Loss: 3.0335 | LR: 3.12e-04 | 1281.5ms/step | 15.6 samples/s +Step 18950/44000 | Loss: 2.9931 | LR: 3.12e-04 | 1283.3ms/step | 15.6 samples/s +Step 18960/44000 | Loss: 3.0092 | LR: 3.11e-04 | 1283.1ms/step | 15.6 samples/s +Step 18970/44000 | Loss: 3.0306 | LR: 3.11e-04 | 1284.2ms/step | 15.6 samples/s +Step 18980/44000 | Loss: 3.0024 | LR: 3.11e-04 | 1286.5ms/step | 15.5 samples/s +Step 18990/44000 | Loss: 3.0222 | LR: 3.11e-04 | 1286.1ms/step | 15.6 samples/s +Step 19000/44000 | Loss: 3.0188 | LR: 3.11e-04 | 1287.8ms/step | 15.5 samples/s +Validation at step 19000: Loss = 3.0341, Perplexity = 20.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-19000 +Step 19010/44000 | Loss: 2.9898 | LR: 3.11e-04 | 3705.3ms/step | 5.4 samples/s +Step 19020/44000 | Loss: 3.0283 | LR: 3.10e-04 | 1277.8ms/step | 15.7 samples/s +Step 19030/44000 | Loss: 3.0071 | LR: 3.10e-04 | 1281.9ms/step | 15.6 samples/s +Step 19040/44000 | Loss: 3.0187 | LR: 3.10e-04 | 1281.8ms/step | 15.6 samples/s +Step 19050/44000 | Loss: 3.0356 | LR: 3.10e-04 | 1282.4ms/step | 15.6 samples/s +Step 19060/44000 | Loss: 3.0521 | LR: 3.10e-04 | 1279.0ms/step | 15.6 samples/s +Step 19070/44000 | Loss: 3.0249 | LR: 3.09e-04 | 1278.2ms/step | 15.6 samples/s +Step 19080/44000 | Loss: 2.9946 | LR: 3.09e-04 | 1278.8ms/step | 15.6 samples/s +Step 19090/44000 | Loss: 3.0136 | LR: 3.09e-04 | 1280.4ms/step | 15.6 samples/s +Step 19100/44000 | Loss: 2.9972 | LR: 3.09e-04 | 1281.3ms/step | 15.6 samples/s +Step 19110/44000 | Loss: 3.0262 | LR: 3.09e-04 | 1281.9ms/step | 15.6 samples/s +Step 19120/44000 | Loss: 3.0217 | LR: 3.09e-04 | 1281.0ms/step | 15.6 samples/s +Step 19130/44000 | Loss: 3.0346 | LR: 3.08e-04 | 1284.2ms/step | 15.6 samples/s +Step 19140/44000 | Loss: 3.0011 | LR: 3.08e-04 | 1284.8ms/step | 15.6 samples/s +Step 19150/44000 | Loss: 3.0124 | LR: 3.08e-04 | 1282.0ms/step | 15.6 samples/s +Step 19160/44000 | Loss: 3.0140 | LR: 3.08e-04 | 1282.8ms/step | 15.6 samples/s +Step 19170/44000 | Loss: 2.9784 | LR: 3.08e-04 | 1280.3ms/step | 15.6 samples/s +Step 19180/44000 | Loss: 3.0382 | LR: 3.08e-04 | 1283.4ms/step | 15.6 samples/s +Step 19190/44000 | Loss: 2.9923 | LR: 3.07e-04 | 1277.9ms/step | 15.7 samples/s +Step 19200/44000 | Loss: 2.9972 | LR: 3.07e-04 | 1282.2ms/step | 15.6 samples/s +Step 19210/44000 | Loss: 3.0228 | LR: 3.07e-04 | 1281.0ms/step | 15.6 samples/s +Step 19220/44000 | Loss: 3.0086 | LR: 3.07e-04 | 1281.8ms/step | 15.6 samples/s +Step 19230/44000 | Loss: 3.0738 | LR: 3.07e-04 | 1281.4ms/step | 15.6 samples/s +Step 19240/44000 | Loss: 3.0225 | LR: 3.06e-04 | 1281.4ms/step | 15.6 samples/s +Step 19250/44000 | Loss: 3.0090 | LR: 3.06e-04 | 1283.6ms/step | 15.6 samples/s +Validation at step 19250: Loss = 3.0317, Perplexity = 20.73 +Step 19260/44000 | Loss: 3.0194 | LR: 3.06e-04 | 3428.7ms/step | 5.8 samples/s +Step 19270/44000 | Loss: 3.0050 | LR: 3.06e-04 | 1278.9ms/step | 15.6 samples/s +Step 19280/44000 | Loss: 2.9931 | LR: 3.06e-04 | 1276.8ms/step | 15.7 samples/s +Step 19290/44000 | Loss: 3.0131 | LR: 3.06e-04 | 1281.2ms/step | 15.6 samples/s +Step 19300/44000 | Loss: 3.0030 | LR: 3.05e-04 | 1279.9ms/step | 15.6 samples/s +Step 19310/44000 | Loss: 3.0298 | LR: 3.05e-04 | 1279.4ms/step | 15.6 samples/s +Step 19320/44000 | Loss: 3.0290 | LR: 3.05e-04 | 1279.7ms/step | 15.6 samples/s +Step 19330/44000 | Loss: 3.0375 | LR: 3.05e-04 | 1280.9ms/step | 15.6 samples/s +Step 19340/44000 | Loss: 3.0344 | LR: 3.05e-04 | 1282.9ms/step | 15.6 samples/s +Step 19350/44000 | Loss: 3.0192 | LR: 3.04e-04 | 1281.7ms/step | 15.6 samples/s +Step 19360/44000 | Loss: 3.0314 | LR: 3.04e-04 | 1279.1ms/step | 15.6 samples/s +Step 19370/44000 | Loss: 3.0261 | LR: 3.04e-04 | 1284.7ms/step | 15.6 samples/s +Step 19380/44000 | Loss: 2.9900 | LR: 3.04e-04 | 1281.8ms/step | 15.6 samples/s +Step 19390/44000 | Loss: 2.9964 | LR: 3.04e-04 | 1285.8ms/step | 15.6 samples/s +Step 19400/44000 | Loss: 3.0185 | LR: 3.04e-04 | 1286.3ms/step | 15.5 samples/s +Step 19410/44000 | Loss: 3.0082 | LR: 3.03e-04 | 1279.2ms/step | 15.6 samples/s +Step 19420/44000 | Loss: 3.0254 | LR: 3.03e-04 | 1283.1ms/step | 15.6 samples/s +Step 19430/44000 | Loss: 2.9854 | LR: 3.03e-04 | 1280.5ms/step | 15.6 samples/s +Step 19440/44000 | Loss: 3.0371 | LR: 3.03e-04 | 1279.5ms/step | 15.6 samples/s +Step 19450/44000 | Loss: 3.0079 | LR: 3.03e-04 | 1280.7ms/step | 15.6 samples/s +Step 19460/44000 | Loss: 3.0240 | LR: 3.03e-04 | 1279.2ms/step | 15.6 samples/s +Step 19470/44000 | Loss: 3.0308 | LR: 3.02e-04 | 1282.5ms/step | 15.6 samples/s +Step 19480/44000 | Loss: 2.9888 | LR: 3.02e-04 | 1280.4ms/step | 15.6 samples/s +Step 19490/44000 | Loss: 3.0391 | LR: 3.02e-04 | 1277.9ms/step | 15.7 samples/s +Step 19500/44000 | Loss: 3.0010 | LR: 3.02e-04 | 1280.5ms/step | 15.6 samples/s +Validation at step 19500: Loss = 3.0281, Perplexity = 20.66 +Step 19510/44000 | Loss: 3.0241 | LR: 3.02e-04 | 3430.0ms/step | 5.8 samples/s +Step 19520/44000 | Loss: 3.0035 | LR: 3.01e-04 | 1276.3ms/step | 15.7 samples/s +Step 19530/44000 | Loss: 2.9830 | LR: 3.01e-04 | 1279.6ms/step | 15.6 samples/s +Step 19540/44000 | Loss: 3.0127 | LR: 3.01e-04 | 1282.6ms/step | 15.6 samples/s +Step 19550/44000 | Loss: 3.0017 | LR: 3.01e-04 | 1283.1ms/step | 15.6 samples/s +Step 19560/44000 | Loss: 3.0075 | LR: 3.01e-04 | 1282.0ms/step | 15.6 samples/s +Step 19570/44000 | Loss: 3.0225 | LR: 3.01e-04 | 1283.7ms/step | 15.6 samples/s +Step 19580/44000 | Loss: 3.0316 | LR: 3.00e-04 | 1283.9ms/step | 15.6 samples/s +Step 19590/44000 | Loss: 3.0176 | LR: 3.00e-04 | 1277.7ms/step | 15.7 samples/s +Step 19600/44000 | Loss: 2.9933 | LR: 3.00e-04 | 1277.7ms/step | 15.7 samples/s +Step 19610/44000 | Loss: 3.0125 | LR: 3.00e-04 | 1280.8ms/step | 15.6 samples/s +Step 19620/44000 | Loss: 3.0092 | LR: 3.00e-04 | 1283.2ms/step | 15.6 samples/s +Step 19630/44000 | Loss: 3.0349 | LR: 3.00e-04 | 1285.8ms/step | 15.6 samples/s +Step 19640/44000 | Loss: 3.0298 | LR: 2.99e-04 | 1283.4ms/step | 15.6 samples/s +Step 19650/44000 | Loss: 2.9983 | LR: 2.99e-04 | 1283.5ms/step | 15.6 samples/s +Step 19660/44000 | Loss: 3.0128 | LR: 2.99e-04 | 1281.9ms/step | 15.6 samples/s +Step 19670/44000 | Loss: 3.0251 | LR: 2.99e-04 | 1283.6ms/step | 15.6 samples/s +Step 19680/44000 | Loss: 3.0389 | LR: 2.99e-04 | 1284.3ms/step | 15.6 samples/s +Step 19690/44000 | Loss: 2.9977 | LR: 2.98e-04 | 1280.7ms/step | 15.6 samples/s +Step 19700/44000 | Loss: 3.0303 | LR: 2.98e-04 | 1276.9ms/step | 15.7 samples/s +Step 19710/44000 | Loss: 3.0239 | LR: 2.98e-04 | 1280.9ms/step | 15.6 samples/s +Step 19720/44000 | Loss: 3.0134 | LR: 2.98e-04 | 1283.4ms/step | 15.6 samples/s +Step 19730/44000 | Loss: 3.0393 | LR: 2.98e-04 | 1283.5ms/step | 15.6 samples/s +Step 19740/44000 | Loss: 3.0120 | LR: 2.98e-04 | 1285.3ms/step | 15.6 samples/s +Step 19750/44000 | Loss: 2.9926 | LR: 2.97e-04 | 1283.4ms/step | 15.6 samples/s +Validation at step 19750: Loss = 3.0259, Perplexity = 20.61 +Step 19760/44000 | Loss: 3.0270 | LR: 2.97e-04 | 3433.5ms/step | 5.8 samples/s +Step 19770/44000 | Loss: 3.0347 | LR: 2.97e-04 | 1283.3ms/step | 15.6 samples/s +Step 19780/44000 | Loss: 3.0284 | LR: 2.97e-04 | 1281.1ms/step | 15.6 samples/s +Step 19790/44000 | Loss: 3.0083 | LR: 2.97e-04 | 1287.6ms/step | 15.5 samples/s +Step 19800/44000 | Loss: 3.0360 | LR: 2.96e-04 | 1285.1ms/step | 15.6 samples/s +Step 19810/44000 | Loss: 2.9963 | LR: 2.96e-04 | 1285.8ms/step | 15.6 samples/s +Step 19820/44000 | Loss: 2.9865 | LR: 2.96e-04 | 1287.8ms/step | 15.5 samples/s +Step 19830/44000 | Loss: 3.0342 | LR: 2.96e-04 | 1284.5ms/step | 15.6 samples/s +Step 19840/44000 | Loss: 3.0104 | LR: 2.96e-04 | 1286.2ms/step | 15.6 samples/s +Step 19850/44000 | Loss: 3.0077 | LR: 2.96e-04 | 1289.8ms/step | 15.5 samples/s +Step 19860/44000 | Loss: 3.0371 | LR: 2.95e-04 | 1285.0ms/step | 15.6 samples/s +Step 19870/44000 | Loss: 3.0321 | LR: 2.95e-04 | 1290.0ms/step | 15.5 samples/s +Step 19880/44000 | Loss: 3.0181 | LR: 2.95e-04 | 1283.2ms/step | 15.6 samples/s +Step 19890/44000 | Loss: 3.0191 | LR: 2.95e-04 | 1282.2ms/step | 15.6 samples/s +Step 19900/44000 | Loss: 3.0051 | LR: 2.95e-04 | 1282.2ms/step | 15.6 samples/s +Step 19910/44000 | Loss: 3.0149 | LR: 2.95e-04 | 1285.6ms/step | 15.6 samples/s +Step 19920/44000 | Loss: 3.0090 | LR: 2.94e-04 | 1286.4ms/step | 15.5 samples/s +Step 19930/44000 | Loss: 3.0197 | LR: 2.94e-04 | 1280.4ms/step | 15.6 samples/s +Step 19940/44000 | Loss: 2.9837 | LR: 2.94e-04 | 1282.6ms/step | 15.6 samples/s +Step 19950/44000 | Loss: 3.0035 | LR: 2.94e-04 | 1280.7ms/step | 15.6 samples/s +Step 19960/44000 | Loss: 3.0438 | LR: 2.94e-04 | 1283.2ms/step | 15.6 samples/s +Step 19970/44000 | Loss: 3.0309 | LR: 2.93e-04 | 1283.2ms/step | 15.6 samples/s +Step 19980/44000 | Loss: 3.0293 | LR: 2.93e-04 | 1283.5ms/step | 15.6 samples/s +Step 19990/44000 | Loss: 3.0176 | LR: 2.93e-04 | 1282.0ms/step | 15.6 samples/s +Step 20000/44000 | Loss: 3.0212 | LR: 2.93e-04 | 1282.7ms/step | 15.6 samples/s +Validation at step 20000: Loss = 3.0231, Perplexity = 20.55 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-20000 +Step 20010/44000 | Loss: 2.9981 | LR: 2.93e-04 | 3690.6ms/step | 5.4 samples/s +Step 20020/44000 | Loss: 3.0223 | LR: 2.93e-04 | 1273.8ms/step | 15.7 samples/s +Step 20030/44000 | Loss: 2.9964 | LR: 2.92e-04 | 1278.1ms/step | 15.6 samples/s +Step 20040/44000 | Loss: 2.9989 | LR: 2.92e-04 | 1278.7ms/step | 15.6 samples/s +Step 20050/44000 | Loss: 3.0154 | LR: 2.92e-04 | 1281.9ms/step | 15.6 samples/s +Step 20060/44000 | Loss: 2.9829 | LR: 2.92e-04 | 1284.2ms/step | 15.6 samples/s +Step 20070/44000 | Loss: 3.0093 | LR: 2.92e-04 | 1283.3ms/step | 15.6 samples/s +Step 20080/44000 | Loss: 2.9925 | LR: 2.91e-04 | 1284.8ms/step | 15.6 samples/s +Step 20090/44000 | Loss: 3.0027 | LR: 2.91e-04 | 1282.1ms/step | 15.6 samples/s +Step 20100/44000 | Loss: 3.0194 | LR: 2.91e-04 | 1280.4ms/step | 15.6 samples/s +Step 20110/44000 | Loss: 3.0101 | LR: 2.91e-04 | 1282.5ms/step | 15.6 samples/s +Step 20120/44000 | Loss: 3.0194 | LR: 2.91e-04 | 1281.8ms/step | 15.6 samples/s +Step 20130/44000 | Loss: 3.0158 | LR: 2.91e-04 | 1281.6ms/step | 15.6 samples/s +Step 20140/44000 | Loss: 3.0165 | LR: 2.90e-04 | 1282.2ms/step | 15.6 samples/s +Step 20150/44000 | Loss: 3.0053 | LR: 2.90e-04 | 1284.0ms/step | 15.6 samples/s +Step 20160/44000 | Loss: 3.0198 | LR: 2.90e-04 | 1283.1ms/step | 15.6 samples/s +Step 20170/44000 | Loss: 2.9756 | LR: 2.90e-04 | 1282.4ms/step | 15.6 samples/s +Step 20180/44000 | Loss: 2.9963 | LR: 2.90e-04 | 1282.1ms/step | 15.6 samples/s +Step 20190/44000 | Loss: 3.0267 | LR: 2.90e-04 | 1279.8ms/step | 15.6 samples/s +Step 20200/44000 | Loss: 2.9856 | LR: 2.89e-04 | 1280.8ms/step | 15.6 samples/s +Step 20210/44000 | Loss: 2.9752 | LR: 2.89e-04 | 1281.8ms/step | 15.6 samples/s +Step 20220/44000 | Loss: 3.0268 | LR: 2.89e-04 | 1282.8ms/step | 15.6 samples/s +Step 20230/44000 | Loss: 3.0135 | LR: 2.89e-04 | 1284.9ms/step | 15.6 samples/s +Step 20240/44000 | Loss: 3.0160 | LR: 2.89e-04 | 1285.1ms/step | 15.6 samples/s +Step 20250/44000 | Loss: 3.0003 | LR: 2.88e-04 | 1282.7ms/step | 15.6 samples/s +Validation at step 20250: Loss = 3.0207, Perplexity = 20.51 +Step 20260/44000 | Loss: 2.9676 | LR: 2.88e-04 | 3430.5ms/step | 5.8 samples/s +Step 20270/44000 | Loss: 3.0183 | LR: 2.88e-04 | 1283.5ms/step | 15.6 samples/s +Step 20280/44000 | Loss: 3.0140 | LR: 2.88e-04 | 1284.3ms/step | 15.6 samples/s +Step 20290/44000 | Loss: 3.0071 | LR: 2.88e-04 | 1282.1ms/step | 15.6 samples/s +Step 20300/44000 | Loss: 3.0302 | LR: 2.88e-04 | 1281.6ms/step | 15.6 samples/s +Step 20310/44000 | Loss: 2.9978 | LR: 2.87e-04 | 1282.4ms/step | 15.6 samples/s +Step 20320/44000 | Loss: 3.0054 | LR: 2.87e-04 | 1281.5ms/step | 15.6 samples/s +Step 20330/44000 | Loss: 2.9951 | LR: 2.87e-04 | 1281.3ms/step | 15.6 samples/s +Step 20340/44000 | Loss: 2.9851 | LR: 2.87e-04 | 1280.8ms/step | 15.6 samples/s +Step 20350/44000 | Loss: 3.0114 | LR: 2.87e-04 | 1284.3ms/step | 15.6 samples/s +Step 20360/44000 | Loss: 3.0245 | LR: 2.86e-04 | 1283.0ms/step | 15.6 samples/s +Step 20370/44000 | Loss: 2.9802 | LR: 2.86e-04 | 1283.4ms/step | 15.6 samples/s +Step 20380/44000 | Loss: 2.9903 | LR: 2.86e-04 | 1283.3ms/step | 15.6 samples/s +Step 20390/44000 | Loss: 3.0057 | LR: 2.86e-04 | 1284.1ms/step | 15.6 samples/s +Step 20400/44000 | Loss: 3.0226 | LR: 2.86e-04 | 1284.1ms/step | 15.6 samples/s +Step 20410/44000 | Loss: 2.9969 | LR: 2.86e-04 | 1283.4ms/step | 15.6 samples/s +Step 20420/44000 | Loss: 3.0206 | LR: 2.85e-04 | 1284.1ms/step | 15.6 samples/s +Step 20430/44000 | Loss: 2.9847 | LR: 2.85e-04 | 1283.6ms/step | 15.6 samples/s +Step 20440/44000 | Loss: 3.0141 | LR: 2.85e-04 | 1282.4ms/step | 15.6 samples/s +Step 20450/44000 | Loss: 3.0038 | LR: 2.85e-04 | 1284.7ms/step | 15.6 samples/s +Step 20460/44000 | Loss: 3.0078 | LR: 2.85e-04 | 1281.5ms/step | 15.6 samples/s +Step 20470/44000 | Loss: 3.0261 | LR: 2.84e-04 | 1280.5ms/step | 15.6 samples/s +Step 20480/44000 | Loss: 3.0025 | LR: 2.84e-04 | 1281.7ms/step | 15.6 samples/s +Step 20490/44000 | Loss: 3.0032 | LR: 2.84e-04 | 1281.5ms/step | 15.6 samples/s +Step 20500/44000 | Loss: 2.9839 | LR: 2.84e-04 | 1286.1ms/step | 15.6 samples/s +Validation at step 20500: Loss = 3.0177, Perplexity = 20.44 +Step 20510/44000 | Loss: 3.0291 | LR: 2.84e-04 | 3428.2ms/step | 5.8 samples/s +Step 20520/44000 | Loss: 2.9849 | LR: 2.84e-04 | 1276.9ms/step | 15.7 samples/s +Step 20530/44000 | Loss: 2.9755 | LR: 2.83e-04 | 1280.0ms/step | 15.6 samples/s +Step 20540/44000 | Loss: 3.0049 | LR: 2.83e-04 | 1283.1ms/step | 15.6 samples/s +Step 20550/44000 | Loss: 3.0168 | LR: 2.83e-04 | 1284.4ms/step | 15.6 samples/s +Step 20560/44000 | Loss: 3.0196 | LR: 2.83e-04 | 1282.4ms/step | 15.6 samples/s +Step 20570/44000 | Loss: 3.0153 | LR: 2.83e-04 | 1282.2ms/step | 15.6 samples/s +Step 20580/44000 | Loss: 2.9869 | LR: 2.83e-04 | 1282.3ms/step | 15.6 samples/s +Step 20590/44000 | Loss: 3.0256 | LR: 2.82e-04 | 1278.7ms/step | 15.6 samples/s +Step 20600/44000 | Loss: 3.0030 | LR: 2.82e-04 | 1280.4ms/step | 15.6 samples/s +Step 20610/44000 | Loss: 3.0097 | LR: 2.82e-04 | 1280.7ms/step | 15.6 samples/s +Step 20620/44000 | Loss: 3.0036 | LR: 2.82e-04 | 1283.4ms/step | 15.6 samples/s +Step 20630/44000 | Loss: 3.0119 | LR: 2.82e-04 | 1281.8ms/step | 15.6 samples/s +Step 20640/44000 | Loss: 2.9731 | LR: 2.81e-04 | 1282.5ms/step | 15.6 samples/s +Step 20650/44000 | Loss: 3.0195 | LR: 2.81e-04 | 1283.9ms/step | 15.6 samples/s +Step 20660/44000 | Loss: 2.9714 | LR: 2.81e-04 | 1284.2ms/step | 15.6 samples/s +Step 20670/44000 | Loss: 3.0170 | LR: 2.81e-04 | 1284.5ms/step | 15.6 samples/s +Step 20680/44000 | Loss: 3.0102 | LR: 2.81e-04 | 1284.4ms/step | 15.6 samples/s +Step 20690/44000 | Loss: 2.9972 | LR: 2.81e-04 | 1279.5ms/step | 15.6 samples/s +Step 20700/44000 | Loss: 3.0098 | LR: 2.80e-04 | 1280.1ms/step | 15.6 samples/s +Step 20710/44000 | Loss: 3.0149 | LR: 2.80e-04 | 1282.6ms/step | 15.6 samples/s +Step 20720/44000 | Loss: 3.0134 | LR: 2.80e-04 | 1285.0ms/step | 15.6 samples/s +Step 20730/44000 | Loss: 3.0289 | LR: 2.80e-04 | 1286.8ms/step | 15.5 samples/s +Step 20740/44000 | Loss: 3.0125 | LR: 2.80e-04 | 1285.5ms/step | 15.6 samples/s +Step 20750/44000 | Loss: 3.0084 | LR: 2.79e-04 | 1286.4ms/step | 15.5 samples/s +Validation at step 20750: Loss = 3.0149, Perplexity = 20.39 +Step 20760/44000 | Loss: 3.0149 | LR: 2.79e-04 | 3465.9ms/step | 5.8 samples/s +Step 20770/44000 | Loss: 2.9971 | LR: 2.79e-04 | 1289.6ms/step | 15.5 samples/s +Step 20780/44000 | Loss: 2.9739 | LR: 2.79e-04 | 1285.8ms/step | 15.6 samples/s +Step 20790/44000 | Loss: 2.9920 | LR: 2.79e-04 | 1283.2ms/step | 15.6 samples/s +Step 20800/44000 | Loss: 2.9949 | LR: 2.79e-04 | 1282.8ms/step | 15.6 samples/s +Step 20810/44000 | Loss: 3.0110 | LR: 2.78e-04 | 1281.9ms/step | 15.6 samples/s +Step 20820/44000 | Loss: 3.0072 | LR: 2.78e-04 | 1280.8ms/step | 15.6 samples/s +Step 20830/44000 | Loss: 3.0125 | LR: 2.78e-04 | 1278.7ms/step | 15.6 samples/s +Step 20840/44000 | Loss: 2.9979 | LR: 2.78e-04 | 1278.6ms/step | 15.6 samples/s +Step 20850/44000 | Loss: 2.9994 | LR: 2.78e-04 | 1282.5ms/step | 15.6 samples/s +Step 20860/44000 | Loss: 3.0188 | LR: 2.77e-04 | 1278.1ms/step | 15.6 samples/s +Step 20870/44000 | Loss: 3.0228 | LR: 2.77e-04 | 1277.9ms/step | 15.7 samples/s +Step 20880/44000 | Loss: 2.9950 | LR: 2.77e-04 | 1279.0ms/step | 15.6 samples/s +Step 20890/44000 | Loss: 2.9802 | LR: 2.77e-04 | 1280.7ms/step | 15.6 samples/s +Step 20900/44000 | Loss: 2.9818 | LR: 2.77e-04 | 1283.8ms/step | 15.6 samples/s +Step 20910/44000 | Loss: 3.0144 | LR: 2.77e-04 | 1278.4ms/step | 15.6 samples/s +Step 20920/44000 | Loss: 2.9773 | LR: 2.76e-04 | 1281.5ms/step | 15.6 samples/s +Step 20930/44000 | Loss: 2.9867 | LR: 2.76e-04 | 1280.0ms/step | 15.6 samples/s +Step 20940/44000 | Loss: 3.0108 | LR: 2.76e-04 | 1281.6ms/step | 15.6 samples/s +Step 20950/44000 | Loss: 2.9738 | LR: 2.76e-04 | 1283.3ms/step | 15.6 samples/s +Step 20960/44000 | Loss: 2.9917 | LR: 2.76e-04 | 1282.2ms/step | 15.6 samples/s +Step 20970/44000 | Loss: 2.9731 | LR: 2.75e-04 | 1282.5ms/step | 15.6 samples/s +Step 20980/44000 | Loss: 2.9778 | LR: 2.75e-04 | 1283.2ms/step | 15.6 samples/s +Step 20990/44000 | Loss: 2.9861 | LR: 2.75e-04 | 1282.5ms/step | 15.6 samples/s +Step 21000/44000 | Loss: 2.9861 | LR: 2.75e-04 | 1283.9ms/step | 15.6 samples/s +Validation at step 21000: Loss = 3.0139, Perplexity = 20.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-21000 +Step 21010/44000 | Loss: 2.9935 | LR: 2.75e-04 | 3699.3ms/step | 5.4 samples/s +Step 21020/44000 | Loss: 2.9933 | LR: 2.75e-04 | 1282.5ms/step | 15.6 samples/s +Step 21030/44000 | Loss: 3.0301 | LR: 2.74e-04 | 1279.6ms/step | 15.6 samples/s +Step 21040/44000 | Loss: 2.9763 | LR: 2.74e-04 | 1282.0ms/step | 15.6 samples/s +Step 21050/44000 | Loss: 2.9931 | LR: 2.74e-04 | 1282.5ms/step | 15.6 samples/s +Step 21060/44000 | Loss: 2.9782 | LR: 2.74e-04 | 1281.6ms/step | 15.6 samples/s +Step 21070/44000 | Loss: 3.0200 | LR: 2.74e-04 | 1282.2ms/step | 15.6 samples/s +Step 21080/44000 | Loss: 2.9653 | LR: 2.73e-04 | 1278.4ms/step | 15.6 samples/s +Step 21090/44000 | Loss: 3.0254 | LR: 2.73e-04 | 1279.8ms/step | 15.6 samples/s +Step 21100/44000 | Loss: 3.0092 | LR: 2.73e-04 | 1284.9ms/step | 15.6 samples/s +Step 21110/44000 | Loss: 2.9983 | LR: 2.73e-04 | 1282.1ms/step | 15.6 samples/s +Step 21120/44000 | Loss: 3.0088 | LR: 2.73e-04 | 1282.9ms/step | 15.6 samples/s +Step 21130/44000 | Loss: 3.0312 | LR: 2.73e-04 | 1283.7ms/step | 15.6 samples/s +Step 21140/44000 | Loss: 2.9932 | LR: 2.72e-04 | 1283.0ms/step | 15.6 samples/s +Step 21150/44000 | Loss: 3.0027 | LR: 2.72e-04 | 1284.3ms/step | 15.6 samples/s +Step 21160/44000 | Loss: 2.9828 | LR: 2.72e-04 | 1283.9ms/step | 15.6 samples/s +Step 21170/44000 | Loss: 2.9981 | LR: 2.72e-04 | 1291.7ms/step | 15.5 samples/s +Step 21180/44000 | Loss: 3.0061 | LR: 2.72e-04 | 1290.0ms/step | 15.5 samples/s +Step 21190/44000 | Loss: 2.9463 | LR: 2.71e-04 | 1284.3ms/step | 15.6 samples/s +Step 21200/44000 | Loss: 3.0172 | LR: 2.71e-04 | 1284.6ms/step | 15.6 samples/s +Step 21210/44000 | Loss: 2.9985 | LR: 2.71e-04 | 1286.1ms/step | 15.6 samples/s +Step 21220/44000 | Loss: 2.9886 | LR: 2.71e-04 | 1282.0ms/step | 15.6 samples/s +Step 21230/44000 | Loss: 3.0021 | LR: 2.71e-04 | 1277.7ms/step | 15.7 samples/s +Step 21240/44000 | Loss: 2.9927 | LR: 2.71e-04 | 1280.4ms/step | 15.6 samples/s +Step 21250/44000 | Loss: 2.9837 | LR: 2.70e-04 | 1282.1ms/step | 15.6 samples/s +Validation at step 21250: Loss = 3.0100, Perplexity = 20.29 +Step 21260/44000 | Loss: 3.0265 | LR: 2.70e-04 | 3435.6ms/step | 5.8 samples/s +Step 21270/44000 | Loss: 3.0048 | LR: 2.70e-04 | 1280.4ms/step | 15.6 samples/s +Step 21280/44000 | Loss: 2.9939 | LR: 2.70e-04 | 1278.7ms/step | 15.6 samples/s +Step 21290/44000 | Loss: 2.9888 | LR: 2.70e-04 | 1281.1ms/step | 15.6 samples/s +Step 21300/44000 | Loss: 2.9996 | LR: 2.70e-04 | 1277.6ms/step | 15.7 samples/s +Step 21310/44000 | Loss: 3.0003 | LR: 2.69e-04 | 1276.0ms/step | 15.7 samples/s +Step 21320/44000 | Loss: 3.0128 | LR: 2.69e-04 | 1277.9ms/step | 15.7 samples/s +Step 21330/44000 | Loss: 2.9967 | LR: 2.69e-04 | 1280.9ms/step | 15.6 samples/s +Step 21340/44000 | Loss: 2.9896 | LR: 2.69e-04 | 1280.7ms/step | 15.6 samples/s +Step 21350/44000 | Loss: 3.0241 | LR: 2.69e-04 | 1282.6ms/step | 15.6 samples/s +Step 21360/44000 | Loss: 2.9935 | LR: 2.68e-04 | 1283.3ms/step | 15.6 samples/s +Step 21370/44000 | Loss: 2.9973 | LR: 2.68e-04 | 1282.7ms/step | 15.6 samples/s +Step 21380/44000 | Loss: 2.9854 | LR: 2.68e-04 | 1284.5ms/step | 15.6 samples/s +Step 21390/44000 | Loss: 3.0168 | LR: 2.68e-04 | 1285.0ms/step | 15.6 samples/s +Step 21400/44000 | Loss: 2.9784 | LR: 2.68e-04 | 1284.3ms/step | 15.6 samples/s +Step 21410/44000 | Loss: 3.0194 | LR: 2.68e-04 | 1278.3ms/step | 15.6 samples/s +Step 21420/44000 | Loss: 2.9666 | LR: 2.67e-04 | 1277.5ms/step | 15.7 samples/s +Step 21430/44000 | Loss: 2.9823 | LR: 2.67e-04 | 1278.3ms/step | 15.6 samples/s +Step 21440/44000 | Loss: 2.9737 | LR: 2.67e-04 | 1279.8ms/step | 15.6 samples/s +Step 21450/44000 | Loss: 2.9918 | LR: 2.67e-04 | 1282.7ms/step | 15.6 samples/s +Step 21460/44000 | Loss: 2.9991 | LR: 2.67e-04 | 1284.2ms/step | 15.6 samples/s +Step 21470/44000 | Loss: 3.0034 | LR: 2.66e-04 | 1283.9ms/step | 15.6 samples/s +Step 21480/44000 | Loss: 2.9840 | LR: 2.66e-04 | 1284.4ms/step | 15.6 samples/s +Step 21490/44000 | Loss: 3.0007 | LR: 2.66e-04 | 1283.6ms/step | 15.6 samples/s +Step 21500/44000 | Loss: 2.9975 | LR: 2.66e-04 | 1281.3ms/step | 15.6 samples/s +Validation at step 21500: Loss = 3.0075, Perplexity = 20.24 +Step 21510/44000 | Loss: 3.0017 | LR: 2.66e-04 | 3432.6ms/step | 5.8 samples/s +Step 21520/44000 | Loss: 2.9904 | LR: 2.66e-04 | 1279.6ms/step | 15.6 samples/s +Step 21530/44000 | Loss: 2.9858 | LR: 2.65e-04 | 1280.8ms/step | 15.6 samples/s +Step 21540/44000 | Loss: 2.9955 | LR: 2.65e-04 | 1279.8ms/step | 15.6 samples/s +Step 21550/44000 | Loss: 2.9975 | LR: 2.65e-04 | 1280.8ms/step | 15.6 samples/s +Step 21560/44000 | Loss: 3.0000 | LR: 2.65e-04 | 1282.2ms/step | 15.6 samples/s +Step 21570/44000 | Loss: 2.9787 | LR: 2.65e-04 | 1282.4ms/step | 15.6 samples/s +Step 21580/44000 | Loss: 2.9720 | LR: 2.64e-04 | 1284.0ms/step | 15.6 samples/s +Step 21590/44000 | Loss: 2.9883 | LR: 2.64e-04 | 1282.0ms/step | 15.6 samples/s +Step 21600/44000 | Loss: 2.9888 | LR: 2.64e-04 | 1282.7ms/step | 15.6 samples/s +Step 21610/44000 | Loss: 2.9849 | LR: 2.64e-04 | 1283.0ms/step | 15.6 samples/s +Step 21620/44000 | Loss: 3.0072 | LR: 2.64e-04 | 1274.7ms/step | 15.7 samples/s +Step 21630/44000 | Loss: 2.9779 | LR: 2.64e-04 | 1275.7ms/step | 15.7 samples/s +Step 21640/44000 | Loss: 2.9691 | LR: 2.63e-04 | 1277.0ms/step | 15.7 samples/s +Step 21650/44000 | Loss: 2.9972 | LR: 2.63e-04 | 1276.2ms/step | 15.7 samples/s +Step 21660/44000 | Loss: 3.0302 | LR: 2.63e-04 | 1279.6ms/step | 15.6 samples/s +Step 21670/44000 | Loss: 2.9771 | LR: 2.63e-04 | 1281.0ms/step | 15.6 samples/s +Step 21680/44000 | Loss: 2.9970 | LR: 2.63e-04 | 1281.1ms/step | 15.6 samples/s +Step 21690/44000 | Loss: 3.0293 | LR: 2.62e-04 | 1282.8ms/step | 15.6 samples/s +Step 21700/44000 | Loss: 3.0100 | LR: 2.62e-04 | 1282.6ms/step | 15.6 samples/s +Step 21710/44000 | Loss: 2.9922 | LR: 2.62e-04 | 1282.2ms/step | 15.6 samples/s +Step 21720/44000 | Loss: 2.9653 | LR: 2.62e-04 | 1282.4ms/step | 15.6 samples/s +Step 21730/44000 | Loss: 2.9902 | LR: 2.62e-04 | 1279.7ms/step | 15.6 samples/s +Step 21740/44000 | Loss: 2.9802 | LR: 2.62e-04 | 1276.4ms/step | 15.7 samples/s +Step 21750/44000 | Loss: 2.9834 | LR: 2.61e-04 | 1282.4ms/step | 15.6 samples/s +Validation at step 21750: Loss = 3.0047, Perplexity = 20.18 +Step 21760/44000 | Loss: 3.0054 | LR: 2.61e-04 | 3430.6ms/step | 5.8 samples/s +Step 21770/44000 | Loss: 2.9897 | LR: 2.61e-04 | 1283.2ms/step | 15.6 samples/s +Step 21780/44000 | Loss: 2.9900 | LR: 2.61e-04 | 1284.1ms/step | 15.6 samples/s +Step 21790/44000 | Loss: 2.9556 | LR: 2.61e-04 | 1280.7ms/step | 15.6 samples/s +Step 21800/44000 | Loss: 2.9824 | LR: 2.60e-04 | 1283.4ms/step | 15.6 samples/s +Step 21810/44000 | Loss: 2.9782 | LR: 2.60e-04 | 1283.7ms/step | 15.6 samples/s +Step 21820/44000 | Loss: 2.9988 | LR: 2.60e-04 | 1281.1ms/step | 15.6 samples/s +Step 21830/44000 | Loss: 2.9703 | LR: 2.60e-04 | 1276.8ms/step | 15.7 samples/s +Step 21840/44000 | Loss: 2.9945 | LR: 2.60e-04 | 1280.2ms/step | 15.6 samples/s +Step 21850/44000 | Loss: 2.9897 | LR: 2.60e-04 | 1284.7ms/step | 15.6 samples/s +Step 21860/44000 | Loss: 2.9932 | LR: 2.59e-04 | 1280.2ms/step | 15.6 samples/s +Step 21870/44000 | Loss: 3.0054 | LR: 2.59e-04 | 1281.2ms/step | 15.6 samples/s +Step 21880/44000 | Loss: 2.9761 | LR: 2.59e-04 | 1282.2ms/step | 15.6 samples/s +Step 21890/44000 | Loss: 2.9774 | LR: 2.59e-04 | 1282.9ms/step | 15.6 samples/s +Step 21900/44000 | Loss: 2.9842 | LR: 2.59e-04 | 1283.4ms/step | 15.6 samples/s +Step 21910/44000 | Loss: 2.9845 | LR: 2.58e-04 | 1282.2ms/step | 15.6 samples/s +Step 21920/44000 | Loss: 2.9931 | LR: 2.58e-04 | 1281.5ms/step | 15.6 samples/s +Step 21930/44000 | Loss: 3.0187 | LR: 2.58e-04 | 1284.1ms/step | 15.6 samples/s +Step 21940/44000 | Loss: 2.9610 | LR: 2.58e-04 | 1283.5ms/step | 15.6 samples/s +Step 21950/44000 | Loss: 2.9922 | LR: 2.58e-04 | 1284.2ms/step | 15.6 samples/s +Step 21960/44000 | Loss: 2.9799 | LR: 2.58e-04 | 1282.7ms/step | 15.6 samples/s +Step 21970/44000 | Loss: 2.9759 | LR: 2.57e-04 | 1279.6ms/step | 15.6 samples/s +Step 21980/44000 | Loss: 2.9952 | LR: 2.57e-04 | 1277.0ms/step | 15.7 samples/s +Step 21990/44000 | Loss: 3.0000 | LR: 2.57e-04 | 1272.7ms/step | 15.7 samples/s +Step 22000/44000 | Loss: 2.9875 | LR: 2.57e-04 | 1282.4ms/step | 15.6 samples/s +Validation at step 22000: Loss = 3.0025, Perplexity = 20.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-22000 +Step 22010/44000 | Loss: 2.9893 | LR: 2.57e-04 | 3707.9ms/step | 5.4 samples/s +Step 22020/44000 | Loss: 3.0021 | LR: 2.56e-04 | 1282.2ms/step | 15.6 samples/s +Step 22030/44000 | Loss: 2.9978 | LR: 2.56e-04 | 1281.5ms/step | 15.6 samples/s +Step 22040/44000 | Loss: 3.0104 | LR: 2.56e-04 | 1280.0ms/step | 15.6 samples/s +Step 22050/44000 | Loss: 2.9835 | LR: 2.56e-04 | 1284.9ms/step | 15.6 samples/s +Step 22060/44000 | Loss: 2.9818 | LR: 2.56e-04 | 1282.1ms/step | 15.6 samples/s +Step 22070/44000 | Loss: 2.9878 | LR: 2.56e-04 | 1279.2ms/step | 15.6 samples/s +Step 22080/44000 | Loss: 2.9793 | LR: 2.55e-04 | 1282.0ms/step | 15.6 samples/s +Step 22090/44000 | Loss: 3.0058 | LR: 2.55e-04 | 1283.0ms/step | 15.6 samples/s +Step 22100/44000 | Loss: 2.9998 | LR: 2.55e-04 | 1280.9ms/step | 15.6 samples/s +Step 22110/44000 | Loss: 2.9836 | LR: 2.55e-04 | 1281.7ms/step | 15.6 samples/s +Step 22120/44000 | Loss: 2.9673 | LR: 2.55e-04 | 1281.1ms/step | 15.6 samples/s +Step 22130/44000 | Loss: 2.9756 | LR: 2.54e-04 | 1282.4ms/step | 15.6 samples/s +Step 22140/44000 | Loss: 2.9872 | LR: 2.54e-04 | 1283.3ms/step | 15.6 samples/s +Step 22150/44000 | Loss: 2.9655 | LR: 2.54e-04 | 1282.0ms/step | 15.6 samples/s +Step 22160/44000 | Loss: 2.9868 | LR: 2.54e-04 | 1281.1ms/step | 15.6 samples/s +Step 22170/44000 | Loss: 3.0047 | LR: 2.54e-04 | 1283.2ms/step | 15.6 samples/s +Step 22180/44000 | Loss: 2.9880 | LR: 2.54e-04 | 1281.7ms/step | 15.6 samples/s +Step 22190/44000 | Loss: 2.9861 | LR: 2.53e-04 | 1279.9ms/step | 15.6 samples/s +Step 22200/44000 | Loss: 2.9856 | LR: 2.53e-04 | 1282.5ms/step | 15.6 samples/s +Step 22210/44000 | Loss: 2.9879 | LR: 2.53e-04 | 1280.6ms/step | 15.6 samples/s +Step 22220/44000 | Loss: 2.9845 | LR: 2.53e-04 | 1280.1ms/step | 15.6 samples/s +Step 22230/44000 | Loss: 3.0016 | LR: 2.53e-04 | 1280.3ms/step | 15.6 samples/s +Step 22240/44000 | Loss: 3.0160 | LR: 2.52e-04 | 1281.0ms/step | 15.6 samples/s +Step 22250/44000 | Loss: 3.0014 | LR: 2.52e-04 | 1281.7ms/step | 15.6 samples/s +Validation at step 22250: Loss = 3.0001, Perplexity = 20.09 +Step 22260/44000 | Loss: 2.9693 | LR: 2.52e-04 | 3437.1ms/step | 5.8 samples/s +Step 22270/44000 | Loss: 3.0046 | LR: 2.52e-04 | 1279.6ms/step | 15.6 samples/s +Step 22280/44000 | Loss: 2.9827 | LR: 2.52e-04 | 1278.5ms/step | 15.6 samples/s +Step 22290/44000 | Loss: 2.9820 | LR: 2.52e-04 | 1278.8ms/step | 15.6 samples/s +Step 22300/44000 | Loss: 3.0178 | LR: 2.51e-04 | 1279.4ms/step | 15.6 samples/s +Step 22310/44000 | Loss: 2.9791 | LR: 2.51e-04 | 1281.4ms/step | 15.6 samples/s +Step 22320/44000 | Loss: 2.9650 | LR: 2.51e-04 | 1283.8ms/step | 15.6 samples/s +Step 22330/44000 | Loss: 2.9668 | LR: 2.51e-04 | 1285.2ms/step | 15.6 samples/s +Step 22340/44000 | Loss: 3.0149 | LR: 2.51e-04 | 1282.1ms/step | 15.6 samples/s +Step 22350/44000 | Loss: 2.9786 | LR: 2.50e-04 | 1283.8ms/step | 15.6 samples/s +Step 22360/44000 | Loss: 2.9784 | LR: 2.50e-04 | 1283.9ms/step | 15.6 samples/s +Step 22370/44000 | Loss: 2.9817 | LR: 2.50e-04 | 1280.3ms/step | 15.6 samples/s +Step 22380/44000 | Loss: 2.9859 | LR: 2.50e-04 | 1282.2ms/step | 15.6 samples/s +Step 22390/44000 | Loss: 2.9746 | LR: 2.50e-04 | 1283.3ms/step | 15.6 samples/s +Step 22400/44000 | Loss: 2.9582 | LR: 2.50e-04 | 1283.7ms/step | 15.6 samples/s +Step 22410/44000 | Loss: 2.9758 | LR: 2.49e-04 | 1284.5ms/step | 15.6 samples/s +Step 22420/44000 | Loss: 2.9762 | LR: 2.49e-04 | 1286.4ms/step | 15.5 samples/s +Step 22430/44000 | Loss: 2.9932 | LR: 2.49e-04 | 1280.5ms/step | 15.6 samples/s +Step 22440/44000 | Loss: 2.9816 | LR: 2.49e-04 | 1274.7ms/step | 15.7 samples/s +Step 22450/44000 | Loss: 2.9986 | LR: 2.49e-04 | 1274.9ms/step | 15.7 samples/s +Step 22460/44000 | Loss: 2.9766 | LR: 2.48e-04 | 1273.3ms/step | 15.7 samples/s +Step 22470/44000 | Loss: 2.9515 | LR: 2.48e-04 | 1277.2ms/step | 15.7 samples/s +Step 22480/44000 | Loss: 2.9663 | LR: 2.48e-04 | 1281.6ms/step | 15.6 samples/s +Step 22490/44000 | Loss: 2.9728 | LR: 2.48e-04 | 1284.5ms/step | 15.6 samples/s +Step 22500/44000 | Loss: 2.9568 | LR: 2.48e-04 | 1282.0ms/step | 15.6 samples/s +Validation at step 22500: Loss = 2.9973, Perplexity = 20.03 +Step 22510/44000 | Loss: 2.9667 | LR: 2.48e-04 | 3454.1ms/step | 5.8 samples/s +Step 22520/44000 | Loss: 3.0126 | LR: 2.47e-04 | 1281.0ms/step | 15.6 samples/s +Step 22530/44000 | Loss: 2.9497 | LR: 2.47e-04 | 1279.3ms/step | 15.6 samples/s +Step 22540/44000 | Loss: 2.9849 | LR: 2.47e-04 | 1282.0ms/step | 15.6 samples/s +Step 22550/44000 | Loss: 2.9520 | LR: 2.47e-04 | 1281.5ms/step | 15.6 samples/s +Step 22560/44000 | Loss: 2.9799 | LR: 2.47e-04 | 1281.0ms/step | 15.6 samples/s +Step 22570/44000 | Loss: 2.9566 | LR: 2.46e-04 | 1280.7ms/step | 15.6 samples/s +Step 22580/44000 | Loss: 2.9939 | LR: 2.46e-04 | 1284.6ms/step | 15.6 samples/s +Step 22590/44000 | Loss: 2.9902 | LR: 2.46e-04 | 1279.9ms/step | 15.6 samples/s +Step 22600/44000 | Loss: 2.9618 | LR: 2.46e-04 | 1282.5ms/step | 15.6 samples/s +Step 22610/44000 | Loss: 2.9790 | LR: 2.46e-04 | 1283.4ms/step | 15.6 samples/s +Step 22620/44000 | Loss: 2.9964 | LR: 2.46e-04 | 1282.5ms/step | 15.6 samples/s +Step 22630/44000 | Loss: 2.9752 | LR: 2.45e-04 | 1281.5ms/step | 15.6 samples/s +Step 22640/44000 | Loss: 2.9896 | LR: 2.45e-04 | 1281.0ms/step | 15.6 samples/s +Step 22650/44000 | Loss: 2.9782 | LR: 2.45e-04 | 1282.8ms/step | 15.6 samples/s +Step 22660/44000 | Loss: 3.0111 | LR: 2.45e-04 | 1284.4ms/step | 15.6 samples/s +Step 22670/44000 | Loss: 2.9859 | LR: 2.45e-04 | 1284.8ms/step | 15.6 samples/s +Step 22680/44000 | Loss: 2.9636 | LR: 2.44e-04 | 1283.4ms/step | 15.6 samples/s +Step 22690/44000 | Loss: 2.9945 | LR: 2.44e-04 | 1281.8ms/step | 15.6 samples/s +Step 22700/44000 | Loss: 2.9973 | LR: 2.44e-04 | 1280.6ms/step | 15.6 samples/s +Step 22710/44000 | Loss: 2.9519 | LR: 2.44e-04 | 1279.2ms/step | 15.6 samples/s +Step 22720/44000 | Loss: 2.9820 | LR: 2.44e-04 | 1280.9ms/step | 15.6 samples/s +Step 22730/44000 | Loss: 2.9972 | LR: 2.44e-04 | 1279.9ms/step | 15.6 samples/s +Step 22740/44000 | Loss: 2.9830 | LR: 2.43e-04 | 1277.9ms/step | 15.7 samples/s +Step 22750/44000 | Loss: 2.9599 | LR: 2.43e-04 | 1283.9ms/step | 15.6 samples/s +Validation at step 22750: Loss = 2.9946, Perplexity = 19.98 +Step 22760/44000 | Loss: 2.9946 | LR: 2.43e-04 | 3431.7ms/step | 5.8 samples/s +Step 22770/44000 | Loss: 2.9759 | LR: 2.43e-04 | 1277.5ms/step | 15.7 samples/s +Step 22780/44000 | Loss: 2.9858 | LR: 2.43e-04 | 1280.1ms/step | 15.6 samples/s +Step 22790/44000 | Loss: 3.0004 | LR: 2.42e-04 | 1280.8ms/step | 15.6 samples/s +Step 22800/44000 | Loss: 3.0024 | LR: 2.42e-04 | 1278.6ms/step | 15.6 samples/s +Step 22810/44000 | Loss: 2.9683 | LR: 2.42e-04 | 1275.3ms/step | 15.7 samples/s +Step 22820/44000 | Loss: 2.9819 | LR: 2.42e-04 | 1281.8ms/step | 15.6 samples/s +Step 22830/44000 | Loss: 3.0086 | LR: 2.42e-04 | 1281.2ms/step | 15.6 samples/s +Step 22840/44000 | Loss: 2.9917 | LR: 2.42e-04 | 1277.8ms/step | 15.7 samples/s +Step 22850/44000 | Loss: 2.9642 | LR: 2.41e-04 | 1282.8ms/step | 15.6 samples/s +Step 22860/44000 | Loss: 3.0032 | LR: 2.41e-04 | 1283.4ms/step | 15.6 samples/s +Step 22870/44000 | Loss: 2.9746 | LR: 2.41e-04 | 1283.1ms/step | 15.6 samples/s +Step 22880/44000 | Loss: 2.9761 | LR: 2.41e-04 | 1283.1ms/step | 15.6 samples/s +Step 22890/44000 | Loss: 2.9631 | LR: 2.41e-04 | 1283.1ms/step | 15.6 samples/s +Step 22900/44000 | Loss: 3.0104 | LR: 2.40e-04 | 1282.8ms/step | 15.6 samples/s +Step 22910/44000 | Loss: 2.9945 | LR: 2.40e-04 | 1279.3ms/step | 15.6 samples/s +Step 22920/44000 | Loss: 2.9824 | LR: 2.40e-04 | 1281.4ms/step | 15.6 samples/s +Step 22930/44000 | Loss: 2.9644 | LR: 2.40e-04 | 1281.4ms/step | 15.6 samples/s +Step 22940/44000 | Loss: 2.9680 | LR: 2.40e-04 | 1281.0ms/step | 15.6 samples/s +Step 22950/44000 | Loss: 2.9736 | LR: 2.40e-04 | 1281.0ms/step | 15.6 samples/s +Step 22960/44000 | Loss: 3.0149 | LR: 2.39e-04 | 1282.0ms/step | 15.6 samples/s +Step 22970/44000 | Loss: 2.9635 | LR: 2.39e-04 | 1281.7ms/step | 15.6 samples/s +Step 22980/44000 | Loss: 2.9778 | LR: 2.39e-04 | 1282.6ms/step | 15.6 samples/s +Step 22990/44000 | Loss: 2.9673 | LR: 2.39e-04 | 1285.0ms/step | 15.6 samples/s +Step 23000/44000 | Loss: 2.9641 | LR: 2.39e-04 | 1283.0ms/step | 15.6 samples/s +Validation at step 23000: Loss = 2.9930, Perplexity = 19.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-23000 +Step 23010/44000 | Loss: 2.9679 | LR: 2.38e-04 | 3702.2ms/step | 5.4 samples/s +Step 23020/44000 | Loss: 2.9785 | LR: 2.38e-04 | 1281.7ms/step | 15.6 samples/s +Step 23030/44000 | Loss: 2.9834 | LR: 2.38e-04 | 1282.3ms/step | 15.6 samples/s +Step 23040/44000 | Loss: 2.9985 | LR: 2.38e-04 | 1283.9ms/step | 15.6 samples/s +Step 23050/44000 | Loss: 3.0167 | LR: 2.38e-04 | 1283.4ms/step | 15.6 samples/s +Step 23060/44000 | Loss: 2.9804 | LR: 2.38e-04 | 1283.0ms/step | 15.6 samples/s +Step 23070/44000 | Loss: 2.9770 | LR: 2.37e-04 | 1283.3ms/step | 15.6 samples/s +Step 23080/44000 | Loss: 2.9801 | LR: 2.37e-04 | 1281.5ms/step | 15.6 samples/s +Step 23090/44000 | Loss: 2.9908 | LR: 2.37e-04 | 1280.4ms/step | 15.6 samples/s +Step 23100/44000 | Loss: 2.9788 | LR: 2.37e-04 | 1278.9ms/step | 15.6 samples/s +Step 23110/44000 | Loss: 2.9646 | LR: 2.37e-04 | 1277.3ms/step | 15.7 samples/s +Step 23120/44000 | Loss: 2.9715 | LR: 2.36e-04 | 1279.4ms/step | 15.6 samples/s +Step 23130/44000 | Loss: 2.9729 | LR: 2.36e-04 | 1284.2ms/step | 15.6 samples/s +Step 23140/44000 | Loss: 2.9515 | LR: 2.36e-04 | 1282.1ms/step | 15.6 samples/s +Step 23150/44000 | Loss: 2.9632 | LR: 2.36e-04 | 1282.0ms/step | 15.6 samples/s +Step 23160/44000 | Loss: 2.9841 | LR: 2.36e-04 | 1281.9ms/step | 15.6 samples/s +Step 23170/44000 | Loss: 2.9722 | LR: 2.36e-04 | 1285.5ms/step | 15.6 samples/s +Step 23180/44000 | Loss: 2.9844 | LR: 2.35e-04 | 1280.9ms/step | 15.6 samples/s +Step 23190/44000 | Loss: 2.9797 | LR: 2.35e-04 | 1284.4ms/step | 15.6 samples/s +Step 23200/44000 | Loss: 2.9794 | LR: 2.35e-04 | 1283.8ms/step | 15.6 samples/s +Step 23210/44000 | Loss: 2.9644 | LR: 2.35e-04 | 1282.8ms/step | 15.6 samples/s +Step 23220/44000 | Loss: 2.9820 | LR: 2.35e-04 | 1281.9ms/step | 15.6 samples/s +Step 23230/44000 | Loss: 2.9602 | LR: 2.34e-04 | 1282.6ms/step | 15.6 samples/s +Step 23240/44000 | Loss: 2.9810 | LR: 2.34e-04 | 1281.4ms/step | 15.6 samples/s +Step 23250/44000 | Loss: 2.9837 | LR: 2.34e-04 | 1283.5ms/step | 15.6 samples/s +Validation at step 23250: Loss = 2.9908, Perplexity = 19.90 +Step 23260/44000 | Loss: 2.9546 | LR: 2.34e-04 | 3432.3ms/step | 5.8 samples/s +Step 23270/44000 | Loss: 2.9786 | LR: 2.34e-04 | 1283.0ms/step | 15.6 samples/s +Step 23280/44000 | Loss: 2.9906 | LR: 2.34e-04 | 1283.2ms/step | 15.6 samples/s +Step 23290/44000 | Loss: 2.9710 | LR: 2.33e-04 | 1284.1ms/step | 15.6 samples/s +Step 23300/44000 | Loss: 2.9634 | LR: 2.33e-04 | 1284.7ms/step | 15.6 samples/s +Step 23310/44000 | Loss: 2.9419 | LR: 2.33e-04 | 1283.9ms/step | 15.6 samples/s +Step 23320/44000 | Loss: 2.9692 | LR: 2.33e-04 | 1281.1ms/step | 15.6 samples/s +Step 23330/44000 | Loss: 2.9610 | LR: 2.33e-04 | 1281.9ms/step | 15.6 samples/s +Step 23340/44000 | Loss: 2.9868 | LR: 2.32e-04 | 1281.6ms/step | 15.6 samples/s +Step 23350/44000 | Loss: 2.9714 | LR: 2.32e-04 | 1282.1ms/step | 15.6 samples/s +Step 23360/44000 | Loss: 3.0141 | LR: 2.32e-04 | 1282.6ms/step | 15.6 samples/s +Step 23370/44000 | Loss: 2.9645 | LR: 2.32e-04 | 1281.6ms/step | 15.6 samples/s +Step 23380/44000 | Loss: 2.9576 | LR: 2.32e-04 | 1281.0ms/step | 15.6 samples/s +Step 23390/44000 | Loss: 2.9440 | LR: 2.32e-04 | 1281.4ms/step | 15.6 samples/s +Step 23400/44000 | Loss: 2.9827 | LR: 2.31e-04 | 1281.4ms/step | 15.6 samples/s +Step 23410/44000 | Loss: 2.9714 | LR: 2.31e-04 | 1283.2ms/step | 15.6 samples/s +Step 23420/44000 | Loss: 2.9925 | LR: 2.31e-04 | 1288.6ms/step | 15.5 samples/s +Step 23430/44000 | Loss: 2.9607 | LR: 2.31e-04 | 1285.3ms/step | 15.6 samples/s +Step 23440/44000 | Loss: 2.9601 | LR: 2.31e-04 | 1285.1ms/step | 15.6 samples/s +Step 23450/44000 | Loss: 2.9805 | LR: 2.30e-04 | 1287.0ms/step | 15.5 samples/s +Step 23460/44000 | Loss: 2.9639 | LR: 2.30e-04 | 1296.6ms/step | 15.4 samples/s +Step 23470/44000 | Loss: 2.9861 | LR: 2.30e-04 | 1289.7ms/step | 15.5 samples/s +Step 23480/44000 | Loss: 2.9595 | LR: 2.30e-04 | 1290.2ms/step | 15.5 samples/s +Step 23490/44000 | Loss: 2.9761 | LR: 2.30e-04 | 1286.0ms/step | 15.6 samples/s +Step 23500/44000 | Loss: 2.9624 | LR: 2.30e-04 | 1284.6ms/step | 15.6 samples/s +Validation at step 23500: Loss = 2.9887, Perplexity = 19.86 +Step 23510/44000 | Loss: 2.9794 | LR: 2.29e-04 | 3446.4ms/step | 5.8 samples/s +Step 23520/44000 | Loss: 2.9697 | LR: 2.29e-04 | 1279.4ms/step | 15.6 samples/s +Step 23530/44000 | Loss: 2.9669 | LR: 2.29e-04 | 1279.0ms/step | 15.6 samples/s +Step 23540/44000 | Loss: 2.9507 | LR: 2.29e-04 | 1278.9ms/step | 15.6 samples/s +Step 23550/44000 | Loss: 2.9516 | LR: 2.29e-04 | 1282.9ms/step | 15.6 samples/s +Step 23560/44000 | Loss: 2.9697 | LR: 2.29e-04 | 1285.8ms/step | 15.6 samples/s +Step 23570/44000 | Loss: 2.9949 | LR: 2.28e-04 | 1281.9ms/step | 15.6 samples/s +Step 23580/44000 | Loss: 2.9456 | LR: 2.28e-04 | 1279.0ms/step | 15.6 samples/s +Step 23590/44000 | Loss: 2.9867 | LR: 2.28e-04 | 1278.3ms/step | 15.6 samples/s +Step 23600/44000 | Loss: 3.0025 | LR: 2.28e-04 | 1280.6ms/step | 15.6 samples/s +Step 23610/44000 | Loss: 2.9615 | LR: 2.28e-04 | 1277.5ms/step | 15.7 samples/s +Step 23620/44000 | Loss: 2.9680 | LR: 2.27e-04 | 1280.2ms/step | 15.6 samples/s +Step 23630/44000 | Loss: 2.9716 | LR: 2.27e-04 | 1280.1ms/step | 15.6 samples/s +Step 23640/44000 | Loss: 2.9866 | LR: 2.27e-04 | 1279.9ms/step | 15.6 samples/s +Step 23650/44000 | Loss: 2.9960 | LR: 2.27e-04 | 1282.3ms/step | 15.6 samples/s +Step 23660/44000 | Loss: 2.9744 | LR: 2.27e-04 | 1279.7ms/step | 15.6 samples/s +Step 23670/44000 | Loss: 2.9833 | LR: 2.27e-04 | 1282.6ms/step | 15.6 samples/s +Step 23680/44000 | Loss: 2.9824 | LR: 2.26e-04 | 1283.3ms/step | 15.6 samples/s +Step 23690/44000 | Loss: 2.9831 | LR: 2.26e-04 | 1278.0ms/step | 15.6 samples/s +Step 23700/44000 | Loss: 2.9678 | LR: 2.26e-04 | 1279.0ms/step | 15.6 samples/s +Step 23710/44000 | Loss: 2.9518 | LR: 2.26e-04 | 1276.7ms/step | 15.7 samples/s +Step 23720/44000 | Loss: 2.9583 | LR: 2.26e-04 | 1277.2ms/step | 15.7 samples/s +Step 23730/44000 | Loss: 2.9771 | LR: 2.25e-04 | 1277.4ms/step | 15.7 samples/s +Step 23740/44000 | Loss: 2.9745 | LR: 2.25e-04 | 1278.9ms/step | 15.6 samples/s +Step 23750/44000 | Loss: 2.9581 | LR: 2.25e-04 | 1280.3ms/step | 15.6 samples/s +Validation at step 23750: Loss = 2.9860, Perplexity = 19.81 +Step 23760/44000 | Loss: 2.9853 | LR: 2.25e-04 | 3436.0ms/step | 5.8 samples/s +Step 23770/44000 | Loss: 2.9590 | LR: 2.25e-04 | 1278.0ms/step | 15.6 samples/s +Step 23780/44000 | Loss: 2.9575 | LR: 2.25e-04 | 1279.2ms/step | 15.6 samples/s +Step 23790/44000 | Loss: 2.9529 | LR: 2.24e-04 | 1275.2ms/step | 15.7 samples/s +Step 23800/44000 | Loss: 2.9767 | LR: 2.24e-04 | 1278.3ms/step | 15.6 samples/s +Step 23810/44000 | Loss: 2.9523 | LR: 2.24e-04 | 1280.6ms/step | 15.6 samples/s +Step 23820/44000 | Loss: 2.9975 | LR: 2.24e-04 | 1283.9ms/step | 15.6 samples/s +Step 23830/44000 | Loss: 2.9641 | LR: 2.24e-04 | 1283.9ms/step | 15.6 samples/s +Step 23840/44000 | Loss: 2.9775 | LR: 2.23e-04 | 1285.9ms/step | 15.6 samples/s +Step 23850/44000 | Loss: 2.9803 | LR: 2.23e-04 | 1289.9ms/step | 15.5 samples/s +Step 23860/44000 | Loss: 2.9687 | LR: 2.23e-04 | 1288.7ms/step | 15.5 samples/s +Step 23870/44000 | Loss: 2.9825 | LR: 2.23e-04 | 1282.9ms/step | 15.6 samples/s +Step 23880/44000 | Loss: 2.9658 | LR: 2.23e-04 | 1284.9ms/step | 15.6 samples/s +Step 23890/44000 | Loss: 2.9658 | LR: 2.23e-04 | 1290.0ms/step | 15.5 samples/s +Step 23900/44000 | Loss: 2.9626 | LR: 2.22e-04 | 1287.5ms/step | 15.5 samples/s +Step 23910/44000 | Loss: 2.9597 | LR: 2.22e-04 | 1289.8ms/step | 15.5 samples/s +Step 23920/44000 | Loss: 3.0026 | LR: 2.22e-04 | 1288.4ms/step | 15.5 samples/s +Step 23930/44000 | Loss: 2.9764 | LR: 2.22e-04 | 1291.5ms/step | 15.5 samples/s +Step 23940/44000 | Loss: 2.9877 | LR: 2.22e-04 | 1287.8ms/step | 15.5 samples/s +Step 23950/44000 | Loss: 2.9815 | LR: 2.21e-04 | 1279.3ms/step | 15.6 samples/s +Step 23960/44000 | Loss: 3.0049 | LR: 2.21e-04 | 1277.3ms/step | 15.7 samples/s +Step 23970/44000 | Loss: 2.9843 | LR: 2.21e-04 | 1282.0ms/step | 15.6 samples/s +Step 23980/44000 | Loss: 2.9645 | LR: 2.21e-04 | 1284.0ms/step | 15.6 samples/s +Step 23990/44000 | Loss: 2.9426 | LR: 2.21e-04 | 1284.0ms/step | 15.6 samples/s +Step 24000/44000 | Loss: 2.9642 | LR: 2.21e-04 | 1283.7ms/step | 15.6 samples/s +Validation at step 24000: Loss = 2.9840, Perplexity = 19.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-24000 +Step 24010/44000 | Loss: 2.9757 | LR: 2.20e-04 | 3729.3ms/step | 5.4 samples/s +Step 24020/44000 | Loss: 2.9705 | LR: 2.20e-04 | 1284.4ms/step | 15.6 samples/s +Step 24030/44000 | Loss: 2.9923 | LR: 2.20e-04 | 1282.0ms/step | 15.6 samples/s +Step 24040/44000 | Loss: 3.0047 | LR: 2.20e-04 | 1282.2ms/step | 15.6 samples/s +Step 24050/44000 | Loss: 2.9940 | LR: 2.20e-04 | 1284.9ms/step | 15.6 samples/s +Step 24060/44000 | Loss: 2.9791 | LR: 2.19e-04 | 1283.7ms/step | 15.6 samples/s +Step 24070/44000 | Loss: 2.9960 | LR: 2.19e-04 | 1281.5ms/step | 15.6 samples/s +Step 24080/44000 | Loss: 2.9547 | LR: 2.19e-04 | 1282.4ms/step | 15.6 samples/s +Step 24090/44000 | Loss: 2.9548 | LR: 2.19e-04 | 1283.3ms/step | 15.6 samples/s +Step 24100/44000 | Loss: 2.9731 | LR: 2.19e-04 | 1284.7ms/step | 15.6 samples/s +Step 24110/44000 | Loss: 2.9830 | LR: 2.19e-04 | 1286.7ms/step | 15.5 samples/s +Step 24120/44000 | Loss: 2.9373 | LR: 2.18e-04 | 1286.4ms/step | 15.5 samples/s +Step 24130/44000 | Loss: 2.9863 | LR: 2.18e-04 | 1284.1ms/step | 15.6 samples/s +Step 24140/44000 | Loss: 2.9623 | LR: 2.18e-04 | 1282.6ms/step | 15.6 samples/s +Step 24150/44000 | Loss: 2.9882 | LR: 2.18e-04 | 1285.3ms/step | 15.6 samples/s +Step 24160/44000 | Loss: 3.0000 | LR: 2.18e-04 | 1285.7ms/step | 15.6 samples/s +Step 24170/44000 | Loss: 2.9559 | LR: 2.17e-04 | 1282.3ms/step | 15.6 samples/s +Step 24180/44000 | Loss: 2.9652 | LR: 2.17e-04 | 1283.9ms/step | 15.6 samples/s +Step 24190/44000 | Loss: 2.9835 | LR: 2.17e-04 | 1283.9ms/step | 15.6 samples/s +Step 24200/44000 | Loss: 2.9569 | LR: 2.17e-04 | 1283.7ms/step | 15.6 samples/s +Step 24210/44000 | Loss: 2.9494 | LR: 2.17e-04 | 1283.7ms/step | 15.6 samples/s +Step 24220/44000 | Loss: 2.9679 | LR: 2.17e-04 | 1279.6ms/step | 15.6 samples/s +Step 24230/44000 | Loss: 2.9896 | LR: 2.16e-04 | 1282.4ms/step | 15.6 samples/s +Step 24240/44000 | Loss: 2.9525 | LR: 2.16e-04 | 1278.8ms/step | 15.6 samples/s +Step 24250/44000 | Loss: 2.9777 | LR: 2.16e-04 | 1282.7ms/step | 15.6 samples/s +Validation at step 24250: Loss = 2.9825, Perplexity = 19.74 +Step 24260/44000 | Loss: 2.9477 | LR: 2.16e-04 | 3433.9ms/step | 5.8 samples/s +Step 24270/44000 | Loss: 2.9665 | LR: 2.16e-04 | 1285.1ms/step | 15.6 samples/s +Step 24280/44000 | Loss: 2.9919 | LR: 2.16e-04 | 1281.9ms/step | 15.6 samples/s +Step 24290/44000 | Loss: 2.9743 | LR: 2.15e-04 | 1283.3ms/step | 15.6 samples/s +Step 24300/44000 | Loss: 2.9585 | LR: 2.15e-04 | 1281.5ms/step | 15.6 samples/s +Step 24310/44000 | Loss: 2.9724 | LR: 2.15e-04 | 1281.1ms/step | 15.6 samples/s +Step 24320/44000 | Loss: 2.9714 | LR: 2.15e-04 | 1283.1ms/step | 15.6 samples/s +Step 24330/44000 | Loss: 2.9891 | LR: 2.15e-04 | 1283.3ms/step | 15.6 samples/s +Step 24340/44000 | Loss: 2.9519 | LR: 2.14e-04 | 1284.6ms/step | 15.6 samples/s +Step 24350/44000 | Loss: 2.9437 | LR: 2.14e-04 | 1279.4ms/step | 15.6 samples/s +Step 24360/44000 | Loss: 2.9691 | LR: 2.14e-04 | 1279.6ms/step | 15.6 samples/s +Step 24370/44000 | Loss: 2.9377 | LR: 2.14e-04 | 1281.9ms/step | 15.6 samples/s +Step 24380/44000 | Loss: 2.9320 | LR: 2.14e-04 | 1283.7ms/step | 15.6 samples/s +Step 24390/44000 | Loss: 2.9401 | LR: 2.14e-04 | 1286.9ms/step | 15.5 samples/s +Step 24400/44000 | Loss: 2.9708 | LR: 2.13e-04 | 1286.0ms/step | 15.6 samples/s +Step 24410/44000 | Loss: 2.9702 | LR: 2.13e-04 | 1284.1ms/step | 15.6 samples/s +Step 24420/44000 | Loss: 2.9829 | LR: 2.13e-04 | 1283.0ms/step | 15.6 samples/s +Step 24430/44000 | Loss: 2.9678 | LR: 2.13e-04 | 1284.9ms/step | 15.6 samples/s +Step 24440/44000 | Loss: 2.9734 | LR: 2.13e-04 | 1282.5ms/step | 15.6 samples/s +Step 24450/44000 | Loss: 2.9410 | LR: 2.12e-04 | 1281.4ms/step | 15.6 samples/s +Step 24460/44000 | Loss: 2.9480 | LR: 2.12e-04 | 1282.8ms/step | 15.6 samples/s +Step 24470/44000 | Loss: 2.9736 | LR: 2.12e-04 | 1282.0ms/step | 15.6 samples/s +Step 24480/44000 | Loss: 2.9680 | LR: 2.12e-04 | 1283.7ms/step | 15.6 samples/s +Step 24490/44000 | Loss: 2.9636 | LR: 2.12e-04 | 1281.8ms/step | 15.6 samples/s +Step 24500/44000 | Loss: 2.9807 | LR: 2.12e-04 | 1278.1ms/step | 15.6 samples/s +Validation at step 24500: Loss = 2.9806, Perplexity = 19.70 +Step 24510/44000 | Loss: 2.9690 | LR: 2.11e-04 | 3432.6ms/step | 5.8 samples/s +Step 24520/44000 | Loss: 2.9386 | LR: 2.11e-04 | 1280.2ms/step | 15.6 samples/s +Step 24530/44000 | Loss: 2.9627 | LR: 2.11e-04 | 1282.8ms/step | 15.6 samples/s +Step 24540/44000 | Loss: 2.9835 | LR: 2.11e-04 | 1278.4ms/step | 15.6 samples/s +Step 24550/44000 | Loss: 2.9967 | LR: 2.11e-04 | 1284.7ms/step | 15.6 samples/s +Step 24560/44000 | Loss: 2.9628 | LR: 2.10e-04 | 1280.6ms/step | 15.6 samples/s +Step 24570/44000 | Loss: 3.0316 | LR: 2.10e-04 | 1278.1ms/step | 15.6 samples/s +Step 24580/44000 | Loss: 2.9459 | LR: 2.10e-04 | 1275.2ms/step | 15.7 samples/s +Step 24590/44000 | Loss: 2.9931 | LR: 2.10e-04 | 1275.1ms/step | 15.7 samples/s +Step 24600/44000 | Loss: 2.9647 | LR: 2.10e-04 | 1281.9ms/step | 15.6 samples/s +Step 24610/44000 | Loss: 2.9799 | LR: 2.10e-04 | 1282.1ms/step | 15.6 samples/s +Step 24620/44000 | Loss: 2.9757 | LR: 2.09e-04 | 1282.3ms/step | 15.6 samples/s +Step 24630/44000 | Loss: 3.0000 | LR: 2.09e-04 | 1280.6ms/step | 15.6 samples/s +Step 24640/44000 | Loss: 2.9679 | LR: 2.09e-04 | 1285.7ms/step | 15.6 samples/s +Step 24650/44000 | Loss: 2.9686 | LR: 2.09e-04 | 1285.5ms/step | 15.6 samples/s +Step 24660/44000 | Loss: 2.9604 | LR: 2.09e-04 | 1282.1ms/step | 15.6 samples/s +Step 24670/44000 | Loss: 2.9614 | LR: 2.09e-04 | 1281.2ms/step | 15.6 samples/s +Step 24680/44000 | Loss: 2.9575 | LR: 2.08e-04 | 1284.9ms/step | 15.6 samples/s +Step 24690/44000 | Loss: 2.9633 | LR: 2.08e-04 | 1282.7ms/step | 15.6 samples/s +Step 24700/44000 | Loss: 2.9591 | LR: 2.08e-04 | 1282.8ms/step | 15.6 samples/s +Step 24710/44000 | Loss: 2.9616 | LR: 2.08e-04 | 1284.4ms/step | 15.6 samples/s +Step 24720/44000 | Loss: 2.9490 | LR: 2.08e-04 | 1286.6ms/step | 15.5 samples/s +Step 24730/44000 | Loss: 2.9589 | LR: 2.07e-04 | 1284.5ms/step | 15.6 samples/s +Step 24740/44000 | Loss: 3.0070 | LR: 2.07e-04 | 1283.2ms/step | 15.6 samples/s +Step 24750/44000 | Loss: 2.9444 | LR: 2.07e-04 | 1284.0ms/step | 15.6 samples/s +Validation at step 24750: Loss = 2.9787, Perplexity = 19.66 +Step 24760/44000 | Loss: 2.9656 | LR: 2.07e-04 | 3442.5ms/step | 5.8 samples/s +Step 24770/44000 | Loss: 2.9648 | LR: 2.07e-04 | 1290.9ms/step | 15.5 samples/s +Step 24780/44000 | Loss: 2.9884 | LR: 2.07e-04 | 1285.3ms/step | 15.6 samples/s +Step 24790/44000 | Loss: 2.9447 | LR: 2.06e-04 | 1294.5ms/step | 15.4 samples/s +Step 24800/44000 | Loss: 2.9481 | LR: 2.06e-04 | 1294.4ms/step | 15.5 samples/s +Step 24810/44000 | Loss: 2.9805 | LR: 2.06e-04 | 1289.1ms/step | 15.5 samples/s +Step 24820/44000 | Loss: 2.9632 | LR: 2.06e-04 | 1285.2ms/step | 15.6 samples/s +Step 24830/44000 | Loss: 2.9401 | LR: 2.06e-04 | 1284.9ms/step | 15.6 samples/s +Step 24840/44000 | Loss: 2.9563 | LR: 2.05e-04 | 1284.0ms/step | 15.6 samples/s +Step 24850/44000 | Loss: 2.9597 | LR: 2.05e-04 | 1292.2ms/step | 15.5 samples/s +Step 24860/44000 | Loss: 2.9595 | LR: 2.05e-04 | 1283.7ms/step | 15.6 samples/s +Step 24870/44000 | Loss: 2.9499 | LR: 2.05e-04 | 1281.1ms/step | 15.6 samples/s +Step 24880/44000 | Loss: 2.9766 | LR: 2.05e-04 | 1281.9ms/step | 15.6 samples/s +Step 24890/44000 | Loss: 2.9998 | LR: 2.05e-04 | 1279.4ms/step | 15.6 samples/s +Step 24900/44000 | Loss: 2.9560 | LR: 2.04e-04 | 1277.8ms/step | 15.7 samples/s +Step 24910/44000 | Loss: 2.9662 | LR: 2.04e-04 | 1276.3ms/step | 15.7 samples/s +Step 24920/44000 | Loss: 2.9725 | LR: 2.04e-04 | 1281.6ms/step | 15.6 samples/s +Step 24930/44000 | Loss: 2.9656 | LR: 2.04e-04 | 1282.1ms/step | 15.6 samples/s +Step 24940/44000 | Loss: 2.9757 | LR: 2.04e-04 | 1281.3ms/step | 15.6 samples/s +Step 24950/44000 | Loss: 2.9524 | LR: 2.04e-04 | 1282.6ms/step | 15.6 samples/s +Step 24960/44000 | Loss: 3.0000 | LR: 2.03e-04 | 1285.0ms/step | 15.6 samples/s +Step 24970/44000 | Loss: 2.9535 | LR: 2.03e-04 | 1282.8ms/step | 15.6 samples/s +Step 24980/44000 | Loss: 2.9858 | LR: 2.03e-04 | 1281.3ms/step | 15.6 samples/s +Step 24990/44000 | Loss: 2.9379 | LR: 2.03e-04 | 1282.7ms/step | 15.6 samples/s +Step 25000/44000 | Loss: 2.9809 | LR: 2.03e-04 | 1283.0ms/step | 15.6 samples/s +Validation at step 25000: Loss = 2.9767, Perplexity = 19.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-25000 +Step 25010/44000 | Loss: 2.9433 | LR: 2.02e-04 | 3701.6ms/step | 5.4 samples/s +Step 25020/44000 | Loss: 2.9738 | LR: 2.02e-04 | 1280.3ms/step | 15.6 samples/s +Step 25030/44000 | Loss: 2.9539 | LR: 2.02e-04 | 1279.9ms/step | 15.6 samples/s +Step 25040/44000 | Loss: 2.9696 | LR: 2.02e-04 | 1285.2ms/step | 15.6 samples/s +Step 25050/44000 | Loss: 2.9597 | LR: 2.02e-04 | 1279.5ms/step | 15.6 samples/s +Step 25060/44000 | Loss: 2.9570 | LR: 2.02e-04 | 1280.6ms/step | 15.6 samples/s +Step 25070/44000 | Loss: 2.9806 | LR: 2.01e-04 | 1283.8ms/step | 15.6 samples/s +Step 25080/44000 | Loss: 2.9434 | LR: 2.01e-04 | 1278.4ms/step | 15.6 samples/s +Step 25090/44000 | Loss: 2.9812 | LR: 2.01e-04 | 1277.2ms/step | 15.7 samples/s +Step 25100/44000 | Loss: 2.9436 | LR: 2.01e-04 | 1275.5ms/step | 15.7 samples/s +Step 25110/44000 | Loss: 2.9675 | LR: 2.01e-04 | 1278.3ms/step | 15.6 samples/s +Step 25120/44000 | Loss: 2.9405 | LR: 2.00e-04 | 1277.3ms/step | 15.7 samples/s +Step 25130/44000 | Loss: 2.9948 | LR: 2.00e-04 | 1280.3ms/step | 15.6 samples/s +Step 25140/44000 | Loss: 3.0039 | LR: 2.00e-04 | 1276.7ms/step | 15.7 samples/s +Step 25150/44000 | Loss: 2.9273 | LR: 2.00e-04 | 1277.6ms/step | 15.7 samples/s +Step 25160/44000 | Loss: 2.9408 | LR: 2.00e-04 | 1277.4ms/step | 15.7 samples/s +Step 25170/44000 | Loss: 2.9756 | LR: 2.00e-04 | 1279.3ms/step | 15.6 samples/s +Step 25180/44000 | Loss: 2.9157 | LR: 1.99e-04 | 1280.9ms/step | 15.6 samples/s +Step 25190/44000 | Loss: 2.9579 | LR: 1.99e-04 | 1285.0ms/step | 15.6 samples/s +Step 25200/44000 | Loss: 2.9805 | LR: 1.99e-04 | 1282.9ms/step | 15.6 samples/s +Step 25210/44000 | Loss: 2.9744 | LR: 1.99e-04 | 1285.7ms/step | 15.6 samples/s +Step 25220/44000 | Loss: 2.9437 | LR: 1.99e-04 | 1282.3ms/step | 15.6 samples/s +Step 25230/44000 | Loss: 2.9649 | LR: 1.99e-04 | 1288.2ms/step | 15.5 samples/s +Step 25240/44000 | Loss: 2.9462 | LR: 1.98e-04 | 1281.7ms/step | 15.6 samples/s +Step 25250/44000 | Loss: 2.9778 | LR: 1.98e-04 | 1277.4ms/step | 15.7 samples/s +Validation at step 25250: Loss = 2.9749, Perplexity = 19.59 +Step 25260/44000 | Loss: 2.9383 | LR: 1.98e-04 | 3437.8ms/step | 5.8 samples/s +Step 25270/44000 | Loss: 2.9693 | LR: 1.98e-04 | 1284.7ms/step | 15.6 samples/s +Step 25280/44000 | Loss: 2.9365 | LR: 1.98e-04 | 1286.9ms/step | 15.5 samples/s +Step 25290/44000 | Loss: 2.9580 | LR: 1.97e-04 | 1281.4ms/step | 15.6 samples/s +Step 25300/44000 | Loss: 2.9637 | LR: 1.97e-04 | 1280.3ms/step | 15.6 samples/s +Step 25310/44000 | Loss: 2.9135 | LR: 1.97e-04 | 1276.8ms/step | 15.7 samples/s +Step 25320/44000 | Loss: 2.9528 | LR: 1.97e-04 | 1281.8ms/step | 15.6 samples/s +Step 25330/44000 | Loss: 2.9632 | LR: 1.97e-04 | 1284.0ms/step | 15.6 samples/s +Step 25340/44000 | Loss: 2.9572 | LR: 1.97e-04 | 1286.1ms/step | 15.6 samples/s +Step 25350/44000 | Loss: 2.9486 | LR: 1.96e-04 | 1283.2ms/step | 15.6 samples/s +Step 25360/44000 | Loss: 2.9961 | LR: 1.96e-04 | 1285.5ms/step | 15.6 samples/s +Step 25370/44000 | Loss: 2.9441 | LR: 1.96e-04 | 1285.6ms/step | 15.6 samples/s +Step 25380/44000 | Loss: 2.9846 | LR: 1.96e-04 | 1284.7ms/step | 15.6 samples/s +Step 25390/44000 | Loss: 2.9559 | LR: 1.96e-04 | 1282.9ms/step | 15.6 samples/s +Step 25400/44000 | Loss: 2.9687 | LR: 1.96e-04 | 1281.7ms/step | 15.6 samples/s +Step 25410/44000 | Loss: 2.9994 | LR: 1.95e-04 | 1281.1ms/step | 15.6 samples/s +Step 25420/44000 | Loss: 2.9823 | LR: 1.95e-04 | 1281.3ms/step | 15.6 samples/s +Step 25430/44000 | Loss: 2.9611 | LR: 1.95e-04 | 1281.5ms/step | 15.6 samples/s +Step 25440/44000 | Loss: 2.9620 | LR: 1.95e-04 | 1282.7ms/step | 15.6 samples/s +Step 25450/44000 | Loss: 2.9969 | LR: 1.95e-04 | 1278.4ms/step | 15.6 samples/s +Step 25460/44000 | Loss: 2.9697 | LR: 1.94e-04 | 1277.6ms/step | 15.7 samples/s +Step 25470/44000 | Loss: 2.9489 | LR: 1.94e-04 | 1281.3ms/step | 15.6 samples/s +Step 25480/44000 | Loss: 2.9638 | LR: 1.94e-04 | 1275.7ms/step | 15.7 samples/s +Step 25490/44000 | Loss: 2.9641 | LR: 1.94e-04 | 1280.9ms/step | 15.6 samples/s +Step 25500/44000 | Loss: 2.9546 | LR: 1.94e-04 | 1285.7ms/step | 15.6 samples/s +Validation at step 25500: Loss = 2.9726, Perplexity = 19.54 +Step 25510/44000 | Loss: 2.9421 | LR: 1.94e-04 | 3440.1ms/step | 5.8 samples/s +Step 25520/44000 | Loss: 2.9417 | LR: 1.93e-04 | 1285.0ms/step | 15.6 samples/s +Step 25530/44000 | Loss: 2.9437 | LR: 1.93e-04 | 1285.3ms/step | 15.6 samples/s +Step 25540/44000 | Loss: 2.9515 | LR: 1.93e-04 | 1279.8ms/step | 15.6 samples/s +Step 25550/44000 | Loss: 2.9626 | LR: 1.93e-04 | 1283.0ms/step | 15.6 samples/s +Step 25560/44000 | Loss: 2.9705 | LR: 1.93e-04 | 1281.8ms/step | 15.6 samples/s +Step 25570/44000 | Loss: 2.9532 | LR: 1.92e-04 | 1281.7ms/step | 15.6 samples/s +Step 25580/44000 | Loss: 2.9871 | LR: 1.92e-04 | 1281.4ms/step | 15.6 samples/s +Step 25590/44000 | Loss: 2.9662 | LR: 1.92e-04 | 1280.9ms/step | 15.6 samples/s +Step 25600/44000 | Loss: 2.9324 | LR: 1.92e-04 | 1281.5ms/step | 15.6 samples/s +Step 25610/44000 | Loss: 2.9474 | LR: 1.92e-04 | 1280.1ms/step | 15.6 samples/s +Step 25620/44000 | Loss: 2.9350 | LR: 1.92e-04 | 1279.6ms/step | 15.6 samples/s +Step 25630/44000 | Loss: 2.9670 | LR: 1.91e-04 | 1282.9ms/step | 15.6 samples/s +Step 25640/44000 | Loss: 2.9501 | LR: 1.91e-04 | 1283.3ms/step | 15.6 samples/s +Step 25650/44000 | Loss: 2.9552 | LR: 1.91e-04 | 1282.5ms/step | 15.6 samples/s +Step 25660/44000 | Loss: 2.9764 | LR: 1.91e-04 | 1280.7ms/step | 15.6 samples/s +Step 25670/44000 | Loss: 2.9668 | LR: 1.91e-04 | 1280.0ms/step | 15.6 samples/s +Step 25680/44000 | Loss: 2.9334 | LR: 1.91e-04 | 1281.7ms/step | 15.6 samples/s +Step 25690/44000 | Loss: 2.9519 | LR: 1.90e-04 | 1281.1ms/step | 15.6 samples/s +Step 25700/44000 | Loss: 2.9357 | LR: 1.90e-04 | 1282.1ms/step | 15.6 samples/s +Step 25710/44000 | Loss: 2.9514 | LR: 1.90e-04 | 1283.4ms/step | 15.6 samples/s +Step 25720/44000 | Loss: 2.9449 | LR: 1.90e-04 | 1279.6ms/step | 15.6 samples/s +Step 25730/44000 | Loss: 2.9388 | LR: 1.90e-04 | 1276.4ms/step | 15.7 samples/s +Step 25740/44000 | Loss: 2.9524 | LR: 1.89e-04 | 1276.7ms/step | 15.7 samples/s +Step 25750/44000 | Loss: 2.9526 | LR: 1.89e-04 | 1280.6ms/step | 15.6 samples/s +Validation at step 25750: Loss = 2.9715, Perplexity = 19.52 +Step 25760/44000 | Loss: 2.9526 | LR: 1.89e-04 | 3436.8ms/step | 5.8 samples/s +Step 25770/44000 | Loss: 2.9771 | LR: 1.89e-04 | 1282.5ms/step | 15.6 samples/s +Step 25780/44000 | Loss: 2.9615 | LR: 1.89e-04 | 1281.8ms/step | 15.6 samples/s +Step 25790/44000 | Loss: 2.9961 | LR: 1.89e-04 | 1281.3ms/step | 15.6 samples/s +Step 25800/44000 | Loss: 2.9188 | LR: 1.88e-04 | 1280.9ms/step | 15.6 samples/s +Step 25810/44000 | Loss: 2.9541 | LR: 1.88e-04 | 1279.6ms/step | 15.6 samples/s +Step 25820/44000 | Loss: 2.9559 | LR: 1.88e-04 | 1282.2ms/step | 15.6 samples/s +Step 25830/44000 | Loss: 2.9577 | LR: 1.88e-04 | 1282.4ms/step | 15.6 samples/s +Step 25840/44000 | Loss: 2.9683 | LR: 1.88e-04 | 1284.4ms/step | 15.6 samples/s +Step 25850/44000 | Loss: 2.9323 | LR: 1.88e-04 | 1279.8ms/step | 15.6 samples/s +Step 25860/44000 | Loss: 2.9292 | LR: 1.87e-04 | 1281.3ms/step | 15.6 samples/s +Step 25870/44000 | Loss: 2.9330 | LR: 1.87e-04 | 1282.2ms/step | 15.6 samples/s +Step 25880/44000 | Loss: 2.9762 | LR: 1.87e-04 | 1285.0ms/step | 15.6 samples/s +Step 25890/44000 | Loss: 2.9556 | LR: 1.87e-04 | 1281.3ms/step | 15.6 samples/s +Step 25900/44000 | Loss: 2.9590 | LR: 1.87e-04 | 1283.7ms/step | 15.6 samples/s +Step 25910/44000 | Loss: 2.9452 | LR: 1.87e-04 | 1283.7ms/step | 15.6 samples/s +Step 25920/44000 | Loss: 2.9422 | LR: 1.86e-04 | 1283.9ms/step | 15.6 samples/s +Step 25930/44000 | Loss: 2.9724 | LR: 1.86e-04 | 1282.3ms/step | 15.6 samples/s +Step 25940/44000 | Loss: 2.9321 | LR: 1.86e-04 | 1282.4ms/step | 15.6 samples/s +Step 25950/44000 | Loss: 2.9332 | LR: 1.86e-04 | 1281.6ms/step | 15.6 samples/s +Step 25960/44000 | Loss: 2.9512 | LR: 1.86e-04 | 1281.6ms/step | 15.6 samples/s +Step 25970/44000 | Loss: 2.9283 | LR: 1.85e-04 | 1283.8ms/step | 15.6 samples/s +Step 25980/44000 | Loss: 2.9712 | LR: 1.85e-04 | 1282.2ms/step | 15.6 samples/s +Step 25990/44000 | Loss: 2.9650 | LR: 1.85e-04 | 1283.2ms/step | 15.6 samples/s +Step 26000/44000 | Loss: 2.9444 | LR: 1.85e-04 | 1284.4ms/step | 15.6 samples/s +Validation at step 26000: Loss = 2.9699, Perplexity = 19.49 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-26000 +Step 26010/44000 | Loss: 2.9585 | LR: 1.85e-04 | 3712.9ms/step | 5.4 samples/s +Step 26020/44000 | Loss: 2.9572 | LR: 1.85e-04 | 1277.0ms/step | 15.7 samples/s +Step 26030/44000 | Loss: 2.9433 | LR: 1.84e-04 | 1279.3ms/step | 15.6 samples/s +Step 26040/44000 | Loss: 2.9316 | LR: 1.84e-04 | 1284.4ms/step | 15.6 samples/s +Step 26050/44000 | Loss: 2.9683 | LR: 1.84e-04 | 1285.0ms/step | 15.6 samples/s +Step 26060/44000 | Loss: 2.9181 | LR: 1.84e-04 | 1281.6ms/step | 15.6 samples/s +Step 26070/44000 | Loss: 2.9483 | LR: 1.84e-04 | 1281.6ms/step | 15.6 samples/s +Step 26080/44000 | Loss: 2.9326 | LR: 1.84e-04 | 1282.6ms/step | 15.6 samples/s +Step 26090/44000 | Loss: 2.9471 | LR: 1.83e-04 | 1284.6ms/step | 15.6 samples/s +Step 26100/44000 | Loss: 2.9479 | LR: 1.83e-04 | 1281.6ms/step | 15.6 samples/s +Step 26110/44000 | Loss: 2.9665 | LR: 1.83e-04 | 1277.7ms/step | 15.7 samples/s +Step 26120/44000 | Loss: 2.9473 | LR: 1.83e-04 | 1276.1ms/step | 15.7 samples/s +Step 26130/44000 | Loss: 2.9668 | LR: 1.83e-04 | 1278.4ms/step | 15.6 samples/s +Step 26140/44000 | Loss: 2.9718 | LR: 1.82e-04 | 1275.6ms/step | 15.7 samples/s +Step 26150/44000 | Loss: 2.9624 | LR: 1.82e-04 | 1278.4ms/step | 15.6 samples/s +Step 26160/44000 | Loss: 2.9479 | LR: 1.82e-04 | 1278.7ms/step | 15.6 samples/s +Step 26170/44000 | Loss: 2.9484 | LR: 1.82e-04 | 1278.6ms/step | 15.6 samples/s +Step 26180/44000 | Loss: 2.9648 | LR: 1.82e-04 | 1284.0ms/step | 15.6 samples/s +Step 26190/44000 | Loss: 2.9353 | LR: 1.82e-04 | 1282.7ms/step | 15.6 samples/s +Step 26200/44000 | Loss: 2.9184 | LR: 1.81e-04 | 1279.4ms/step | 15.6 samples/s +Step 26210/44000 | Loss: 2.9814 | LR: 1.81e-04 | 1279.6ms/step | 15.6 samples/s +Step 26220/44000 | Loss: 2.9555 | LR: 1.81e-04 | 1281.9ms/step | 15.6 samples/s +Step 26230/44000 | Loss: 2.9644 | LR: 1.81e-04 | 1281.4ms/step | 15.6 samples/s +Step 26240/44000 | Loss: 2.9566 | LR: 1.81e-04 | 1281.9ms/step | 15.6 samples/s +Step 26250/44000 | Loss: 2.9643 | LR: 1.81e-04 | 1284.7ms/step | 15.6 samples/s +Validation at step 26250: Loss = 2.9686, Perplexity = 19.46 +Step 26260/44000 | Loss: 2.9473 | LR: 1.80e-04 | 3433.2ms/step | 5.8 samples/s +Step 26270/44000 | Loss: 2.9443 | LR: 1.80e-04 | 1280.9ms/step | 15.6 samples/s +Step 26280/44000 | Loss: 2.9636 | LR: 1.80e-04 | 1280.9ms/step | 15.6 samples/s +Step 26290/44000 | Loss: 2.9786 | LR: 1.80e-04 | 1281.0ms/step | 15.6 samples/s +Step 26300/44000 | Loss: 2.9602 | LR: 1.80e-04 | 1280.6ms/step | 15.6 samples/s +Step 26310/44000 | Loss: 2.9455 | LR: 1.80e-04 | 1281.3ms/step | 15.6 samples/s +Step 26320/44000 | Loss: 2.9644 | LR: 1.79e-04 | 1276.6ms/step | 15.7 samples/s +Step 26330/44000 | Loss: 2.9943 | LR: 1.79e-04 | 1279.9ms/step | 15.6 samples/s +Step 26340/44000 | Loss: 2.9500 | LR: 1.79e-04 | 1281.8ms/step | 15.6 samples/s +Step 26350/44000 | Loss: 2.9661 | LR: 1.79e-04 | 1283.2ms/step | 15.6 samples/s +Step 26360/44000 | Loss: 2.9407 | LR: 1.79e-04 | 1284.7ms/step | 15.6 samples/s +Step 26370/44000 | Loss: 2.9670 | LR: 1.78e-04 | 1281.3ms/step | 15.6 samples/s +Step 26380/44000 | Loss: 2.9707 | LR: 1.78e-04 | 1279.4ms/step | 15.6 samples/s +Step 26390/44000 | Loss: 2.9705 | LR: 1.78e-04 | 1278.6ms/step | 15.6 samples/s +Step 26400/44000 | Loss: 2.9696 | LR: 1.78e-04 | 1281.8ms/step | 15.6 samples/s +Step 26410/44000 | Loss: 2.9360 | LR: 1.78e-04 | 1283.0ms/step | 15.6 samples/s +Step 26420/44000 | Loss: 2.9305 | LR: 1.78e-04 | 1280.2ms/step | 15.6 samples/s +Step 26430/44000 | Loss: 2.9731 | LR: 1.77e-04 | 1276.9ms/step | 15.7 samples/s +Step 26440/44000 | Loss: 2.9299 | LR: 1.77e-04 | 1278.9ms/step | 15.6 samples/s +Step 26450/44000 | Loss: 2.9537 | LR: 1.77e-04 | 1281.4ms/step | 15.6 samples/s +Step 26460/44000 | Loss: 2.9560 | LR: 1.77e-04 | 1279.7ms/step | 15.6 samples/s +Step 26470/44000 | Loss: 2.9483 | LR: 1.77e-04 | 1280.9ms/step | 15.6 samples/s +Step 26480/44000 | Loss: 2.9473 | LR: 1.77e-04 | 1278.9ms/step | 15.6 samples/s +Step 26490/44000 | Loss: 2.9833 | LR: 1.76e-04 | 1278.9ms/step | 15.6 samples/s +Step 26500/44000 | Loss: 2.9651 | LR: 1.76e-04 | 1282.7ms/step | 15.6 samples/s +Validation at step 26500: Loss = 2.9669, Perplexity = 19.43 +Step 26510/44000 | Loss: 2.9770 | LR: 1.76e-04 | 3441.2ms/step | 5.8 samples/s +Step 26520/44000 | Loss: 2.9538 | LR: 1.76e-04 | 1274.3ms/step | 15.7 samples/s +Step 26530/44000 | Loss: 2.9567 | LR: 1.76e-04 | 1274.7ms/step | 15.7 samples/s +Step 26540/44000 | Loss: 2.9625 | LR: 1.76e-04 | 1274.0ms/step | 15.7 samples/s +Step 26550/44000 | Loss: 2.9697 | LR: 1.75e-04 | 1278.6ms/step | 15.6 samples/s +Step 26560/44000 | Loss: 3.0004 | LR: 1.75e-04 | 1279.0ms/step | 15.6 samples/s +Step 26570/44000 | Loss: 2.9273 | LR: 1.75e-04 | 1283.3ms/step | 15.6 samples/s +Step 26580/44000 | Loss: 2.9776 | LR: 1.75e-04 | 1288.5ms/step | 15.5 samples/s +Step 26590/44000 | Loss: 2.9820 | LR: 1.75e-04 | 1280.6ms/step | 15.6 samples/s +Step 26600/44000 | Loss: 2.9543 | LR: 1.74e-04 | 1281.7ms/step | 15.6 samples/s +Step 26610/44000 | Loss: 2.9601 | LR: 1.74e-04 | 1283.3ms/step | 15.6 samples/s +Step 26620/44000 | Loss: 2.9389 | LR: 1.74e-04 | 1285.8ms/step | 15.6 samples/s +Step 26630/44000 | Loss: 2.9477 | LR: 1.74e-04 | 1280.3ms/step | 15.6 samples/s +Step 26640/44000 | Loss: 2.9415 | LR: 1.74e-04 | 1283.1ms/step | 15.6 samples/s +Step 26650/44000 | Loss: 2.9739 | LR: 1.74e-04 | 1286.6ms/step | 15.5 samples/s +Step 26660/44000 | Loss: 2.9634 | LR: 1.73e-04 | 1285.4ms/step | 15.6 samples/s +Step 26670/44000 | Loss: 2.9500 | LR: 1.73e-04 | 1282.7ms/step | 15.6 samples/s +Step 26680/44000 | Loss: 2.9638 | LR: 1.73e-04 | 1277.3ms/step | 15.7 samples/s +Step 26690/44000 | Loss: 2.9729 | LR: 1.73e-04 | 1284.0ms/step | 15.6 samples/s +Step 26700/44000 | Loss: 2.9683 | LR: 1.73e-04 | 1279.3ms/step | 15.6 samples/s +Step 26710/44000 | Loss: 2.9374 | LR: 1.73e-04 | 1274.1ms/step | 15.7 samples/s +Step 26720/44000 | Loss: 2.9459 | LR: 1.72e-04 | 1275.0ms/step | 15.7 samples/s +Step 26730/44000 | Loss: 2.9333 | LR: 1.72e-04 | 1277.7ms/step | 15.7 samples/s +Step 26740/44000 | Loss: 2.9558 | LR: 1.72e-04 | 1280.7ms/step | 15.6 samples/s +Step 26750/44000 | Loss: 2.9942 | LR: 1.72e-04 | 1275.5ms/step | 15.7 samples/s +Validation at step 26750: Loss = 2.9657, Perplexity = 19.41 +Step 26760/44000 | Loss: 2.9461 | LR: 1.72e-04 | 3428.7ms/step | 5.8 samples/s +Step 26770/44000 | Loss: 2.9236 | LR: 1.72e-04 | 1280.6ms/step | 15.6 samples/s +Step 26780/44000 | Loss: 2.9363 | LR: 1.71e-04 | 1281.2ms/step | 15.6 samples/s +Step 26790/44000 | Loss: 2.9488 | LR: 1.71e-04 | 1281.2ms/step | 15.6 samples/s +Step 26800/44000 | Loss: 2.9600 | LR: 1.71e-04 | 1284.8ms/step | 15.6 samples/s +Step 26810/44000 | Loss: 2.9602 | LR: 1.71e-04 | 1286.5ms/step | 15.5 samples/s +Step 26820/44000 | Loss: 2.9769 | LR: 1.71e-04 | 1283.7ms/step | 15.6 samples/s +Step 26830/44000 | Loss: 2.9547 | LR: 1.71e-04 | 1282.5ms/step | 15.6 samples/s +Step 26840/44000 | Loss: 2.9747 | LR: 1.70e-04 | 1284.1ms/step | 15.6 samples/s +Step 26850/44000 | Loss: 2.9519 | LR: 1.70e-04 | 1285.3ms/step | 15.6 samples/s +Step 26860/44000 | Loss: 2.9364 | LR: 1.70e-04 | 1284.3ms/step | 15.6 samples/s +Step 26870/44000 | Loss: 2.9633 | LR: 1.70e-04 | 1284.7ms/step | 15.6 samples/s +Step 26880/44000 | Loss: 2.9284 | LR: 1.70e-04 | 1283.8ms/step | 15.6 samples/s +Step 26890/44000 | Loss: 2.9741 | LR: 1.69e-04 | 1278.0ms/step | 15.6 samples/s +Step 26900/44000 | Loss: 2.9242 | LR: 1.69e-04 | 1278.8ms/step | 15.6 samples/s +Step 26910/44000 | Loss: 2.9675 | LR: 1.69e-04 | 1274.3ms/step | 15.7 samples/s +Step 26920/44000 | Loss: 2.9414 | LR: 1.69e-04 | 1278.6ms/step | 15.6 samples/s +Step 26930/44000 | Loss: 2.9428 | LR: 1.69e-04 | 1285.2ms/step | 15.6 samples/s +Step 26940/44000 | Loss: 2.9559 | LR: 1.69e-04 | 1284.4ms/step | 15.6 samples/s +Step 26950/44000 | Loss: 2.9594 | LR: 1.68e-04 | 1284.1ms/step | 15.6 samples/s +Step 26960/44000 | Loss: 2.9415 | LR: 1.68e-04 | 1277.9ms/step | 15.7 samples/s +Step 26970/44000 | Loss: 2.9450 | LR: 1.68e-04 | 1274.1ms/step | 15.7 samples/s +Step 26980/44000 | Loss: 2.9770 | LR: 1.68e-04 | 1280.1ms/step | 15.6 samples/s +Step 26990/44000 | Loss: 2.9396 | LR: 1.68e-04 | 1280.8ms/step | 15.6 samples/s +Step 27000/44000 | Loss: 2.9524 | LR: 1.68e-04 | 1280.3ms/step | 15.6 samples/s +Validation at step 27000: Loss = 2.9642, Perplexity = 19.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-27000 +Step 27010/44000 | Loss: 2.9688 | LR: 1.67e-04 | 3713.3ms/step | 5.4 samples/s +Step 27020/44000 | Loss: 2.9491 | LR: 1.67e-04 | 1285.7ms/step | 15.6 samples/s +Step 27030/44000 | Loss: 2.9682 | LR: 1.67e-04 | 1275.8ms/step | 15.7 samples/s +Step 27040/44000 | Loss: 2.9556 | LR: 1.67e-04 | 1277.4ms/step | 15.7 samples/s +Step 27050/44000 | Loss: 2.9656 | LR: 1.67e-04 | 1282.0ms/step | 15.6 samples/s +Step 27060/44000 | Loss: 2.9234 | LR: 1.67e-04 | 1282.0ms/step | 15.6 samples/s +Step 27070/44000 | Loss: 2.9603 | LR: 1.66e-04 | 1284.0ms/step | 15.6 samples/s +Step 27080/44000 | Loss: 2.9604 | LR: 1.66e-04 | 1284.8ms/step | 15.6 samples/s +Step 27090/44000 | Loss: 2.9475 | LR: 1.66e-04 | 1279.2ms/step | 15.6 samples/s +Step 27100/44000 | Loss: 2.9345 | LR: 1.66e-04 | 1277.4ms/step | 15.7 samples/s +Step 27110/44000 | Loss: 2.9287 | LR: 1.66e-04 | 1282.1ms/step | 15.6 samples/s +Step 27120/44000 | Loss: 2.9355 | LR: 1.66e-04 | 1281.7ms/step | 15.6 samples/s +Step 27130/44000 | Loss: 2.9296 | LR: 1.65e-04 | 1280.4ms/step | 15.6 samples/s +Step 27140/44000 | Loss: 2.9438 | LR: 1.65e-04 | 1281.1ms/step | 15.6 samples/s +Step 27150/44000 | Loss: 2.9621 | LR: 1.65e-04 | 1282.5ms/step | 15.6 samples/s +Step 27160/44000 | Loss: 2.9277 | LR: 1.65e-04 | 1281.6ms/step | 15.6 samples/s +Step 27170/44000 | Loss: 2.9553 | LR: 1.65e-04 | 1281.7ms/step | 15.6 samples/s +Step 27180/44000 | Loss: 2.9628 | LR: 1.65e-04 | 1284.0ms/step | 15.6 samples/s +Step 27190/44000 | Loss: 2.9588 | LR: 1.64e-04 | 1281.4ms/step | 15.6 samples/s +Step 27200/44000 | Loss: 2.9659 | LR: 1.64e-04 | 1281.0ms/step | 15.6 samples/s +Step 27210/44000 | Loss: 2.9497 | LR: 1.64e-04 | 1281.3ms/step | 15.6 samples/s +Step 27220/44000 | Loss: 2.9331 | LR: 1.64e-04 | 1285.1ms/step | 15.6 samples/s +Step 27230/44000 | Loss: 2.9263 | LR: 1.64e-04 | 1283.0ms/step | 15.6 samples/s +Step 27240/44000 | Loss: 2.9589 | LR: 1.63e-04 | 1280.7ms/step | 15.6 samples/s +Step 27250/44000 | Loss: 2.9285 | LR: 1.63e-04 | 1282.1ms/step | 15.6 samples/s +Validation at step 27250: Loss = 2.9629, Perplexity = 19.35 +Step 27260/44000 | Loss: 2.9674 | LR: 1.63e-04 | 3433.8ms/step | 5.8 samples/s +Step 27270/44000 | Loss: 2.9792 | LR: 1.63e-04 | 1280.9ms/step | 15.6 samples/s +Step 27280/44000 | Loss: 2.9698 | LR: 1.63e-04 | 1282.4ms/step | 15.6 samples/s +Step 27290/44000 | Loss: 2.9389 | LR: 1.63e-04 | 1285.5ms/step | 15.6 samples/s +Step 27300/44000 | Loss: 2.9272 | LR: 1.62e-04 | 1285.9ms/step | 15.6 samples/s +Step 27310/44000 | Loss: 2.9593 | LR: 1.62e-04 | 1287.0ms/step | 15.5 samples/s +Step 27320/44000 | Loss: 2.9659 | LR: 1.62e-04 | 1286.2ms/step | 15.5 samples/s +Step 27330/44000 | Loss: 2.9683 | LR: 1.62e-04 | 1282.3ms/step | 15.6 samples/s +Step 27340/44000 | Loss: 2.9250 | LR: 1.62e-04 | 1277.1ms/step | 15.7 samples/s +Step 27350/44000 | Loss: 2.9233 | LR: 1.62e-04 | 1280.2ms/step | 15.6 samples/s +Step 27360/44000 | Loss: 2.9303 | LR: 1.61e-04 | 1274.3ms/step | 15.7 samples/s +Step 27370/44000 | Loss: 2.9366 | LR: 1.61e-04 | 1276.4ms/step | 15.7 samples/s +Step 27380/44000 | Loss: 2.9815 | LR: 1.61e-04 | 1278.4ms/step | 15.6 samples/s +Step 27390/44000 | Loss: 2.9395 | LR: 1.61e-04 | 1278.1ms/step | 15.6 samples/s +Step 27400/44000 | Loss: 2.9664 | LR: 1.61e-04 | 1280.2ms/step | 15.6 samples/s +Step 27410/44000 | Loss: 2.9482 | LR: 1.61e-04 | 1279.2ms/step | 15.6 samples/s +Step 27420/44000 | Loss: 2.9423 | LR: 1.60e-04 | 1278.2ms/step | 15.6 samples/s +Step 27430/44000 | Loss: 2.9650 | LR: 1.60e-04 | 1272.0ms/step | 15.7 samples/s +Step 27440/44000 | Loss: 2.9548 | LR: 1.60e-04 | 1279.3ms/step | 15.6 samples/s +Step 27450/44000 | Loss: 2.9224 | LR: 1.60e-04 | 1278.9ms/step | 15.6 samples/s +Step 27460/44000 | Loss: 2.9433 | LR: 1.60e-04 | 1283.2ms/step | 15.6 samples/s +Step 27470/44000 | Loss: 2.9736 | LR: 1.60e-04 | 1284.7ms/step | 15.6 samples/s +Step 27480/44000 | Loss: 2.9462 | LR: 1.59e-04 | 1284.8ms/step | 15.6 samples/s +Step 27490/44000 | Loss: 2.9644 | LR: 1.59e-04 | 1284.9ms/step | 15.6 samples/s +Step 27500/44000 | Loss: 2.9531 | LR: 1.59e-04 | 1283.8ms/step | 15.6 samples/s +Validation at step 27500: Loss = 2.9619, Perplexity = 19.34 +Step 27510/44000 | Loss: 2.9873 | LR: 1.59e-04 | 3445.9ms/step | 5.8 samples/s +Step 27520/44000 | Loss: 2.9454 | LR: 1.59e-04 | 1287.9ms/step | 15.5 samples/s +Step 27530/44000 | Loss: 2.9584 | LR: 1.59e-04 | 1284.8ms/step | 15.6 samples/s +Step 27540/44000 | Loss: 2.9176 | LR: 1.58e-04 | 1286.4ms/step | 15.5 samples/s +Step 27550/44000 | Loss: 2.9172 | LR: 1.58e-04 | 1283.2ms/step | 15.6 samples/s +Step 27560/44000 | Loss: 2.9331 | LR: 1.58e-04 | 1288.0ms/step | 15.5 samples/s +Step 27570/44000 | Loss: 2.9229 | LR: 1.58e-04 | 1283.8ms/step | 15.6 samples/s +Step 27580/44000 | Loss: 2.9737 | LR: 1.58e-04 | 1285.2ms/step | 15.6 samples/s +Step 27590/44000 | Loss: 2.9398 | LR: 1.58e-04 | 1285.6ms/step | 15.6 samples/s +Step 27600/44000 | Loss: 2.9538 | LR: 1.57e-04 | 1279.3ms/step | 15.6 samples/s +Step 27610/44000 | Loss: 2.9233 | LR: 1.57e-04 | 1282.1ms/step | 15.6 samples/s +Step 27620/44000 | Loss: 2.9560 | LR: 1.57e-04 | 1277.5ms/step | 15.7 samples/s +Step 27630/44000 | Loss: 2.9215 | LR: 1.57e-04 | 1285.2ms/step | 15.6 samples/s +Step 27640/44000 | Loss: 2.9828 | LR: 1.57e-04 | 1284.6ms/step | 15.6 samples/s +Step 27650/44000 | Loss: 3.0029 | LR: 1.57e-04 | 1285.5ms/step | 15.6 samples/s +Step 27660/44000 | Loss: 2.9526 | LR: 1.56e-04 | 1286.2ms/step | 15.5 samples/s +Step 27670/44000 | Loss: 2.9417 | LR: 1.56e-04 | 1287.8ms/step | 15.5 samples/s +Step 27680/44000 | Loss: 2.9579 | LR: 1.56e-04 | 1283.5ms/step | 15.6 samples/s +Step 27690/44000 | Loss: 2.9463 | LR: 1.56e-04 | 1285.8ms/step | 15.6 samples/s +Step 27700/44000 | Loss: 2.9388 | LR: 1.56e-04 | 1282.3ms/step | 15.6 samples/s +Step 27710/44000 | Loss: 2.9570 | LR: 1.56e-04 | 1283.9ms/step | 15.6 samples/s +Step 27720/44000 | Loss: 2.9416 | LR: 1.55e-04 | 1285.7ms/step | 15.6 samples/s +Step 27730/44000 | Loss: 2.9446 | LR: 1.55e-04 | 1284.5ms/step | 15.6 samples/s +Step 27740/44000 | Loss: 2.9416 | LR: 1.55e-04 | 1282.5ms/step | 15.6 samples/s +Step 27750/44000 | Loss: 2.9408 | LR: 1.55e-04 | 1282.8ms/step | 15.6 samples/s +Validation at step 27750: Loss = 2.9605, Perplexity = 19.31 +Step 27760/44000 | Loss: 2.9455 | LR: 1.55e-04 | 3432.7ms/step | 5.8 samples/s +Step 27770/44000 | Loss: 2.9424 | LR: 1.55e-04 | 1278.4ms/step | 15.6 samples/s +Step 27780/44000 | Loss: 2.9416 | LR: 1.54e-04 | 1281.5ms/step | 15.6 samples/s +Step 27790/44000 | Loss: 2.9046 | LR: 1.54e-04 | 1283.5ms/step | 15.6 samples/s +Step 27800/44000 | Loss: 2.9379 | LR: 1.54e-04 | 1282.6ms/step | 15.6 samples/s +Step 27810/44000 | Loss: 2.9092 | LR: 1.54e-04 | 1280.9ms/step | 15.6 samples/s +Step 27820/44000 | Loss: 2.9330 | LR: 1.54e-04 | 1278.7ms/step | 15.6 samples/s +Step 27830/44000 | Loss: 2.9503 | LR: 1.54e-04 | 1281.5ms/step | 15.6 samples/s +Step 27840/44000 | Loss: 2.9636 | LR: 1.53e-04 | 1276.1ms/step | 15.7 samples/s +Step 27850/44000 | Loss: 2.9309 | LR: 1.53e-04 | 1280.7ms/step | 15.6 samples/s +Step 27860/44000 | Loss: 2.9395 | LR: 1.53e-04 | 1281.7ms/step | 15.6 samples/s +Step 27870/44000 | Loss: 2.9709 | LR: 1.53e-04 | 1281.6ms/step | 15.6 samples/s +Step 27880/44000 | Loss: 2.9413 | LR: 1.53e-04 | 1282.7ms/step | 15.6 samples/s +Step 27890/44000 | Loss: 2.9212 | LR: 1.53e-04 | 1281.5ms/step | 15.6 samples/s +Step 27900/44000 | Loss: 2.9273 | LR: 1.52e-04 | 1281.5ms/step | 15.6 samples/s +Step 27910/44000 | Loss: 2.9398 | LR: 1.52e-04 | 1281.5ms/step | 15.6 samples/s +Step 27920/44000 | Loss: 2.9775 | LR: 1.52e-04 | 1283.9ms/step | 15.6 samples/s +Step 27930/44000 | Loss: 2.9545 | LR: 1.52e-04 | 1288.6ms/step | 15.5 samples/s +Step 27940/44000 | Loss: 2.9470 | LR: 1.52e-04 | 1294.1ms/step | 15.5 samples/s +Step 27950/44000 | Loss: 2.9668 | LR: 1.52e-04 | 1284.9ms/step | 15.6 samples/s +Step 27960/44000 | Loss: 2.9479 | LR: 1.51e-04 | 1287.2ms/step | 15.5 samples/s +Step 27970/44000 | Loss: 2.9416 | LR: 1.51e-04 | 1282.1ms/step | 15.6 samples/s +Step 27980/44000 | Loss: 2.9484 | LR: 1.51e-04 | 1283.3ms/step | 15.6 samples/s +Step 27990/44000 | Loss: 2.9404 | LR: 1.51e-04 | 1284.0ms/step | 15.6 samples/s +Step 28000/44000 | Loss: 2.9434 | LR: 1.51e-04 | 1282.7ms/step | 15.6 samples/s +Validation at step 28000: Loss = 2.9590, Perplexity = 19.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-28000 +Step 28010/44000 | Loss: 2.9438 | LR: 1.51e-04 | 3724.8ms/step | 5.4 samples/s +Step 28020/44000 | Loss: 2.9818 | LR: 1.50e-04 | 1278.6ms/step | 15.6 samples/s +Step 28030/44000 | Loss: 2.9379 | LR: 1.50e-04 | 1276.9ms/step | 15.7 samples/s +Step 28040/44000 | Loss: 2.9590 | LR: 1.50e-04 | 1282.0ms/step | 15.6 samples/s +Step 28050/44000 | Loss: 2.9622 | LR: 1.50e-04 | 1281.7ms/step | 15.6 samples/s +Step 28060/44000 | Loss: 2.9683 | LR: 1.50e-04 | 1281.9ms/step | 15.6 samples/s +Step 28070/44000 | Loss: 2.9404 | LR: 1.50e-04 | 1282.4ms/step | 15.6 samples/s +Step 28080/44000 | Loss: 2.9239 | LR: 1.49e-04 | 1283.8ms/step | 15.6 samples/s +Step 28090/44000 | Loss: 2.9500 | LR: 1.49e-04 | 1284.2ms/step | 15.6 samples/s +Step 28100/44000 | Loss: 2.9436 | LR: 1.49e-04 | 1283.5ms/step | 15.6 samples/s +Step 28110/44000 | Loss: 2.9227 | LR: 1.49e-04 | 1280.7ms/step | 15.6 samples/s +Step 28120/44000 | Loss: 2.9522 | LR: 1.49e-04 | 1283.1ms/step | 15.6 samples/s +Step 28130/44000 | Loss: 2.9518 | LR: 1.49e-04 | 1283.8ms/step | 15.6 samples/s +Step 28140/44000 | Loss: 2.9462 | LR: 1.48e-04 | 1284.8ms/step | 15.6 samples/s +Step 28150/44000 | Loss: 2.9386 | LR: 1.48e-04 | 1281.4ms/step | 15.6 samples/s +Step 28160/44000 | Loss: 2.9534 | LR: 1.48e-04 | 1278.2ms/step | 15.6 samples/s +Step 28170/44000 | Loss: 2.9276 | LR: 1.48e-04 | 1282.5ms/step | 15.6 samples/s +Step 28180/44000 | Loss: 2.9655 | LR: 1.48e-04 | 1283.2ms/step | 15.6 samples/s +Step 28190/44000 | Loss: 2.9578 | LR: 1.48e-04 | 1277.9ms/step | 15.7 samples/s +Step 28200/44000 | Loss: 2.9673 | LR: 1.47e-04 | 1278.2ms/step | 15.6 samples/s +Step 28210/44000 | Loss: 2.9129 | LR: 1.47e-04 | 1281.7ms/step | 15.6 samples/s +Step 28220/44000 | Loss: 2.9298 | LR: 1.47e-04 | 1282.7ms/step | 15.6 samples/s +Step 28230/44000 | Loss: 2.9523 | LR: 1.47e-04 | 1279.7ms/step | 15.6 samples/s +Step 28240/44000 | Loss: 2.9140 | LR: 1.47e-04 | 1277.6ms/step | 15.7 samples/s +Step 28250/44000 | Loss: 2.9273 | LR: 1.47e-04 | 1276.5ms/step | 15.7 samples/s +Validation at step 28250: Loss = 2.9583, Perplexity = 19.26 +Step 28260/44000 | Loss: 2.9466 | LR: 1.46e-04 | 3425.1ms/step | 5.8 samples/s +Step 28270/44000 | Loss: 2.9313 | LR: 1.46e-04 | 1277.9ms/step | 15.7 samples/s +Step 28280/44000 | Loss: 2.9736 | LR: 1.46e-04 | 1281.5ms/step | 15.6 samples/s +Step 28290/44000 | Loss: 2.9019 | LR: 1.46e-04 | 1279.8ms/step | 15.6 samples/s +Step 28300/44000 | Loss: 2.9430 | LR: 1.46e-04 | 1282.2ms/step | 15.6 samples/s +Step 28310/44000 | Loss: 2.9186 | LR: 1.46e-04 | 1279.4ms/step | 15.6 samples/s +Step 28320/44000 | Loss: 2.9815 | LR: 1.45e-04 | 1279.5ms/step | 15.6 samples/s +Step 28330/44000 | Loss: 2.9427 | LR: 1.45e-04 | 1279.9ms/step | 15.6 samples/s +Step 28340/44000 | Loss: 2.9557 | LR: 1.45e-04 | 1283.6ms/step | 15.6 samples/s +Step 28350/44000 | Loss: 2.9327 | LR: 1.45e-04 | 1279.9ms/step | 15.6 samples/s +Step 28360/44000 | Loss: 2.9416 | LR: 1.45e-04 | 1281.4ms/step | 15.6 samples/s +Step 28370/44000 | Loss: 2.9182 | LR: 1.45e-04 | 1281.6ms/step | 15.6 samples/s +Step 28380/44000 | Loss: 2.9263 | LR: 1.44e-04 | 1282.7ms/step | 15.6 samples/s +Step 28390/44000 | Loss: 2.9408 | LR: 1.44e-04 | 1281.9ms/step | 15.6 samples/s +Step 28400/44000 | Loss: 2.9408 | LR: 1.44e-04 | 1281.8ms/step | 15.6 samples/s +Step 28410/44000 | Loss: 2.9667 | LR: 1.44e-04 | 1281.6ms/step | 15.6 samples/s +Step 28420/44000 | Loss: 2.9876 | LR: 1.44e-04 | 1282.1ms/step | 15.6 samples/s +Step 28430/44000 | Loss: 2.9119 | LR: 1.44e-04 | 1280.2ms/step | 15.6 samples/s +Step 28440/44000 | Loss: 2.9525 | LR: 1.43e-04 | 1277.7ms/step | 15.7 samples/s +Step 28450/44000 | Loss: 2.9489 | LR: 1.43e-04 | 1280.0ms/step | 15.6 samples/s +Step 28460/44000 | Loss: 2.9561 | LR: 1.43e-04 | 1278.9ms/step | 15.6 samples/s +Step 28470/44000 | Loss: 2.9547 | LR: 1.43e-04 | 1280.3ms/step | 15.6 samples/s +Step 28480/44000 | Loss: 2.9345 | LR: 1.43e-04 | 1279.8ms/step | 15.6 samples/s +Step 28490/44000 | Loss: 2.9418 | LR: 1.43e-04 | 1282.3ms/step | 15.6 samples/s +Step 28500/44000 | Loss: 2.9433 | LR: 1.42e-04 | 1283.8ms/step | 15.6 samples/s +Validation at step 28500: Loss = 2.9570, Perplexity = 19.24 +Step 28510/44000 | Loss: 2.9690 | LR: 1.42e-04 | 3430.6ms/step | 5.8 samples/s +Step 28520/44000 | Loss: 2.9467 | LR: 1.42e-04 | 1281.6ms/step | 15.6 samples/s +Step 28530/44000 | Loss: 2.9348 | LR: 1.42e-04 | 1283.3ms/step | 15.6 samples/s +Step 28540/44000 | Loss: 2.9621 | LR: 1.42e-04 | 1282.7ms/step | 15.6 samples/s +Step 28550/44000 | Loss: 2.9800 | LR: 1.42e-04 | 1285.0ms/step | 15.6 samples/s +Step 28560/44000 | Loss: 2.9296 | LR: 1.41e-04 | 1283.6ms/step | 15.6 samples/s +Step 28570/44000 | Loss: 2.9213 | LR: 1.41e-04 | 1283.8ms/step | 15.6 samples/s +Step 28580/44000 | Loss: 2.9541 | LR: 1.41e-04 | 1280.2ms/step | 15.6 samples/s +Step 28590/44000 | Loss: 2.9701 | LR: 1.41e-04 | 1280.9ms/step | 15.6 samples/s +Step 28600/44000 | Loss: 2.9712 | LR: 1.41e-04 | 1281.3ms/step | 15.6 samples/s +Step 28610/44000 | Loss: 2.9345 | LR: 1.41e-04 | 1280.4ms/step | 15.6 samples/s +Step 28620/44000 | Loss: 2.9200 | LR: 1.40e-04 | 1282.7ms/step | 15.6 samples/s +Step 28630/44000 | Loss: 2.9293 | LR: 1.40e-04 | 1285.8ms/step | 15.6 samples/s +Step 28640/44000 | Loss: 2.9124 | LR: 1.40e-04 | 1283.4ms/step | 15.6 samples/s +Step 28650/44000 | Loss: 2.9353 | LR: 1.40e-04 | 1282.1ms/step | 15.6 samples/s +Step 28660/44000 | Loss: 2.9331 | LR: 1.40e-04 | 1283.6ms/step | 15.6 samples/s +Step 28670/44000 | Loss: 2.9020 | LR: 1.40e-04 | 1281.9ms/step | 15.6 samples/s +Step 28680/44000 | Loss: 2.9529 | LR: 1.39e-04 | 1282.6ms/step | 15.6 samples/s +Step 28690/44000 | Loss: 2.9478 | LR: 1.39e-04 | 1283.4ms/step | 15.6 samples/s +Step 28700/44000 | Loss: 2.9509 | LR: 1.39e-04 | 1280.8ms/step | 15.6 samples/s +Step 28710/44000 | Loss: 2.9563 | LR: 1.39e-04 | 1282.0ms/step | 15.6 samples/s +Step 28720/44000 | Loss: 2.9311 | LR: 1.39e-04 | 1285.6ms/step | 15.6 samples/s +Step 28730/44000 | Loss: 2.9333 | LR: 1.39e-04 | 1285.5ms/step | 15.6 samples/s +Step 28740/44000 | Loss: 2.9564 | LR: 1.38e-04 | 1284.5ms/step | 15.6 samples/s +Step 28750/44000 | Loss: 2.9204 | LR: 1.38e-04 | 1285.9ms/step | 15.6 samples/s +Validation at step 28750: Loss = 2.9561, Perplexity = 19.22 +Step 28760/44000 | Loss: 2.9471 | LR: 1.38e-04 | 3434.9ms/step | 5.8 samples/s +Step 28770/44000 | Loss: 2.9147 | LR: 1.38e-04 | 1278.3ms/step | 15.6 samples/s +Step 28780/44000 | Loss: 2.9494 | LR: 1.38e-04 | 1277.5ms/step | 15.7 samples/s +Step 28790/44000 | Loss: 2.9580 | LR: 1.38e-04 | 1278.9ms/step | 15.6 samples/s +Step 28800/44000 | Loss: 2.9611 | LR: 1.38e-04 | 1285.5ms/step | 15.6 samples/s +Step 28810/44000 | Loss: 2.9461 | LR: 1.37e-04 | 1280.1ms/step | 15.6 samples/s +Step 28820/44000 | Loss: 2.9204 | LR: 1.37e-04 | 1284.4ms/step | 15.6 samples/s +Step 28830/44000 | Loss: 2.9530 | LR: 1.37e-04 | 1282.1ms/step | 15.6 samples/s +Step 28840/44000 | Loss: 2.9482 | LR: 1.37e-04 | 1279.2ms/step | 15.6 samples/s +Step 28850/44000 | Loss: 2.9255 | LR: 1.37e-04 | 1284.3ms/step | 15.6 samples/s +Step 28860/44000 | Loss: 2.9561 | LR: 1.37e-04 | 1286.0ms/step | 15.6 samples/s +Step 28870/44000 | Loss: 2.9609 | LR: 1.36e-04 | 1282.3ms/step | 15.6 samples/s +Step 28880/44000 | Loss: 2.9342 | LR: 1.36e-04 | 1286.1ms/step | 15.6 samples/s +Step 28890/44000 | Loss: 2.9463 | LR: 1.36e-04 | 1284.1ms/step | 15.6 samples/s +Step 28900/44000 | Loss: 2.9410 | LR: 1.36e-04 | 1285.6ms/step | 15.6 samples/s +Step 28910/44000 | Loss: 2.9377 | LR: 1.36e-04 | 1284.1ms/step | 15.6 samples/s +Step 28920/44000 | Loss: 2.9517 | LR: 1.36e-04 | 1284.5ms/step | 15.6 samples/s +Step 28930/44000 | Loss: 2.9419 | LR: 1.35e-04 | 1285.1ms/step | 15.6 samples/s +Step 28940/44000 | Loss: 2.9379 | LR: 1.35e-04 | 1284.4ms/step | 15.6 samples/s +Step 28950/44000 | Loss: 2.9818 | LR: 1.35e-04 | 1286.7ms/step | 15.5 samples/s +Step 28960/44000 | Loss: 2.9408 | LR: 1.35e-04 | 1285.1ms/step | 15.6 samples/s +Step 28970/44000 | Loss: 2.9633 | LR: 1.35e-04 | 1285.4ms/step | 15.6 samples/s +Step 28980/44000 | Loss: 2.9479 | LR: 1.35e-04 | 1282.1ms/step | 15.6 samples/s +Step 28990/44000 | Loss: 2.9377 | LR: 1.34e-04 | 1283.7ms/step | 15.6 samples/s +Step 29000/44000 | Loss: 2.9370 | LR: 1.34e-04 | 1281.7ms/step | 15.6 samples/s +Validation at step 29000: Loss = 2.9550, Perplexity = 19.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-29000 +Step 29010/44000 | Loss: 2.9313 | LR: 1.34e-04 | 3691.7ms/step | 5.4 samples/s +Step 29020/44000 | Loss: 2.9432 | LR: 1.34e-04 | 1283.6ms/step | 15.6 samples/s +Step 29030/44000 | Loss: 2.9298 | LR: 1.34e-04 | 1283.8ms/step | 15.6 samples/s +Step 29040/44000 | Loss: 2.9160 | LR: 1.34e-04 | 1282.7ms/step | 15.6 samples/s +Step 29050/44000 | Loss: 2.9218 | LR: 1.33e-04 | 1282.5ms/step | 15.6 samples/s +Step 29060/44000 | Loss: 2.9508 | LR: 1.33e-04 | 1283.6ms/step | 15.6 samples/s +Step 29070/44000 | Loss: 2.9461 | LR: 1.33e-04 | 1281.8ms/step | 15.6 samples/s +Step 29080/44000 | Loss: 2.9159 | LR: 1.33e-04 | 1282.4ms/step | 15.6 samples/s +Step 29090/44000 | Loss: 2.9500 | LR: 1.33e-04 | 1281.4ms/step | 15.6 samples/s +Step 29100/44000 | Loss: 2.9585 | LR: 1.33e-04 | 1283.2ms/step | 15.6 samples/s +Step 29110/44000 | Loss: 2.9561 | LR: 1.33e-04 | 1278.1ms/step | 15.6 samples/s +Step 29120/44000 | Loss: 2.9242 | LR: 1.32e-04 | 1278.7ms/step | 15.6 samples/s +Step 29130/44000 | Loss: 2.9513 | LR: 1.32e-04 | 1275.7ms/step | 15.7 samples/s +Step 29140/44000 | Loss: 2.9445 | LR: 1.32e-04 | 1279.4ms/step | 15.6 samples/s +Step 29150/44000 | Loss: 2.9630 | LR: 1.32e-04 | 1284.7ms/step | 15.6 samples/s +Step 29160/44000 | Loss: 2.9386 | LR: 1.32e-04 | 1281.3ms/step | 15.6 samples/s +Step 29170/44000 | Loss: 2.9161 | LR: 1.32e-04 | 1284.8ms/step | 15.6 samples/s +Step 29180/44000 | Loss: 2.9176 | LR: 1.31e-04 | 1282.8ms/step | 15.6 samples/s +Step 29190/44000 | Loss: 2.9323 | LR: 1.31e-04 | 1280.2ms/step | 15.6 samples/s +Step 29200/44000 | Loss: 2.9397 | LR: 1.31e-04 | 1281.5ms/step | 15.6 samples/s +Step 29210/44000 | Loss: 2.8935 | LR: 1.31e-04 | 1283.4ms/step | 15.6 samples/s +Step 29220/44000 | Loss: 2.9356 | LR: 1.31e-04 | 1283.3ms/step | 15.6 samples/s +Step 29230/44000 | Loss: 2.9261 | LR: 1.31e-04 | 1282.9ms/step | 15.6 samples/s +Step 29240/44000 | Loss: 2.9512 | LR: 1.30e-04 | 1282.1ms/step | 15.6 samples/s +Step 29250/44000 | Loss: 2.9330 | LR: 1.30e-04 | 1283.2ms/step | 15.6 samples/s +Validation at step 29250: Loss = 2.9540, Perplexity = 19.18 +Step 29260/44000 | Loss: 2.9213 | LR: 1.30e-04 | 3436.6ms/step | 5.8 samples/s +Step 29270/44000 | Loss: 2.9396 | LR: 1.30e-04 | 1285.7ms/step | 15.6 samples/s +Step 29280/44000 | Loss: 2.9299 | LR: 1.30e-04 | 1288.3ms/step | 15.5 samples/s +Step 29290/44000 | Loss: 2.9465 | LR: 1.30e-04 | 1284.8ms/step | 15.6 samples/s +Step 29300/44000 | Loss: 2.9508 | LR: 1.29e-04 | 1287.5ms/step | 15.5 samples/s +Step 29310/44000 | Loss: 2.9545 | LR: 1.29e-04 | 1286.5ms/step | 15.5 samples/s +Step 29320/44000 | Loss: 2.9504 | LR: 1.29e-04 | 1282.0ms/step | 15.6 samples/s +Step 29330/44000 | Loss: 2.9549 | LR: 1.29e-04 | 1280.3ms/step | 15.6 samples/s +Step 29340/44000 | Loss: 2.9116 | LR: 1.29e-04 | 1286.3ms/step | 15.5 samples/s +Step 29350/44000 | Loss: 2.9547 | LR: 1.29e-04 | 1284.1ms/step | 15.6 samples/s +Step 29360/44000 | Loss: 2.9595 | LR: 1.29e-04 | 1283.0ms/step | 15.6 samples/s +Step 29370/44000 | Loss: 2.9302 | LR: 1.28e-04 | 1283.5ms/step | 15.6 samples/s +Step 29380/44000 | Loss: 2.9387 | LR: 1.28e-04 | 1285.2ms/step | 15.6 samples/s +Step 29390/44000 | Loss: 2.9578 | LR: 1.28e-04 | 1285.1ms/step | 15.6 samples/s +Step 29400/44000 | Loss: 2.9403 | LR: 1.28e-04 | 1284.7ms/step | 15.6 samples/s +Step 29410/44000 | Loss: 2.9315 | LR: 1.28e-04 | 1283.0ms/step | 15.6 samples/s +Step 29420/44000 | Loss: 2.9712 | LR: 1.28e-04 | 1279.5ms/step | 15.6 samples/s +Step 29430/44000 | Loss: 2.9443 | LR: 1.27e-04 | 1283.4ms/step | 15.6 samples/s +Step 29440/44000 | Loss: 2.9450 | LR: 1.27e-04 | 1282.7ms/step | 15.6 samples/s +Step 29450/44000 | Loss: 2.9635 | LR: 1.27e-04 | 1282.2ms/step | 15.6 samples/s +Step 29460/44000 | Loss: 2.9506 | LR: 1.27e-04 | 1281.2ms/step | 15.6 samples/s +Step 29470/44000 | Loss: 2.9454 | LR: 1.27e-04 | 1282.0ms/step | 15.6 samples/s +Step 29480/44000 | Loss: 2.9441 | LR: 1.27e-04 | 1283.8ms/step | 15.6 samples/s +Step 29490/44000 | Loss: 2.9328 | LR: 1.26e-04 | 1282.0ms/step | 15.6 samples/s +Step 29500/44000 | Loss: 2.9295 | LR: 1.26e-04 | 1283.9ms/step | 15.6 samples/s +Validation at step 29500: Loss = 2.9529, Perplexity = 19.16 +Step 29510/44000 | Loss: 2.9331 | LR: 1.26e-04 | 3430.2ms/step | 5.8 samples/s +Step 29520/44000 | Loss: 2.9459 | LR: 1.26e-04 | 1282.7ms/step | 15.6 samples/s +Step 29530/44000 | Loss: 2.9245 | LR: 1.26e-04 | 1283.4ms/step | 15.6 samples/s +Step 29540/44000 | Loss: 2.9223 | LR: 1.26e-04 | 1283.6ms/step | 15.6 samples/s +Step 29550/44000 | Loss: 2.9302 | LR: 1.26e-04 | 1281.6ms/step | 15.6 samples/s +Step 29560/44000 | Loss: 2.9293 | LR: 1.25e-04 | 1282.9ms/step | 15.6 samples/s +Step 29570/44000 | Loss: 2.9375 | LR: 1.25e-04 | 1282.5ms/step | 15.6 samples/s +Step 29580/44000 | Loss: 2.9262 | LR: 1.25e-04 | 1281.9ms/step | 15.6 samples/s +Step 29590/44000 | Loss: 2.9585 | LR: 1.25e-04 | 1283.1ms/step | 15.6 samples/s +Step 29600/44000 | Loss: 2.9469 | LR: 1.25e-04 | 1282.0ms/step | 15.6 samples/s +Step 29610/44000 | Loss: 2.9393 | LR: 1.25e-04 | 1283.7ms/step | 15.6 samples/s +Step 29620/44000 | Loss: 2.9211 | LR: 1.24e-04 | 1283.7ms/step | 15.6 samples/s +Step 29630/44000 | Loss: 2.9636 | LR: 1.24e-04 | 1282.1ms/step | 15.6 samples/s +Step 29640/44000 | Loss: 2.9337 | LR: 1.24e-04 | 1285.8ms/step | 15.6 samples/s +Step 29650/44000 | Loss: 2.9260 | LR: 1.24e-04 | 1285.8ms/step | 15.6 samples/s +Step 29660/44000 | Loss: 2.9819 | LR: 1.24e-04 | 1283.4ms/step | 15.6 samples/s +Step 29670/44000 | Loss: 2.9246 | LR: 1.24e-04 | 1284.0ms/step | 15.6 samples/s +Step 29680/44000 | Loss: 2.9365 | LR: 1.23e-04 | 1281.4ms/step | 15.6 samples/s +Step 29690/44000 | Loss: 2.9439 | LR: 1.23e-04 | 1280.4ms/step | 15.6 samples/s +Step 29700/44000 | Loss: 2.9471 | LR: 1.23e-04 | 1287.6ms/step | 15.5 samples/s +Step 29710/44000 | Loss: 2.9574 | LR: 1.23e-04 | 1282.5ms/step | 15.6 samples/s +Step 29720/44000 | Loss: 2.8985 | LR: 1.23e-04 | 1282.0ms/step | 15.6 samples/s +Step 29730/44000 | Loss: 2.9472 | LR: 1.23e-04 | 1281.5ms/step | 15.6 samples/s +Step 29740/44000 | Loss: 2.9127 | LR: 1.23e-04 | 1281.3ms/step | 15.6 samples/s +Step 29750/44000 | Loss: 2.9288 | LR: 1.22e-04 | 1282.8ms/step | 15.6 samples/s +Validation at step 29750: Loss = 2.9518, Perplexity = 19.14 +Step 29760/44000 | Loss: 2.9592 | LR: 1.22e-04 | 3429.6ms/step | 5.8 samples/s +Step 29770/44000 | Loss: 2.9303 | LR: 1.22e-04 | 1280.5ms/step | 15.6 samples/s +Step 29780/44000 | Loss: 2.9392 | LR: 1.22e-04 | 1279.5ms/step | 15.6 samples/s +Step 29790/44000 | Loss: 2.9074 | LR: 1.22e-04 | 1279.8ms/step | 15.6 samples/s +Step 29800/44000 | Loss: 2.9165 | LR: 1.22e-04 | 1284.3ms/step | 15.6 samples/s +Step 29810/44000 | Loss: 2.9236 | LR: 1.21e-04 | 1281.8ms/step | 15.6 samples/s +Step 29820/44000 | Loss: 2.9692 | LR: 1.21e-04 | 1280.6ms/step | 15.6 samples/s +Step 29830/44000 | Loss: 2.9562 | LR: 1.21e-04 | 1282.6ms/step | 15.6 samples/s +Step 29840/44000 | Loss: 2.9035 | LR: 1.21e-04 | 1277.6ms/step | 15.7 samples/s +Step 29850/44000 | Loss: 2.9277 | LR: 1.21e-04 | 1281.7ms/step | 15.6 samples/s +Step 29860/44000 | Loss: 2.9442 | LR: 1.21e-04 | 1282.1ms/step | 15.6 samples/s +Step 29870/44000 | Loss: 2.9129 | LR: 1.21e-04 | 1283.3ms/step | 15.6 samples/s +Step 29880/44000 | Loss: 2.9312 | LR: 1.20e-04 | 1278.0ms/step | 15.6 samples/s +Step 29890/44000 | Loss: 2.9373 | LR: 1.20e-04 | 1280.7ms/step | 15.6 samples/s +Step 29900/44000 | Loss: 2.9764 | LR: 1.20e-04 | 1285.4ms/step | 15.6 samples/s +Step 29910/44000 | Loss: 2.9306 | LR: 1.20e-04 | 1287.7ms/step | 15.5 samples/s +Step 29920/44000 | Loss: 2.9230 | LR: 1.20e-04 | 1284.0ms/step | 15.6 samples/s +Step 29930/44000 | Loss: 2.9292 | LR: 1.20e-04 | 1283.6ms/step | 15.6 samples/s +Step 29940/44000 | Loss: 2.9257 | LR: 1.19e-04 | 1282.4ms/step | 15.6 samples/s +Step 29950/44000 | Loss: 2.9353 | LR: 1.19e-04 | 1283.3ms/step | 15.6 samples/s +Step 29960/44000 | Loss: 2.9169 | LR: 1.19e-04 | 1281.6ms/step | 15.6 samples/s +Step 29970/44000 | Loss: 2.9300 | LR: 1.19e-04 | 1285.1ms/step | 15.6 samples/s +Step 29980/44000 | Loss: 2.9271 | LR: 1.19e-04 | 1283.8ms/step | 15.6 samples/s +Step 29990/44000 | Loss: 2.9417 | LR: 1.19e-04 | 1283.1ms/step | 15.6 samples/s +Step 30000/44000 | Loss: 2.9184 | LR: 1.19e-04 | 1283.0ms/step | 15.6 samples/s +Validation at step 30000: Loss = 2.9512, Perplexity = 19.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-30000 +Step 30010/44000 | Loss: 2.9480 | LR: 1.18e-04 | 3721.9ms/step | 5.4 samples/s +Step 30020/44000 | Loss: 2.9585 | LR: 1.18e-04 | 1280.4ms/step | 15.6 samples/s +Step 30030/44000 | Loss: 2.9274 | LR: 1.18e-04 | 1282.0ms/step | 15.6 samples/s +Step 30040/44000 | Loss: 2.9454 | LR: 1.18e-04 | 1281.1ms/step | 15.6 samples/s +Step 30050/44000 | Loss: 2.9187 | LR: 1.18e-04 | 1279.0ms/step | 15.6 samples/s +Step 30060/44000 | Loss: 2.9433 | LR: 1.18e-04 | 1280.5ms/step | 15.6 samples/s +Step 30070/44000 | Loss: 2.8973 | LR: 1.17e-04 | 1279.7ms/step | 15.6 samples/s +Step 30080/44000 | Loss: 2.9689 | LR: 1.17e-04 | 1279.9ms/step | 15.6 samples/s +Step 30090/44000 | Loss: 2.9465 | LR: 1.17e-04 | 1279.8ms/step | 15.6 samples/s +Step 30100/44000 | Loss: 2.9463 | LR: 1.17e-04 | 1282.8ms/step | 15.6 samples/s +Step 30110/44000 | Loss: 2.9332 | LR: 1.17e-04 | 1283.2ms/step | 15.6 samples/s +Step 30120/44000 | Loss: 2.9218 | LR: 1.17e-04 | 1283.2ms/step | 15.6 samples/s +Step 30130/44000 | Loss: 2.9503 | LR: 1.17e-04 | 1285.4ms/step | 15.6 samples/s +Step 30140/44000 | Loss: 2.9410 | LR: 1.16e-04 | 1281.6ms/step | 15.6 samples/s +Step 30150/44000 | Loss: 2.9233 | LR: 1.16e-04 | 1280.2ms/step | 15.6 samples/s +Step 30160/44000 | Loss: 2.9099 | LR: 1.16e-04 | 1280.6ms/step | 15.6 samples/s +Step 30170/44000 | Loss: 2.9128 | LR: 1.16e-04 | 1285.1ms/step | 15.6 samples/s +Step 30180/44000 | Loss: 2.9472 | LR: 1.16e-04 | 1283.6ms/step | 15.6 samples/s +Step 30190/44000 | Loss: 2.9237 | LR: 1.16e-04 | 1280.8ms/step | 15.6 samples/s +Step 30200/44000 | Loss: 2.9325 | LR: 1.15e-04 | 1283.3ms/step | 15.6 samples/s +Step 30210/44000 | Loss: 2.9332 | LR: 1.15e-04 | 1285.8ms/step | 15.6 samples/s +Step 30220/44000 | Loss: 2.9082 | LR: 1.15e-04 | 1279.9ms/step | 15.6 samples/s +Step 30230/44000 | Loss: 2.9531 | LR: 1.15e-04 | 1277.3ms/step | 15.7 samples/s +Step 30240/44000 | Loss: 2.9365 | LR: 1.15e-04 | 1282.4ms/step | 15.6 samples/s +Step 30250/44000 | Loss: 2.9239 | LR: 1.15e-04 | 1286.3ms/step | 15.5 samples/s +Validation at step 30250: Loss = 2.9503, Perplexity = 19.11 +Step 30260/44000 | Loss: 2.9282 | LR: 1.15e-04 | 3437.8ms/step | 5.8 samples/s +Step 30270/44000 | Loss: 2.9169 | LR: 1.14e-04 | 1280.1ms/step | 15.6 samples/s +Step 30280/44000 | Loss: 2.9607 | LR: 1.14e-04 | 1280.7ms/step | 15.6 samples/s +Step 30290/44000 | Loss: 2.9187 | LR: 1.14e-04 | 1278.1ms/step | 15.6 samples/s +Step 30300/44000 | Loss: 2.9016 | LR: 1.14e-04 | 1278.0ms/step | 15.6 samples/s +Step 30310/44000 | Loss: 2.9116 | LR: 1.14e-04 | 1280.5ms/step | 15.6 samples/s +Step 30320/44000 | Loss: 2.9106 | LR: 1.14e-04 | 1280.7ms/step | 15.6 samples/s +Step 30330/44000 | Loss: 2.9287 | LR: 1.13e-04 | 1275.7ms/step | 15.7 samples/s +Step 30340/44000 | Loss: 2.9344 | LR: 1.13e-04 | 1280.7ms/step | 15.6 samples/s +Step 30350/44000 | Loss: 2.8991 | LR: 1.13e-04 | 1281.2ms/step | 15.6 samples/s +Step 30360/44000 | Loss: 2.9344 | LR: 1.13e-04 | 1281.0ms/step | 15.6 samples/s +Step 30370/44000 | Loss: 2.9482 | LR: 1.13e-04 | 1281.2ms/step | 15.6 samples/s +Step 30380/44000 | Loss: 2.9566 | LR: 1.13e-04 | 1280.5ms/step | 15.6 samples/s +Step 30390/44000 | Loss: 2.9280 | LR: 1.13e-04 | 1282.8ms/step | 15.6 samples/s +Step 30400/44000 | Loss: 2.9292 | LR: 1.12e-04 | 1283.9ms/step | 15.6 samples/s +Step 30410/44000 | Loss: 2.9370 | LR: 1.12e-04 | 1281.0ms/step | 15.6 samples/s +Step 30420/44000 | Loss: 2.9328 | LR: 1.12e-04 | 1278.2ms/step | 15.6 samples/s +Step 30430/44000 | Loss: 2.9335 | LR: 1.12e-04 | 1278.0ms/step | 15.6 samples/s +Step 30440/44000 | Loss: 2.9361 | LR: 1.12e-04 | 1279.6ms/step | 15.6 samples/s +Step 30450/44000 | Loss: 2.9064 | LR: 1.12e-04 | 1276.0ms/step | 15.7 samples/s +Step 30460/44000 | Loss: 2.9355 | LR: 1.11e-04 | 1278.7ms/step | 15.6 samples/s +Step 30470/44000 | Loss: 2.9476 | LR: 1.11e-04 | 1279.8ms/step | 15.6 samples/s +Step 30480/44000 | Loss: 2.9360 | LR: 1.11e-04 | 1280.6ms/step | 15.6 samples/s +Step 30490/44000 | Loss: 2.9301 | LR: 1.11e-04 | 1279.5ms/step | 15.6 samples/s +Step 30500/44000 | Loss: 2.9257 | LR: 1.11e-04 | 1281.2ms/step | 15.6 samples/s +Validation at step 30500: Loss = 2.9494, Perplexity = 19.09 +Step 30510/44000 | Loss: 2.9243 | LR: 1.11e-04 | 3431.0ms/step | 5.8 samples/s +Step 30520/44000 | Loss: 2.9294 | LR: 1.11e-04 | 1277.8ms/step | 15.7 samples/s +Step 30530/44000 | Loss: 2.9706 | LR: 1.10e-04 | 1278.7ms/step | 15.6 samples/s +Step 30540/44000 | Loss: 2.9667 | LR: 1.10e-04 | 1276.7ms/step | 15.7 samples/s +Step 30550/44000 | Loss: 2.9387 | LR: 1.10e-04 | 1279.5ms/step | 15.6 samples/s +Step 30560/44000 | Loss: 2.9389 | LR: 1.10e-04 | 1281.3ms/step | 15.6 samples/s +Step 30570/44000 | Loss: 2.9510 | LR: 1.10e-04 | 1285.1ms/step | 15.6 samples/s +Step 30580/44000 | Loss: 2.9464 | LR: 1.10e-04 | 1282.6ms/step | 15.6 samples/s +Step 30590/44000 | Loss: 2.9359 | LR: 1.10e-04 | 1279.3ms/step | 15.6 samples/s +Step 30600/44000 | Loss: 2.9296 | LR: 1.09e-04 | 1280.7ms/step | 15.6 samples/s +Step 30610/44000 | Loss: 2.9329 | LR: 1.09e-04 | 1280.6ms/step | 15.6 samples/s +Step 30620/44000 | Loss: 2.9443 | LR: 1.09e-04 | 1283.5ms/step | 15.6 samples/s +Step 30630/44000 | Loss: 2.9162 | LR: 1.09e-04 | 1280.9ms/step | 15.6 samples/s +Step 30640/44000 | Loss: 2.9453 | LR: 1.09e-04 | 1279.4ms/step | 15.6 samples/s +Step 30650/44000 | Loss: 2.9544 | LR: 1.09e-04 | 1283.3ms/step | 15.6 samples/s +Step 30660/44000 | Loss: 2.9716 | LR: 1.08e-04 | 1285.8ms/step | 15.6 samples/s +Step 30670/44000 | Loss: 2.9591 | LR: 1.08e-04 | 1279.8ms/step | 15.6 samples/s +Step 30680/44000 | Loss: 2.9419 | LR: 1.08e-04 | 1278.6ms/step | 15.6 samples/s +Step 30690/44000 | Loss: 2.9269 | LR: 1.08e-04 | 1279.1ms/step | 15.6 samples/s +Step 30700/44000 | Loss: 2.9471 | LR: 1.08e-04 | 1282.4ms/step | 15.6 samples/s +Step 30710/44000 | Loss: 2.9291 | LR: 1.08e-04 | 1280.6ms/step | 15.6 samples/s +Step 30720/44000 | Loss: 2.9475 | LR: 1.08e-04 | 1281.4ms/step | 15.6 samples/s +Step 30730/44000 | Loss: 2.9295 | LR: 1.07e-04 | 1278.6ms/step | 15.6 samples/s +Step 30740/44000 | Loss: 2.9478 | LR: 1.07e-04 | 1283.3ms/step | 15.6 samples/s +Step 30750/44000 | Loss: 2.9316 | LR: 1.07e-04 | 1282.3ms/step | 15.6 samples/s +Validation at step 30750: Loss = 2.9486, Perplexity = 19.08 +Step 30760/44000 | Loss: 2.9340 | LR: 1.07e-04 | 3431.6ms/step | 5.8 samples/s +Step 30770/44000 | Loss: 2.9769 | LR: 1.07e-04 | 1281.8ms/step | 15.6 samples/s +Step 30780/44000 | Loss: 2.9281 | LR: 1.07e-04 | 1281.0ms/step | 15.6 samples/s +Step 30790/44000 | Loss: 2.9337 | LR: 1.07e-04 | 1280.9ms/step | 15.6 samples/s +Step 30800/44000 | Loss: 2.9418 | LR: 1.06e-04 | 1284.4ms/step | 15.6 samples/s +Step 30810/44000 | Loss: 2.9094 | LR: 1.06e-04 | 1287.4ms/step | 15.5 samples/s +Step 30820/44000 | Loss: 2.9398 | LR: 1.06e-04 | 1285.8ms/step | 15.6 samples/s +Step 30830/44000 | Loss: 2.9792 | LR: 1.06e-04 | 1287.1ms/step | 15.5 samples/s +Step 30840/44000 | Loss: 2.9494 | LR: 1.06e-04 | 1284.3ms/step | 15.6 samples/s +Step 30850/44000 | Loss: 2.9217 | LR: 1.06e-04 | 1285.1ms/step | 15.6 samples/s +Step 30860/44000 | Loss: 2.9508 | LR: 1.05e-04 | 1284.4ms/step | 15.6 samples/s +Step 30870/44000 | Loss: 2.9099 | LR: 1.05e-04 | 1281.9ms/step | 15.6 samples/s +Step 30880/44000 | Loss: 2.9281 | LR: 1.05e-04 | 1282.4ms/step | 15.6 samples/s +Step 30890/44000 | Loss: 2.8829 | LR: 1.05e-04 | 1283.1ms/step | 15.6 samples/s +Step 30900/44000 | Loss: 2.9121 | LR: 1.05e-04 | 1282.9ms/step | 15.6 samples/s +Step 30910/44000 | Loss: 2.9238 | LR: 1.05e-04 | 1282.2ms/step | 15.6 samples/s +Step 30920/44000 | Loss: 2.9400 | LR: 1.05e-04 | 1279.2ms/step | 15.6 samples/s +Step 30930/44000 | Loss: 2.9248 | LR: 1.04e-04 | 1278.1ms/step | 15.6 samples/s +Step 30940/44000 | Loss: 2.9107 | LR: 1.04e-04 | 1278.6ms/step | 15.6 samples/s +Step 30950/44000 | Loss: 2.9350 | LR: 1.04e-04 | 1281.6ms/step | 15.6 samples/s +Step 30960/44000 | Loss: 2.9286 | LR: 1.04e-04 | 1286.2ms/step | 15.5 samples/s +Step 30970/44000 | Loss: 2.9681 | LR: 1.04e-04 | 1284.6ms/step | 15.6 samples/s +Step 30980/44000 | Loss: 2.9145 | LR: 1.04e-04 | 1280.9ms/step | 15.6 samples/s +Step 30990/44000 | Loss: 2.9194 | LR: 1.04e-04 | 1282.6ms/step | 15.6 samples/s +Step 31000/44000 | Loss: 2.9084 | LR: 1.03e-04 | 1284.3ms/step | 15.6 samples/s +Validation at step 31000: Loss = 2.9479, Perplexity = 19.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-31000 +Step 31010/44000 | Loss: 2.9181 | LR: 1.03e-04 | 3694.0ms/step | 5.4 samples/s +Step 31020/44000 | Loss: 2.9457 | LR: 1.03e-04 | 1281.9ms/step | 15.6 samples/s +Step 31030/44000 | Loss: 2.8993 | LR: 1.03e-04 | 1279.9ms/step | 15.6 samples/s +Step 31040/44000 | Loss: 2.9297 | LR: 1.03e-04 | 1277.2ms/step | 15.7 samples/s +Step 31050/44000 | Loss: 2.9405 | LR: 1.03e-04 | 1277.8ms/step | 15.7 samples/s +Step 31060/44000 | Loss: 2.9492 | LR: 1.03e-04 | 1281.7ms/step | 15.6 samples/s +Step 31070/44000 | Loss: 2.9373 | LR: 1.02e-04 | 1281.5ms/step | 15.6 samples/s +Step 31080/44000 | Loss: 2.9278 | LR: 1.02e-04 | 1282.2ms/step | 15.6 samples/s +Step 31090/44000 | Loss: 2.9247 | LR: 1.02e-04 | 1282.7ms/step | 15.6 samples/s +Step 31100/44000 | Loss: 2.9164 | LR: 1.02e-04 | 1281.7ms/step | 15.6 samples/s +Step 31110/44000 | Loss: 2.9191 | LR: 1.02e-04 | 1283.5ms/step | 15.6 samples/s +Step 31120/44000 | Loss: 2.9365 | LR: 1.02e-04 | 1282.3ms/step | 15.6 samples/s +Step 31130/44000 | Loss: 2.9669 | LR: 1.02e-04 | 1283.2ms/step | 15.6 samples/s +Step 31140/44000 | Loss: 2.9190 | LR: 1.01e-04 | 1283.1ms/step | 15.6 samples/s +Step 31150/44000 | Loss: 2.9648 | LR: 1.01e-04 | 1283.0ms/step | 15.6 samples/s +Step 31160/44000 | Loss: 2.9370 | LR: 1.01e-04 | 1283.9ms/step | 15.6 samples/s +Step 31170/44000 | Loss: 2.9295 | LR: 1.01e-04 | 1281.8ms/step | 15.6 samples/s +Step 31180/44000 | Loss: 2.9390 | LR: 1.01e-04 | 1281.7ms/step | 15.6 samples/s +Step 31190/44000 | Loss: 2.9091 | LR: 1.01e-04 | 1280.9ms/step | 15.6 samples/s +Step 31200/44000 | Loss: 2.9221 | LR: 1.00e-04 | 1285.5ms/step | 15.6 samples/s +Step 31210/44000 | Loss: 2.9422 | LR: 1.00e-04 | 1280.6ms/step | 15.6 samples/s +Step 31220/44000 | Loss: 2.9292 | LR: 1.00e-04 | 1282.8ms/step | 15.6 samples/s +Step 31230/44000 | Loss: 2.9635 | LR: 1.00e-04 | 1280.1ms/step | 15.6 samples/s +Step 31240/44000 | Loss: 2.9304 | LR: 9.99e-05 | 1283.8ms/step | 15.6 samples/s +Step 31250/44000 | Loss: 2.9143 | LR: 9.98e-05 | 1277.3ms/step | 15.7 samples/s +Validation at step 31250: Loss = 2.9470, Perplexity = 19.05 +Step 31260/44000 | Loss: 2.9451 | LR: 9.96e-05 | 3429.0ms/step | 5.8 samples/s +Step 31270/44000 | Loss: 2.9580 | LR: 9.95e-05 | 1280.6ms/step | 15.6 samples/s +Step 31280/44000 | Loss: 2.9181 | LR: 9.93e-05 | 1278.3ms/step | 15.6 samples/s +Step 31290/44000 | Loss: 2.9467 | LR: 9.92e-05 | 1278.3ms/step | 15.6 samples/s +Step 31300/44000 | Loss: 2.9440 | LR: 9.90e-05 | 1280.7ms/step | 15.6 samples/s +Step 31310/44000 | Loss: 2.9327 | LR: 9.89e-05 | 1281.0ms/step | 15.6 samples/s +Step 31320/44000 | Loss: 2.9287 | LR: 9.88e-05 | 1283.2ms/step | 15.6 samples/s +Step 31330/44000 | Loss: 2.9575 | LR: 9.86e-05 | 1283.3ms/step | 15.6 samples/s +Step 31340/44000 | Loss: 2.9239 | LR: 9.85e-05 | 1283.0ms/step | 15.6 samples/s +Step 31350/44000 | Loss: 2.9033 | LR: 9.83e-05 | 1281.3ms/step | 15.6 samples/s +Step 31360/44000 | Loss: 2.9128 | LR: 9.82e-05 | 1284.5ms/step | 15.6 samples/s +Step 31370/44000 | Loss: 2.9458 | LR: 9.80e-05 | 1287.3ms/step | 15.5 samples/s +Step 31380/44000 | Loss: 2.9584 | LR: 9.79e-05 | 1280.6ms/step | 15.6 samples/s +Step 31390/44000 | Loss: 2.8989 | LR: 9.77e-05 | 1282.3ms/step | 15.6 samples/s +Step 31400/44000 | Loss: 2.9322 | LR: 9.76e-05 | 1281.3ms/step | 15.6 samples/s +Step 31410/44000 | Loss: 2.9019 | LR: 9.75e-05 | 1281.5ms/step | 15.6 samples/s +Step 31420/44000 | Loss: 2.9209 | LR: 9.73e-05 | 1280.7ms/step | 15.6 samples/s +Step 31430/44000 | Loss: 2.9436 | LR: 9.72e-05 | 1280.4ms/step | 15.6 samples/s +Step 31440/44000 | Loss: 2.9288 | LR: 9.70e-05 | 1278.9ms/step | 15.6 samples/s +Step 31450/44000 | Loss: 2.9398 | LR: 9.69e-05 | 1282.8ms/step | 15.6 samples/s +Step 31460/44000 | Loss: 2.9194 | LR: 9.67e-05 | 1280.4ms/step | 15.6 samples/s +Step 31470/44000 | Loss: 2.9465 | LR: 9.66e-05 | 1281.5ms/step | 15.6 samples/s +Step 31480/44000 | Loss: 2.9106 | LR: 9.65e-05 | 1283.0ms/step | 15.6 samples/s +Step 31490/44000 | Loss: 2.9214 | LR: 9.63e-05 | 1284.7ms/step | 15.6 samples/s +Step 31500/44000 | Loss: 2.9331 | LR: 9.62e-05 | 1283.9ms/step | 15.6 samples/s +Validation at step 31500: Loss = 2.9467, Perplexity = 19.04 +Step 31510/44000 | Loss: 2.9144 | LR: 9.60e-05 | 3432.1ms/step | 5.8 samples/s +Step 31520/44000 | Loss: 2.9249 | LR: 9.59e-05 | 1282.1ms/step | 15.6 samples/s +Step 31530/44000 | Loss: 2.9383 | LR: 9.57e-05 | 1277.1ms/step | 15.7 samples/s +Step 31540/44000 | Loss: 2.9196 | LR: 9.56e-05 | 1280.4ms/step | 15.6 samples/s +Step 31550/44000 | Loss: 2.9444 | LR: 9.54e-05 | 1281.8ms/step | 15.6 samples/s +Step 31560/44000 | Loss: 2.9312 | LR: 9.53e-05 | 1285.2ms/step | 15.6 samples/s +Step 31570/44000 | Loss: 2.9205 | LR: 9.52e-05 | 1288.1ms/step | 15.5 samples/s +Step 31580/44000 | Loss: 2.9434 | LR: 9.50e-05 | 1288.0ms/step | 15.5 samples/s +Step 31590/44000 | Loss: 2.9458 | LR: 9.49e-05 | 1283.0ms/step | 15.6 samples/s +Step 31600/44000 | Loss: 2.9279 | LR: 9.47e-05 | 1285.6ms/step | 15.6 samples/s +Step 31610/44000 | Loss: 2.9172 | LR: 9.46e-05 | 1285.1ms/step | 15.6 samples/s +Step 31620/44000 | Loss: 2.9402 | LR: 9.45e-05 | 1283.3ms/step | 15.6 samples/s +Step 31630/44000 | Loss: 2.9228 | LR: 9.43e-05 | 1279.5ms/step | 15.6 samples/s +Step 31640/44000 | Loss: 2.8986 | LR: 9.42e-05 | 1280.7ms/step | 15.6 samples/s +Step 31650/44000 | Loss: 2.9026 | LR: 9.40e-05 | 1282.6ms/step | 15.6 samples/s +Step 31660/44000 | Loss: 2.9184 | LR: 9.39e-05 | 1286.0ms/step | 15.6 samples/s +Step 31670/44000 | Loss: 2.9130 | LR: 9.37e-05 | 1283.9ms/step | 15.6 samples/s +Step 31680/44000 | Loss: 2.9238 | LR: 9.36e-05 | 1283.0ms/step | 15.6 samples/s +Step 31690/44000 | Loss: 2.9336 | LR: 9.35e-05 | 1283.3ms/step | 15.6 samples/s +Step 31700/44000 | Loss: 2.9488 | LR: 9.33e-05 | 1283.2ms/step | 15.6 samples/s +Step 31710/44000 | Loss: 2.9251 | LR: 9.32e-05 | 1281.4ms/step | 15.6 samples/s +Step 31720/44000 | Loss: 2.9236 | LR: 9.30e-05 | 1281.5ms/step | 15.6 samples/s +Step 31730/44000 | Loss: 2.8992 | LR: 9.29e-05 | 1279.9ms/step | 15.6 samples/s +Step 31740/44000 | Loss: 2.9328 | LR: 9.28e-05 | 1283.6ms/step | 15.6 samples/s +Step 31750/44000 | Loss: 2.9411 | LR: 9.26e-05 | 1283.7ms/step | 15.6 samples/s +Validation at step 31750: Loss = 2.9455, Perplexity = 19.02 +Step 31760/44000 | Loss: 2.9150 | LR: 9.25e-05 | 3426.6ms/step | 5.8 samples/s +Step 31770/44000 | Loss: 2.9406 | LR: 9.23e-05 | 1278.9ms/step | 15.6 samples/s +Step 31780/44000 | Loss: 2.9306 | LR: 9.22e-05 | 1279.7ms/step | 15.6 samples/s +Step 31790/44000 | Loss: 2.9343 | LR: 9.20e-05 | 1285.3ms/step | 15.6 samples/s +Step 31800/44000 | Loss: 2.9039 | LR: 9.19e-05 | 1284.5ms/step | 15.6 samples/s +Step 31810/44000 | Loss: 2.9039 | LR: 9.18e-05 | 1281.9ms/step | 15.6 samples/s +Step 31820/44000 | Loss: 2.9216 | LR: 9.16e-05 | 1282.3ms/step | 15.6 samples/s +Step 31830/44000 | Loss: 2.9266 | LR: 9.15e-05 | 1282.2ms/step | 15.6 samples/s +Step 31840/44000 | Loss: 2.9395 | LR: 9.13e-05 | 1282.4ms/step | 15.6 samples/s +Step 31850/44000 | Loss: 2.9423 | LR: 9.12e-05 | 1283.8ms/step | 15.6 samples/s +Step 31860/44000 | Loss: 2.9493 | LR: 9.11e-05 | 1278.6ms/step | 15.6 samples/s +Step 31870/44000 | Loss: 2.9268 | LR: 9.09e-05 | 1281.7ms/step | 15.6 samples/s +Step 31880/44000 | Loss: 2.9603 | LR: 9.08e-05 | 1282.1ms/step | 15.6 samples/s +Step 31890/44000 | Loss: 2.9018 | LR: 9.06e-05 | 1285.5ms/step | 15.6 samples/s +Step 31900/44000 | Loss: 2.9196 | LR: 9.05e-05 | 1288.3ms/step | 15.5 samples/s +Step 31910/44000 | Loss: 2.9520 | LR: 9.04e-05 | 1283.7ms/step | 15.6 samples/s +Step 31920/44000 | Loss: 2.8788 | LR: 9.02e-05 | 1279.9ms/step | 15.6 samples/s +Step 31930/44000 | Loss: 2.9347 | LR: 9.01e-05 | 1282.5ms/step | 15.6 samples/s +Step 31940/44000 | Loss: 2.9434 | LR: 8.99e-05 | 1284.0ms/step | 15.6 samples/s +Step 31950/44000 | Loss: 2.9326 | LR: 8.98e-05 | 1283.5ms/step | 15.6 samples/s +Step 31960/44000 | Loss: 2.9469 | LR: 8.97e-05 | 1283.6ms/step | 15.6 samples/s +Step 31970/44000 | Loss: 2.9158 | LR: 8.95e-05 | 1280.0ms/step | 15.6 samples/s +Step 31980/44000 | Loss: 2.9277 | LR: 8.94e-05 | 1282.4ms/step | 15.6 samples/s +Step 31990/44000 | Loss: 2.8959 | LR: 8.92e-05 | 1282.4ms/step | 15.6 samples/s +Step 32000/44000 | Loss: 2.9176 | LR: 8.91e-05 | 1291.6ms/step | 15.5 samples/s +Validation at step 32000: Loss = 2.9450, Perplexity = 19.01 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-32000 +Step 32010/44000 | Loss: 2.9397 | LR: 8.90e-05 | 3735.2ms/step | 5.4 samples/s +Step 32020/44000 | Loss: 2.9297 | LR: 8.88e-05 | 1284.9ms/step | 15.6 samples/s +Step 32030/44000 | Loss: 2.9372 | LR: 8.87e-05 | 1278.7ms/step | 15.6 samples/s +Step 32040/44000 | Loss: 2.9391 | LR: 8.86e-05 | 1281.0ms/step | 15.6 samples/s +Step 32050/44000 | Loss: 2.9129 | LR: 8.84e-05 | 1280.6ms/step | 15.6 samples/s +Step 32060/44000 | Loss: 2.8988 | LR: 8.83e-05 | 1284.3ms/step | 15.6 samples/s +Step 32070/44000 | Loss: 2.9241 | LR: 8.81e-05 | 1285.1ms/step | 15.6 samples/s +Step 32080/44000 | Loss: 2.8984 | LR: 8.80e-05 | 1287.0ms/step | 15.5 samples/s +Step 32090/44000 | Loss: 2.9328 | LR: 8.79e-05 | 1281.0ms/step | 15.6 samples/s +Step 32100/44000 | Loss: 2.9345 | LR: 8.77e-05 | 1284.3ms/step | 15.6 samples/s +Step 32110/44000 | Loss: 2.9174 | LR: 8.76e-05 | 1280.7ms/step | 15.6 samples/s +Step 32120/44000 | Loss: 2.9225 | LR: 8.74e-05 | 1286.6ms/step | 15.5 samples/s +Step 32130/44000 | Loss: 2.9161 | LR: 8.73e-05 | 1278.9ms/step | 15.6 samples/s +Step 32140/44000 | Loss: 2.9126 | LR: 8.72e-05 | 1282.9ms/step | 15.6 samples/s +Step 32150/44000 | Loss: 2.9310 | LR: 8.70e-05 | 1281.7ms/step | 15.6 samples/s +Step 32160/44000 | Loss: 2.9002 | LR: 8.69e-05 | 1281.5ms/step | 15.6 samples/s +Step 32170/44000 | Loss: 2.9247 | LR: 8.68e-05 | 1281.8ms/step | 15.6 samples/s +Step 32180/44000 | Loss: 2.9581 | LR: 8.66e-05 | 1281.5ms/step | 15.6 samples/s +Step 32190/44000 | Loss: 2.9291 | LR: 8.65e-05 | 1283.0ms/step | 15.6 samples/s +Step 32200/44000 | Loss: 2.9424 | LR: 8.63e-05 | 1283.0ms/step | 15.6 samples/s +Step 32210/44000 | Loss: 2.9470 | LR: 8.62e-05 | 1283.7ms/step | 15.6 samples/s +Step 32220/44000 | Loss: 2.9039 | LR: 8.61e-05 | 1280.2ms/step | 15.6 samples/s +Step 32230/44000 | Loss: 2.9332 | LR: 8.59e-05 | 1279.9ms/step | 15.6 samples/s +Step 32240/44000 | Loss: 2.9214 | LR: 8.58e-05 | 1277.4ms/step | 15.7 samples/s +Step 32250/44000 | Loss: 2.9404 | LR: 8.57e-05 | 1280.4ms/step | 15.6 samples/s +Validation at step 32250: Loss = 2.9447, Perplexity = 19.00 +Step 32260/44000 | Loss: 2.9537 | LR: 8.55e-05 | 3436.8ms/step | 5.8 samples/s +Step 32270/44000 | Loss: 2.9281 | LR: 8.54e-05 | 1283.9ms/step | 15.6 samples/s +Step 32280/44000 | Loss: 2.9503 | LR: 8.53e-05 | 1284.1ms/step | 15.6 samples/s +Step 32290/44000 | Loss: 2.9186 | LR: 8.51e-05 | 1283.8ms/step | 15.6 samples/s +Step 32300/44000 | Loss: 2.9143 | LR: 8.50e-05 | 1283.3ms/step | 15.6 samples/s +Step 32310/44000 | Loss: 2.9365 | LR: 8.48e-05 | 1280.5ms/step | 15.6 samples/s +Step 32320/44000 | Loss: 2.9554 | LR: 8.47e-05 | 1281.8ms/step | 15.6 samples/s +Step 32330/44000 | Loss: 2.9245 | LR: 8.46e-05 | 1282.9ms/step | 15.6 samples/s +Step 32340/44000 | Loss: 2.9219 | LR: 8.44e-05 | 1284.2ms/step | 15.6 samples/s +Step 32350/44000 | Loss: 2.9446 | LR: 8.43e-05 | 1284.2ms/step | 15.6 samples/s +Step 32360/44000 | Loss: 2.9242 | LR: 8.42e-05 | 1283.4ms/step | 15.6 samples/s +Step 32370/44000 | Loss: 2.9223 | LR: 8.40e-05 | 1283.3ms/step | 15.6 samples/s +Step 32380/44000 | Loss: 2.9232 | LR: 8.39e-05 | 1283.7ms/step | 15.6 samples/s +Step 32390/44000 | Loss: 2.9250 | LR: 8.38e-05 | 1284.0ms/step | 15.6 samples/s +Step 32400/44000 | Loss: 2.9317 | LR: 8.36e-05 | 1283.6ms/step | 15.6 samples/s +Step 32410/44000 | Loss: 2.9281 | LR: 8.35e-05 | 1286.4ms/step | 15.5 samples/s +Step 32420/44000 | Loss: 2.9227 | LR: 8.33e-05 | 1283.6ms/step | 15.6 samples/s +Step 32430/44000 | Loss: 2.9714 | LR: 8.32e-05 | 1286.9ms/step | 15.5 samples/s +Step 32440/44000 | Loss: 2.9381 | LR: 8.31e-05 | 1282.1ms/step | 15.6 samples/s +Step 32450/44000 | Loss: 2.9304 | LR: 8.29e-05 | 1281.0ms/step | 15.6 samples/s +Step 32460/44000 | Loss: 2.9499 | LR: 8.28e-05 | 1280.3ms/step | 15.6 samples/s +Step 32470/44000 | Loss: 2.9074 | LR: 8.27e-05 | 1279.7ms/step | 15.6 samples/s +Step 32480/44000 | Loss: 2.9400 | LR: 8.25e-05 | 1282.3ms/step | 15.6 samples/s +Step 32490/44000 | Loss: 2.9176 | LR: 8.24e-05 | 1281.6ms/step | 15.6 samples/s +Step 32500/44000 | Loss: 2.9353 | LR: 8.23e-05 | 1284.3ms/step | 15.6 samples/s +Validation at step 32500: Loss = 2.9445, Perplexity = 19.00 +Step 32510/44000 | Loss: 2.9026 | LR: 8.21e-05 | 3428.9ms/step | 5.8 samples/s +Step 32520/44000 | Loss: 2.9278 | LR: 8.20e-05 | 1279.0ms/step | 15.6 samples/s +Step 32530/44000 | Loss: 2.9574 | LR: 8.19e-05 | 1280.7ms/step | 15.6 samples/s +Step 32540/44000 | Loss: 2.9128 | LR: 8.17e-05 | 1282.8ms/step | 15.6 samples/s +Step 32550/44000 | Loss: 2.8830 | LR: 8.16e-05 | 1283.3ms/step | 15.6 samples/s +Step 32560/44000 | Loss: 2.9496 | LR: 8.15e-05 | 1284.4ms/step | 15.6 samples/s +Step 32570/44000 | Loss: 2.9296 | LR: 8.13e-05 | 1282.3ms/step | 15.6 samples/s +Step 32580/44000 | Loss: 2.9356 | LR: 8.12e-05 | 1281.7ms/step | 15.6 samples/s +Step 32590/44000 | Loss: 2.9600 | LR: 8.11e-05 | 1283.1ms/step | 15.6 samples/s +Step 32600/44000 | Loss: 2.9348 | LR: 8.09e-05 | 1277.3ms/step | 15.7 samples/s +Step 32610/44000 | Loss: 2.9366 | LR: 8.08e-05 | 1282.9ms/step | 15.6 samples/s +Step 32620/44000 | Loss: 2.9457 | LR: 8.07e-05 | 1282.1ms/step | 15.6 samples/s +Step 32630/44000 | Loss: 2.9048 | LR: 8.05e-05 | 1282.5ms/step | 15.6 samples/s +Step 32640/44000 | Loss: 2.9668 | LR: 8.04e-05 | 1282.7ms/step | 15.6 samples/s +Step 32650/44000 | Loss: 2.9450 | LR: 8.03e-05 | 1282.5ms/step | 15.6 samples/s +Step 32660/44000 | Loss: 2.9319 | LR: 8.01e-05 | 1282.0ms/step | 15.6 samples/s +Step 32670/44000 | Loss: 2.9687 | LR: 8.00e-05 | 1283.9ms/step | 15.6 samples/s +Step 32680/44000 | Loss: 2.9262 | LR: 7.99e-05 | 1283.9ms/step | 15.6 samples/s +Step 32690/44000 | Loss: 2.9356 | LR: 7.97e-05 | 1284.4ms/step | 15.6 samples/s +Step 32700/44000 | Loss: 2.9160 | LR: 7.96e-05 | 1283.2ms/step | 15.6 samples/s +Step 32710/44000 | Loss: 2.9454 | LR: 7.95e-05 | 1280.8ms/step | 15.6 samples/s +Step 32720/44000 | Loss: 2.9440 | LR: 7.93e-05 | 1281.1ms/step | 15.6 samples/s +Step 32730/44000 | Loss: 2.9213 | LR: 7.92e-05 | 1280.1ms/step | 15.6 samples/s +Step 32740/44000 | Loss: 2.9108 | LR: 7.91e-05 | 1281.7ms/step | 15.6 samples/s +Step 32750/44000 | Loss: 2.9031 | LR: 7.89e-05 | 1280.8ms/step | 15.6 samples/s +Validation at step 32750: Loss = 2.9441, Perplexity = 18.99 +Step 32760/44000 | Loss: 2.9369 | LR: 7.88e-05 | 3429.9ms/step | 5.8 samples/s +Step 32770/44000 | Loss: 2.9329 | LR: 7.87e-05 | 1281.0ms/step | 15.6 samples/s +Step 32780/44000 | Loss: 2.9594 | LR: 7.85e-05 | 1277.4ms/step | 15.7 samples/s +Step 32790/44000 | Loss: 2.9646 | LR: 7.84e-05 | 1281.6ms/step | 15.6 samples/s +Step 32800/44000 | Loss: 2.9191 | LR: 7.83e-05 | 1281.6ms/step | 15.6 samples/s +Step 32810/44000 | Loss: 2.9298 | LR: 7.81e-05 | 1282.1ms/step | 15.6 samples/s +Step 32820/44000 | Loss: 2.9482 | LR: 7.80e-05 | 1280.4ms/step | 15.6 samples/s +Step 32830/44000 | Loss: 2.9447 | LR: 7.79e-05 | 1281.3ms/step | 15.6 samples/s +Step 32840/44000 | Loss: 2.8967 | LR: 7.77e-05 | 1283.0ms/step | 15.6 samples/s +Step 32850/44000 | Loss: 2.9266 | LR: 7.76e-05 | 1284.9ms/step | 15.6 samples/s +Step 32860/44000 | Loss: 2.9252 | LR: 7.75e-05 | 1284.5ms/step | 15.6 samples/s +Step 32870/44000 | Loss: 2.9410 | LR: 7.73e-05 | 1281.7ms/step | 15.6 samples/s +Step 32880/44000 | Loss: 2.9015 | LR: 7.72e-05 | 1282.7ms/step | 15.6 samples/s +Step 32890/44000 | Loss: 2.9395 | LR: 7.71e-05 | 1281.6ms/step | 15.6 samples/s +Step 32900/44000 | Loss: 2.9342 | LR: 7.70e-05 | 1281.6ms/step | 15.6 samples/s +Step 32910/44000 | Loss: 2.9248 | LR: 7.68e-05 | 1282.4ms/step | 15.6 samples/s +Step 32920/44000 | Loss: 2.9511 | LR: 7.67e-05 | 1285.1ms/step | 15.6 samples/s +Step 32930/44000 | Loss: 2.9219 | LR: 7.66e-05 | 1284.4ms/step | 15.6 samples/s +Step 32940/44000 | Loss: 2.9092 | LR: 7.64e-05 | 1282.1ms/step | 15.6 samples/s +Step 32950/44000 | Loss: 2.9148 | LR: 7.63e-05 | 1284.7ms/step | 15.6 samples/s +Step 32960/44000 | Loss: 2.9225 | LR: 7.62e-05 | 1286.0ms/step | 15.6 samples/s +Step 32970/44000 | Loss: 2.9278 | LR: 7.60e-05 | 1286.8ms/step | 15.5 samples/s +Step 32980/44000 | Loss: 2.9298 | LR: 7.59e-05 | 1284.3ms/step | 15.6 samples/s +Step 32990/44000 | Loss: 2.9323 | LR: 7.58e-05 | 1286.0ms/step | 15.6 samples/s +Step 33000/44000 | Loss: 2.9536 | LR: 7.56e-05 | 1284.4ms/step | 15.6 samples/s +Validation at step 33000: Loss = 2.9433, Perplexity = 18.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-33000 +Step 33010/44000 | Loss: 2.9425 | LR: 7.55e-05 | 3734.2ms/step | 5.4 samples/s +Step 33020/44000 | Loss: 2.9049 | LR: 7.54e-05 | 1282.5ms/step | 15.6 samples/s +Step 33030/44000 | Loss: 2.9497 | LR: 7.53e-05 | 1281.3ms/step | 15.6 samples/s +Step 33040/44000 | Loss: 2.9344 | LR: 7.51e-05 | 1279.6ms/step | 15.6 samples/s +Step 33050/44000 | Loss: 2.9441 | LR: 7.50e-05 | 1281.5ms/step | 15.6 samples/s +Step 33060/44000 | Loss: 2.9124 | LR: 7.49e-05 | 1282.9ms/step | 15.6 samples/s +Step 33070/44000 | Loss: 2.9436 | LR: 7.47e-05 | 1281.5ms/step | 15.6 samples/s +Step 33080/44000 | Loss: 2.9434 | LR: 7.46e-05 | 1288.8ms/step | 15.5 samples/s +Step 33090/44000 | Loss: 2.9325 | LR: 7.45e-05 | 1281.0ms/step | 15.6 samples/s +Step 33100/44000 | Loss: 2.9062 | LR: 7.44e-05 | 1283.4ms/step | 15.6 samples/s +Step 33110/44000 | Loss: 2.8837 | LR: 7.42e-05 | 1283.1ms/step | 15.6 samples/s +Step 33120/44000 | Loss: 2.9148 | LR: 7.41e-05 | 1283.7ms/step | 15.6 samples/s +Step 33130/44000 | Loss: 2.9330 | LR: 7.40e-05 | 1281.8ms/step | 15.6 samples/s +Step 33140/44000 | Loss: 2.9493 | LR: 7.38e-05 | 1283.3ms/step | 15.6 samples/s +Step 33150/44000 | Loss: 2.9413 | LR: 7.37e-05 | 1281.1ms/step | 15.6 samples/s +Step 33160/44000 | Loss: 2.9175 | LR: 7.36e-05 | 1280.2ms/step | 15.6 samples/s +Step 33170/44000 | Loss: 2.9231 | LR: 7.34e-05 | 1282.2ms/step | 15.6 samples/s +Step 33180/44000 | Loss: 2.9467 | LR: 7.33e-05 | 1282.1ms/step | 15.6 samples/s +Step 33190/44000 | Loss: 2.9477 | LR: 7.32e-05 | 1279.8ms/step | 15.6 samples/s +Step 33200/44000 | Loss: 2.9373 | LR: 7.31e-05 | 1279.6ms/step | 15.6 samples/s +Step 33210/44000 | Loss: 2.9244 | LR: 7.29e-05 | 1280.3ms/step | 15.6 samples/s +Step 33220/44000 | Loss: 2.9416 | LR: 7.28e-05 | 1282.7ms/step | 15.6 samples/s +Step 33230/44000 | Loss: 2.9247 | LR: 7.27e-05 | 1284.7ms/step | 15.6 samples/s +Step 33240/44000 | Loss: 2.9188 | LR: 7.26e-05 | 1282.5ms/step | 15.6 samples/s +Step 33250/44000 | Loss: 2.9285 | LR: 7.24e-05 | 1284.8ms/step | 15.6 samples/s +Validation at step 33250: Loss = 2.9428, Perplexity = 18.97 +Step 33260/44000 | Loss: 2.9415 | LR: 7.23e-05 | 3435.5ms/step | 5.8 samples/s +Step 33270/44000 | Loss: 2.9225 | LR: 7.22e-05 | 1281.4ms/step | 15.6 samples/s +Step 33280/44000 | Loss: 2.9397 | LR: 7.20e-05 | 1282.5ms/step | 15.6 samples/s +Step 33290/44000 | Loss: 2.8882 | LR: 7.19e-05 | 1282.2ms/step | 15.6 samples/s +Step 33300/44000 | Loss: 2.9315 | LR: 7.18e-05 | 1280.0ms/step | 15.6 samples/s +Step 33310/44000 | Loss: 2.9454 | LR: 7.17e-05 | 1277.2ms/step | 15.7 samples/s +Step 33320/44000 | Loss: 2.9278 | LR: 7.15e-05 | 1283.7ms/step | 15.6 samples/s +Step 33330/44000 | Loss: 2.9171 | LR: 7.14e-05 | 1280.7ms/step | 15.6 samples/s +Step 33340/44000 | Loss: 2.9197 | LR: 7.13e-05 | 1279.6ms/step | 15.6 samples/s +Step 33350/44000 | Loss: 2.9303 | LR: 7.11e-05 | 1284.6ms/step | 15.6 samples/s +Step 33360/44000 | Loss: 2.9410 | LR: 7.10e-05 | 1281.7ms/step | 15.6 samples/s +Step 33370/44000 | Loss: 2.9013 | LR: 7.09e-05 | 1285.4ms/step | 15.6 samples/s +Step 33380/44000 | Loss: 2.9174 | LR: 7.08e-05 | 1286.3ms/step | 15.5 samples/s +Step 33390/44000 | Loss: 2.9481 | LR: 7.06e-05 | 1285.8ms/step | 15.6 samples/s +Step 33400/44000 | Loss: 2.8955 | LR: 7.05e-05 | 1287.3ms/step | 15.5 samples/s +Step 33410/44000 | Loss: 2.9422 | LR: 7.04e-05 | 1292.1ms/step | 15.5 samples/s +Step 33420/44000 | Loss: 2.9201 | LR: 7.03e-05 | 1286.9ms/step | 15.5 samples/s +Step 33430/44000 | Loss: 2.9179 | LR: 7.01e-05 | 1283.8ms/step | 15.6 samples/s +Step 33440/44000 | Loss: 2.9450 | LR: 7.00e-05 | 1283.6ms/step | 15.6 samples/s +Step 33450/44000 | Loss: 2.9405 | LR: 6.99e-05 | 1286.7ms/step | 15.5 samples/s +Step 33460/44000 | Loss: 2.9338 | LR: 6.98e-05 | 1281.5ms/step | 15.6 samples/s +Step 33470/44000 | Loss: 2.9250 | LR: 6.96e-05 | 1281.9ms/step | 15.6 samples/s +Step 33480/44000 | Loss: 2.9120 | LR: 6.95e-05 | 1277.1ms/step | 15.7 samples/s +Step 33490/44000 | Loss: 2.9429 | LR: 6.94e-05 | 1280.1ms/step | 15.6 samples/s +Step 33500/44000 | Loss: 2.9014 | LR: 6.93e-05 | 1280.8ms/step | 15.6 samples/s +Validation at step 33500: Loss = 2.9426, Perplexity = 18.96 +Step 33510/44000 | Loss: 2.9069 | LR: 6.91e-05 | 3430.6ms/step | 5.8 samples/s +Step 33520/44000 | Loss: 2.9460 | LR: 6.90e-05 | 1280.0ms/step | 15.6 samples/s +Step 33530/44000 | Loss: 2.9082 | LR: 6.89e-05 | 1279.3ms/step | 15.6 samples/s +Step 33540/44000 | Loss: 2.9152 | LR: 6.88e-05 | 1282.0ms/step | 15.6 samples/s +Step 33550/44000 | Loss: 2.9245 | LR: 6.86e-05 | 1283.2ms/step | 15.6 samples/s +Step 33560/44000 | Loss: 2.9059 | LR: 6.85e-05 | 1280.8ms/step | 15.6 samples/s +Step 33570/44000 | Loss: 2.8995 | LR: 6.84e-05 | 1281.5ms/step | 15.6 samples/s +Step 33580/44000 | Loss: 2.9466 | LR: 6.83e-05 | 1281.2ms/step | 15.6 samples/s +Step 33590/44000 | Loss: 2.9424 | LR: 6.81e-05 | 1280.1ms/step | 15.6 samples/s +Step 33600/44000 | Loss: 2.9487 | LR: 6.80e-05 | 1283.4ms/step | 15.6 samples/s +Step 33610/44000 | Loss: 2.9037 | LR: 6.79e-05 | 1282.4ms/step | 15.6 samples/s +Step 33620/44000 | Loss: 2.9190 | LR: 6.78e-05 | 1282.8ms/step | 15.6 samples/s +Step 33630/44000 | Loss: 2.9084 | LR: 6.76e-05 | 1281.8ms/step | 15.6 samples/s +Step 33640/44000 | Loss: 2.9215 | LR: 6.75e-05 | 1282.9ms/step | 15.6 samples/s +Step 33650/44000 | Loss: 2.9299 | LR: 6.74e-05 | 1281.9ms/step | 15.6 samples/s +Step 33660/44000 | Loss: 2.9346 | LR: 6.73e-05 | 1281.5ms/step | 15.6 samples/s +Step 33670/44000 | Loss: 2.9421 | LR: 6.71e-05 | 1277.7ms/step | 15.7 samples/s +Step 33680/44000 | Loss: 2.9273 | LR: 6.70e-05 | 1283.8ms/step | 15.6 samples/s +Step 33690/44000 | Loss: 2.9247 | LR: 6.69e-05 | 1282.2ms/step | 15.6 samples/s +Step 33700/44000 | Loss: 2.9413 | LR: 6.68e-05 | 1280.7ms/step | 15.6 samples/s +Step 33710/44000 | Loss: 2.9558 | LR: 6.66e-05 | 1283.0ms/step | 15.6 samples/s +Step 33720/44000 | Loss: 2.9331 | LR: 6.65e-05 | 1280.0ms/step | 15.6 samples/s +Step 33730/44000 | Loss: 2.9158 | LR: 6.64e-05 | 1280.5ms/step | 15.6 samples/s +Step 33740/44000 | Loss: 2.9309 | LR: 6.63e-05 | 1282.8ms/step | 15.6 samples/s +Step 33750/44000 | Loss: 2.9348 | LR: 6.61e-05 | 1284.4ms/step | 15.6 samples/s +Validation at step 33750: Loss = 2.9420, Perplexity = 18.95 +Step 33760/44000 | Loss: 2.9018 | LR: 6.60e-05 | 3429.7ms/step | 5.8 samples/s +Step 33770/44000 | Loss: 2.9503 | LR: 6.59e-05 | 1282.4ms/step | 15.6 samples/s +Step 33780/44000 | Loss: 2.9518 | LR: 6.58e-05 | 1280.1ms/step | 15.6 samples/s +Step 33790/44000 | Loss: 2.9054 | LR: 6.57e-05 | 1282.9ms/step | 15.6 samples/s +Step 33800/44000 | Loss: 2.9387 | LR: 6.55e-05 | 1283.1ms/step | 15.6 samples/s +Step 33810/44000 | Loss: 2.9554 | LR: 6.54e-05 | 1284.2ms/step | 15.6 samples/s +Step 33820/44000 | Loss: 2.9282 | LR: 6.53e-05 | 1284.0ms/step | 15.6 samples/s +Step 33830/44000 | Loss: 2.9487 | LR: 6.52e-05 | 1281.9ms/step | 15.6 samples/s +Step 33840/44000 | Loss: 2.9299 | LR: 6.50e-05 | 1280.0ms/step | 15.6 samples/s +Step 33850/44000 | Loss: 2.9398 | LR: 6.49e-05 | 1278.6ms/step | 15.6 samples/s +Step 33860/44000 | Loss: 2.9231 | LR: 6.48e-05 | 1283.0ms/step | 15.6 samples/s +Step 33870/44000 | Loss: 2.9331 | LR: 6.47e-05 | 1283.7ms/step | 15.6 samples/s +Step 33880/44000 | Loss: 2.9634 | LR: 6.46e-05 | 1285.2ms/step | 15.6 samples/s +Step 33890/44000 | Loss: 2.9211 | LR: 6.44e-05 | 1281.7ms/step | 15.6 samples/s +Step 33900/44000 | Loss: 2.9467 | LR: 6.43e-05 | 1279.4ms/step | 15.6 samples/s +Step 33910/44000 | Loss: 2.9369 | LR: 6.42e-05 | 1282.7ms/step | 15.6 samples/s +Step 33920/44000 | Loss: 2.9296 | LR: 6.41e-05 | 1280.5ms/step | 15.6 samples/s +Step 33930/44000 | Loss: 2.8887 | LR: 6.40e-05 | 1280.5ms/step | 15.6 samples/s +Step 33940/44000 | Loss: 2.9249 | LR: 6.38e-05 | 1282.9ms/step | 15.6 samples/s +Step 33950/44000 | Loss: 2.9208 | LR: 6.37e-05 | 1285.5ms/step | 15.6 samples/s +Step 33960/44000 | Loss: 2.9108 | LR: 6.36e-05 | 1285.6ms/step | 15.6 samples/s +Step 33970/44000 | Loss: 2.9178 | LR: 6.35e-05 | 1283.8ms/step | 15.6 samples/s +Step 33980/44000 | Loss: 2.9525 | LR: 6.33e-05 | 1282.0ms/step | 15.6 samples/s +Step 33990/44000 | Loss: 2.9445 | LR: 6.32e-05 | 1277.5ms/step | 15.7 samples/s +Step 34000/44000 | Loss: 2.9473 | LR: 6.31e-05 | 1278.7ms/step | 15.6 samples/s +Validation at step 34000: Loss = 2.9415, Perplexity = 18.94 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-34000 +Step 34010/44000 | Loss: 2.9267 | LR: 6.30e-05 | 3679.7ms/step | 5.4 samples/s +Step 34020/44000 | Loss: 2.9199 | LR: 6.29e-05 | 1283.7ms/step | 15.6 samples/s +Step 34030/44000 | Loss: 2.9683 | LR: 6.27e-05 | 1282.8ms/step | 15.6 samples/s +Step 34040/44000 | Loss: 2.9203 | LR: 6.26e-05 | 1284.9ms/step | 15.6 samples/s +Step 34050/44000 | Loss: 2.9217 | LR: 6.25e-05 | 1282.7ms/step | 15.6 samples/s +Step 34060/44000 | Loss: 2.9274 | LR: 6.24e-05 | 1279.7ms/step | 15.6 samples/s +Step 34070/44000 | Loss: 2.9507 | LR: 6.23e-05 | 1286.4ms/step | 15.5 samples/s +Step 34080/44000 | Loss: 2.9336 | LR: 6.21e-05 | 1284.5ms/step | 15.6 samples/s +Step 34090/44000 | Loss: 2.9153 | LR: 6.20e-05 | 1284.1ms/step | 15.6 samples/s +Step 34100/44000 | Loss: 2.9314 | LR: 6.19e-05 | 1284.9ms/step | 15.6 samples/s +Step 34110/44000 | Loss: 2.8995 | LR: 6.18e-05 | 1283.9ms/step | 15.6 samples/s +Step 34120/44000 | Loss: 2.9234 | LR: 6.17e-05 | 1286.0ms/step | 15.6 samples/s +Step 34130/44000 | Loss: 2.9275 | LR: 6.15e-05 | 1283.5ms/step | 15.6 samples/s +Step 34140/44000 | Loss: 2.9368 | LR: 6.14e-05 | 1279.1ms/step | 15.6 samples/s +Step 34150/44000 | Loss: 2.9319 | LR: 6.13e-05 | 1278.4ms/step | 15.6 samples/s +Step 34160/44000 | Loss: 2.9605 | LR: 6.12e-05 | 1277.4ms/step | 15.7 samples/s +Step 34170/44000 | Loss: 2.9363 | LR: 6.11e-05 | 1280.8ms/step | 15.6 samples/s +Step 34180/44000 | Loss: 2.9137 | LR: 6.09e-05 | 1277.8ms/step | 15.7 samples/s +Step 34190/44000 | Loss: 2.9380 | LR: 6.08e-05 | 1279.6ms/step | 15.6 samples/s +Step 34200/44000 | Loss: 2.9565 | LR: 6.07e-05 | 1278.0ms/step | 15.6 samples/s +Step 34210/44000 | Loss: 2.9355 | LR: 6.06e-05 | 1279.2ms/step | 15.6 samples/s +Step 34220/44000 | Loss: 2.9274 | LR: 6.05e-05 | 1282.9ms/step | 15.6 samples/s +Step 34230/44000 | Loss: 2.9432 | LR: 6.04e-05 | 1276.4ms/step | 15.7 samples/s +Step 34240/44000 | Loss: 2.9611 | LR: 6.02e-05 | 1286.0ms/step | 15.6 samples/s +Step 34250/44000 | Loss: 2.9380 | LR: 6.01e-05 | 1283.8ms/step | 15.6 samples/s +Validation at step 34250: Loss = 2.9415, Perplexity = 18.94 +Step 34260/44000 | Loss: 2.9370 | LR: 6.00e-05 | 3429.2ms/step | 5.8 samples/s +Step 34270/44000 | Loss: 2.9101 | LR: 5.99e-05 | 1286.8ms/step | 15.5 samples/s +Step 34280/44000 | Loss: 2.9524 | LR: 5.98e-05 | 1298.5ms/step | 15.4 samples/s +Step 34290/44000 | Loss: 2.9133 | LR: 5.96e-05 | 1295.6ms/step | 15.4 samples/s +Step 34300/44000 | Loss: 2.9126 | LR: 5.95e-05 | 1288.6ms/step | 15.5 samples/s +Step 34310/44000 | Loss: 2.9261 | LR: 5.94e-05 | 1290.1ms/step | 15.5 samples/s +Step 34320/44000 | Loss: 2.9762 | LR: 5.93e-05 | 1282.2ms/step | 15.6 samples/s +Step 34330/44000 | Loss: 2.8974 | LR: 5.92e-05 | 1281.1ms/step | 15.6 samples/s +Step 34340/44000 | Loss: 2.8999 | LR: 5.91e-05 | 1280.1ms/step | 15.6 samples/s +Step 34350/44000 | Loss: 2.9218 | LR: 5.89e-05 | 1289.1ms/step | 15.5 samples/s +Step 34360/44000 | Loss: 2.9341 | LR: 5.88e-05 | 1295.3ms/step | 15.4 samples/s +Step 34370/44000 | Loss: 2.9210 | LR: 5.87e-05 | 1284.3ms/step | 15.6 samples/s +Step 34380/44000 | Loss: 2.9195 | LR: 5.86e-05 | 1285.8ms/step | 15.6 samples/s +Step 34390/44000 | Loss: 2.9318 | LR: 5.85e-05 | 1284.6ms/step | 15.6 samples/s +Step 34400/44000 | Loss: 2.9311 | LR: 5.84e-05 | 1285.5ms/step | 15.6 samples/s +Step 34410/44000 | Loss: 2.9490 | LR: 5.82e-05 | 1280.2ms/step | 15.6 samples/s +Step 34420/44000 | Loss: 2.9288 | LR: 5.81e-05 | 1285.0ms/step | 15.6 samples/s +Step 34430/44000 | Loss: 2.9247 | LR: 5.80e-05 | 1283.9ms/step | 15.6 samples/s +Step 34440/44000 | Loss: 2.9368 | LR: 5.79e-05 | 1283.5ms/step | 15.6 samples/s +Step 34450/44000 | Loss: 2.9163 | LR: 5.78e-05 | 1282.5ms/step | 15.6 samples/s +Step 34460/44000 | Loss: 2.9456 | LR: 5.77e-05 | 1280.8ms/step | 15.6 samples/s +Step 34470/44000 | Loss: 2.9276 | LR: 5.75e-05 | 1280.6ms/step | 15.6 samples/s +Step 34480/44000 | Loss: 2.9332 | LR: 5.74e-05 | 1281.7ms/step | 15.6 samples/s +Step 34490/44000 | Loss: 2.8963 | LR: 5.73e-05 | 1281.4ms/step | 15.6 samples/s +Step 34500/44000 | Loss: 2.9452 | LR: 5.72e-05 | 1283.0ms/step | 15.6 samples/s +Validation at step 34500: Loss = 2.9411, Perplexity = 18.94 +Step 34510/44000 | Loss: 2.9272 | LR: 5.71e-05 | 3430.1ms/step | 5.8 samples/s +Step 34520/44000 | Loss: 2.9207 | LR: 5.70e-05 | 1279.8ms/step | 15.6 samples/s +Step 34530/44000 | Loss: 2.9121 | LR: 5.69e-05 | 1278.1ms/step | 15.6 samples/s +Step 34540/44000 | Loss: 2.9313 | LR: 5.67e-05 | 1283.6ms/step | 15.6 samples/s +Step 34550/44000 | Loss: 2.9483 | LR: 5.66e-05 | 1285.2ms/step | 15.6 samples/s +Step 34560/44000 | Loss: 2.8980 | LR: 5.65e-05 | 1283.1ms/step | 15.6 samples/s +Step 34570/44000 | Loss: 2.9144 | LR: 5.64e-05 | 1283.7ms/step | 15.6 samples/s +Step 34580/44000 | Loss: 2.9073 | LR: 5.63e-05 | 1283.8ms/step | 15.6 samples/s +Step 34590/44000 | Loss: 2.9200 | LR: 5.62e-05 | 1284.5ms/step | 15.6 samples/s +Step 34600/44000 | Loss: 2.9328 | LR: 5.60e-05 | 1280.6ms/step | 15.6 samples/s +Step 34610/44000 | Loss: 2.9333 | LR: 5.59e-05 | 1282.7ms/step | 15.6 samples/s +Step 34620/44000 | Loss: 2.9286 | LR: 5.58e-05 | 1284.0ms/step | 15.6 samples/s +Step 34630/44000 | Loss: 2.9163 | LR: 5.57e-05 | 1282.2ms/step | 15.6 samples/s +Step 34640/44000 | Loss: 2.9242 | LR: 5.56e-05 | 1283.9ms/step | 15.6 samples/s +Step 34650/44000 | Loss: 2.9388 | LR: 5.55e-05 | 1283.5ms/step | 15.6 samples/s +Step 34660/44000 | Loss: 2.9449 | LR: 5.54e-05 | 1283.9ms/step | 15.6 samples/s +Step 34670/44000 | Loss: 2.8966 | LR: 5.52e-05 | 1283.2ms/step | 15.6 samples/s +Step 34680/44000 | Loss: 2.9297 | LR: 5.51e-05 | 1283.2ms/step | 15.6 samples/s +Step 34690/44000 | Loss: 2.9127 | LR: 5.50e-05 | 1283.1ms/step | 15.6 samples/s +Step 34700/44000 | Loss: 2.9034 | LR: 5.49e-05 | 1282.8ms/step | 15.6 samples/s +Step 34710/44000 | Loss: 2.9407 | LR: 5.48e-05 | 1280.1ms/step | 15.6 samples/s +Step 34720/44000 | Loss: 2.9150 | LR: 5.47e-05 | 1275.4ms/step | 15.7 samples/s +Step 34730/44000 | Loss: 2.9406 | LR: 5.46e-05 | 1279.7ms/step | 15.6 samples/s +Step 34740/44000 | Loss: 2.9145 | LR: 5.45e-05 | 1281.1ms/step | 15.6 samples/s +Step 34750/44000 | Loss: 2.9236 | LR: 5.43e-05 | 1281.1ms/step | 15.6 samples/s +Validation at step 34750: Loss = 2.9409, Perplexity = 18.93 +Step 34760/44000 | Loss: 2.9102 | LR: 5.42e-05 | 3430.5ms/step | 5.8 samples/s +Step 34770/44000 | Loss: 2.9163 | LR: 5.41e-05 | 1275.2ms/step | 15.7 samples/s +Step 34780/44000 | Loss: 2.9564 | LR: 5.40e-05 | 1283.0ms/step | 15.6 samples/s +Step 34790/44000 | Loss: 2.9514 | LR: 5.39e-05 | 1284.8ms/step | 15.6 samples/s +Step 34800/44000 | Loss: 2.9301 | LR: 5.38e-05 | 1280.1ms/step | 15.6 samples/s +Step 34810/44000 | Loss: 2.9472 | LR: 5.37e-05 | 1282.5ms/step | 15.6 samples/s +Step 34820/44000 | Loss: 2.9277 | LR: 5.36e-05 | 1285.4ms/step | 15.6 samples/s +Step 34830/44000 | Loss: 2.9254 | LR: 5.34e-05 | 1282.8ms/step | 15.6 samples/s +Step 34840/44000 | Loss: 2.9359 | LR: 5.33e-05 | 1282.5ms/step | 15.6 samples/s +Step 34850/44000 | Loss: 2.9448 | LR: 5.32e-05 | 1280.6ms/step | 15.6 samples/s +Step 34860/44000 | Loss: 2.9354 | LR: 5.31e-05 | 1283.2ms/step | 15.6 samples/s +Step 34870/44000 | Loss: 2.9163 | LR: 5.30e-05 | 1281.1ms/step | 15.6 samples/s +Step 34880/44000 | Loss: 2.9240 | LR: 5.29e-05 | 1283.9ms/step | 15.6 samples/s +Step 34890/44000 | Loss: 2.9195 | LR: 5.28e-05 | 1283.9ms/step | 15.6 samples/s +Step 34900/44000 | Loss: 2.9363 | LR: 5.27e-05 | 1281.9ms/step | 15.6 samples/s +Step 34910/44000 | Loss: 2.9340 | LR: 5.25e-05 | 1278.3ms/step | 15.6 samples/s +Step 34920/44000 | Loss: 2.9206 | LR: 5.24e-05 | 1283.2ms/step | 15.6 samples/s +Step 34930/44000 | Loss: 2.9068 | LR: 5.23e-05 | 1282.4ms/step | 15.6 samples/s +Step 34940/44000 | Loss: 2.8824 | LR: 5.22e-05 | 1281.0ms/step | 15.6 samples/s +Step 34950/44000 | Loss: 2.9391 | LR: 5.21e-05 | 1283.1ms/step | 15.6 samples/s +Step 34960/44000 | Loss: 2.9300 | LR: 5.20e-05 | 1283.2ms/step | 15.6 samples/s +Step 34970/44000 | Loss: 2.9103 | LR: 5.19e-05 | 1285.0ms/step | 15.6 samples/s +Step 34980/44000 | Loss: 2.9335 | LR: 5.18e-05 | 1283.6ms/step | 15.6 samples/s +Step 34990/44000 | Loss: 2.9355 | LR: 5.17e-05 | 1280.2ms/step | 15.6 samples/s +Step 35000/44000 | Loss: 2.9057 | LR: 5.15e-05 | 1281.9ms/step | 15.6 samples/s +Validation at step 35000: Loss = 2.9406, Perplexity = 18.93 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-35000 +Step 35010/44000 | Loss: 2.9295 | LR: 5.14e-05 | 3693.9ms/step | 5.4 samples/s +Step 35020/44000 | Loss: 2.8974 | LR: 5.13e-05 | 1284.2ms/step | 15.6 samples/s +Step 35030/44000 | Loss: 2.9060 | LR: 5.12e-05 | 1285.0ms/step | 15.6 samples/s +Step 35040/44000 | Loss: 2.9172 | LR: 5.11e-05 | 1285.3ms/step | 15.6 samples/s +Step 35050/44000 | Loss: 2.9091 | LR: 5.10e-05 | 1285.4ms/step | 15.6 samples/s +Step 35060/44000 | Loss: 2.9196 | LR: 5.09e-05 | 1281.7ms/step | 15.6 samples/s +Step 35070/44000 | Loss: 2.8879 | LR: 5.08e-05 | 1283.5ms/step | 15.6 samples/s +Step 35080/44000 | Loss: 2.9216 | LR: 5.07e-05 | 1283.6ms/step | 15.6 samples/s +Step 35090/44000 | Loss: 2.9110 | LR: 5.06e-05 | 1280.8ms/step | 15.6 samples/s +Step 35100/44000 | Loss: 2.9521 | LR: 5.04e-05 | 1279.1ms/step | 15.6 samples/s +Step 35110/44000 | Loss: 2.9157 | LR: 5.03e-05 | 1283.4ms/step | 15.6 samples/s +Step 35120/44000 | Loss: 2.9326 | LR: 5.02e-05 | 1282.2ms/step | 15.6 samples/s +Step 35130/44000 | Loss: 2.9293 | LR: 5.01e-05 | 1281.4ms/step | 15.6 samples/s +Step 35140/44000 | Loss: 2.8976 | LR: 5.00e-05 | 1282.4ms/step | 15.6 samples/s +Step 35150/44000 | Loss: 2.9373 | LR: 4.99e-05 | 1280.2ms/step | 15.6 samples/s +Step 35160/44000 | Loss: 2.9584 | LR: 4.98e-05 | 1279.8ms/step | 15.6 samples/s +Step 35170/44000 | Loss: 2.9362 | LR: 4.97e-05 | 1282.5ms/step | 15.6 samples/s +Step 35180/44000 | Loss: 2.9425 | LR: 4.96e-05 | 1281.9ms/step | 15.6 samples/s +Step 35190/44000 | Loss: 2.9116 | LR: 4.95e-05 | 1282.1ms/step | 15.6 samples/s +Step 35200/44000 | Loss: 2.9426 | LR: 4.94e-05 | 1284.9ms/step | 15.6 samples/s +Step 35210/44000 | Loss: 2.9378 | LR: 4.93e-05 | 1284.1ms/step | 15.6 samples/s +Step 35220/44000 | Loss: 2.9151 | LR: 4.91e-05 | 1285.7ms/step | 15.6 samples/s +Step 35230/44000 | Loss: 2.9234 | LR: 4.90e-05 | 1282.4ms/step | 15.6 samples/s +Step 35240/44000 | Loss: 2.9130 | LR: 4.89e-05 | 1285.1ms/step | 15.6 samples/s +Step 35250/44000 | Loss: 2.9596 | LR: 4.88e-05 | 1282.9ms/step | 15.6 samples/s +Validation at step 35250: Loss = 2.9404, Perplexity = 18.92 +Step 35260/44000 | Loss: 2.9164 | LR: 4.87e-05 | 3430.9ms/step | 5.8 samples/s +Step 35270/44000 | Loss: 2.9481 | LR: 4.86e-05 | 1281.6ms/step | 15.6 samples/s +Step 35280/44000 | Loss: 2.9031 | LR: 4.85e-05 | 1281.9ms/step | 15.6 samples/s +Step 35290/44000 | Loss: 2.9135 | LR: 4.84e-05 | 1282.6ms/step | 15.6 samples/s +Step 35300/44000 | Loss: 2.9153 | LR: 4.83e-05 | 1278.8ms/step | 15.6 samples/s +Step 35310/44000 | Loss: 2.9339 | LR: 4.82e-05 | 1283.3ms/step | 15.6 samples/s +Step 35320/44000 | Loss: 2.8973 | LR: 4.81e-05 | 1287.2ms/step | 15.5 samples/s +Step 35330/44000 | Loss: 2.9318 | LR: 4.80e-05 | 1282.3ms/step | 15.6 samples/s +Step 35340/44000 | Loss: 2.9279 | LR: 4.79e-05 | 1282.0ms/step | 15.6 samples/s +Step 35350/44000 | Loss: 2.9539 | LR: 4.77e-05 | 1276.3ms/step | 15.7 samples/s +Step 35360/44000 | Loss: 2.9097 | LR: 4.76e-05 | 1279.7ms/step | 15.6 samples/s +Step 35370/44000 | Loss: 2.9203 | LR: 4.75e-05 | 1282.8ms/step | 15.6 samples/s +Step 35380/44000 | Loss: 2.9087 | LR: 4.74e-05 | 1286.4ms/step | 15.5 samples/s +Step 35390/44000 | Loss: 2.9149 | LR: 4.73e-05 | 1282.1ms/step | 15.6 samples/s +Step 35400/44000 | Loss: 2.9033 | LR: 4.72e-05 | 1281.9ms/step | 15.6 samples/s +Step 35410/44000 | Loss: 2.9365 | LR: 4.71e-05 | 1280.1ms/step | 15.6 samples/s +Step 35420/44000 | Loss: 2.9077 | LR: 4.70e-05 | 1282.5ms/step | 15.6 samples/s +Step 35430/44000 | Loss: 2.9678 | LR: 4.69e-05 | 1282.1ms/step | 15.6 samples/s +Step 35440/44000 | Loss: 2.9477 | LR: 4.68e-05 | 1281.5ms/step | 15.6 samples/s +Step 35450/44000 | Loss: 2.9297 | LR: 4.67e-05 | 1281.9ms/step | 15.6 samples/s +Step 35460/44000 | Loss: 2.9485 | LR: 4.66e-05 | 1279.5ms/step | 15.6 samples/s +Step 35470/44000 | Loss: 2.9210 | LR: 4.65e-05 | 1278.4ms/step | 15.6 samples/s +Step 35480/44000 | Loss: 2.9291 | LR: 4.64e-05 | 1276.5ms/step | 15.7 samples/s +Step 35490/44000 | Loss: 2.9360 | LR: 4.63e-05 | 1279.1ms/step | 15.6 samples/s +Step 35500/44000 | Loss: 2.9082 | LR: 4.62e-05 | 1282.1ms/step | 15.6 samples/s +Validation at step 35500: Loss = 2.9402, Perplexity = 18.92 +Step 35510/44000 | Loss: 2.9200 | LR: 4.61e-05 | 3431.6ms/step | 5.8 samples/s +Step 35520/44000 | Loss: 2.9287 | LR: 4.59e-05 | 1281.5ms/step | 15.6 samples/s +Step 35530/44000 | Loss: 2.9278 | LR: 4.58e-05 | 1283.0ms/step | 15.6 samples/s +Step 35540/44000 | Loss: 2.9467 | LR: 4.57e-05 | 1282.4ms/step | 15.6 samples/s +Step 35550/44000 | Loss: 2.9481 | LR: 4.56e-05 | 1279.4ms/step | 15.6 samples/s +Step 35560/44000 | Loss: 2.9374 | LR: 4.55e-05 | 1283.5ms/step | 15.6 samples/s +Step 35570/44000 | Loss: 2.9491 | LR: 4.54e-05 | 1284.4ms/step | 15.6 samples/s +Step 35580/44000 | Loss: 2.9302 | LR: 4.53e-05 | 1286.6ms/step | 15.5 samples/s +Step 35590/44000 | Loss: 2.9411 | LR: 4.52e-05 | 1285.3ms/step | 15.6 samples/s +Step 35600/44000 | Loss: 2.9049 | LR: 4.51e-05 | 1284.4ms/step | 15.6 samples/s +Step 35610/44000 | Loss: 2.9434 | LR: 4.50e-05 | 1282.3ms/step | 15.6 samples/s +Step 35620/44000 | Loss: 2.9203 | LR: 4.49e-05 | 1280.1ms/step | 15.6 samples/s +Step 35630/44000 | Loss: 2.9322 | LR: 4.48e-05 | 1287.0ms/step | 15.5 samples/s +Step 35640/44000 | Loss: 2.9088 | LR: 4.47e-05 | 1289.3ms/step | 15.5 samples/s +Step 35650/44000 | Loss: 2.9175 | LR: 4.46e-05 | 1287.3ms/step | 15.5 samples/s +Step 35660/44000 | Loss: 2.9312 | LR: 4.45e-05 | 1286.2ms/step | 15.5 samples/s +Step 35670/44000 | Loss: 2.9210 | LR: 4.44e-05 | 1292.1ms/step | 15.5 samples/s +Step 35680/44000 | Loss: 2.9257 | LR: 4.43e-05 | 1284.6ms/step | 15.6 samples/s +Step 35690/44000 | Loss: 2.9361 | LR: 4.42e-05 | 1280.4ms/step | 15.6 samples/s +Step 35700/44000 | Loss: 2.9139 | LR: 4.41e-05 | 1288.2ms/step | 15.5 samples/s +Step 35710/44000 | Loss: 2.9283 | LR: 4.40e-05 | 1285.9ms/step | 15.6 samples/s +Step 35720/44000 | Loss: 2.9105 | LR: 4.39e-05 | 1285.0ms/step | 15.6 samples/s +Step 35730/44000 | Loss: 2.9014 | LR: 4.38e-05 | 1284.0ms/step | 15.6 samples/s +Step 35740/44000 | Loss: 2.9328 | LR: 4.37e-05 | 1283.1ms/step | 15.6 samples/s +Step 35750/44000 | Loss: 2.9133 | LR: 4.36e-05 | 1278.8ms/step | 15.6 samples/s +Validation at step 35750: Loss = 2.9397, Perplexity = 18.91 +Step 35760/44000 | Loss: 2.9432 | LR: 4.35e-05 | 3430.5ms/step | 5.8 samples/s +Step 35770/44000 | Loss: 2.9175 | LR: 4.34e-05 | 1283.1ms/step | 15.6 samples/s +Step 35780/44000 | Loss: 2.9081 | LR: 4.33e-05 | 1283.5ms/step | 15.6 samples/s +Step 35790/44000 | Loss: 2.9343 | LR: 4.32e-05 | 1283.6ms/step | 15.6 samples/s +Step 35800/44000 | Loss: 2.9424 | LR: 4.31e-05 | 1283.6ms/step | 15.6 samples/s +Step 35810/44000 | Loss: 2.9392 | LR: 4.29e-05 | 1283.6ms/step | 15.6 samples/s +Step 35820/44000 | Loss: 2.9409 | LR: 4.28e-05 | 1281.2ms/step | 15.6 samples/s +Step 35830/44000 | Loss: 2.9588 | LR: 4.27e-05 | 1280.3ms/step | 15.6 samples/s +Step 35840/44000 | Loss: 2.9195 | LR: 4.26e-05 | 1282.0ms/step | 15.6 samples/s +Step 35850/44000 | Loss: 2.9287 | LR: 4.25e-05 | 1281.8ms/step | 15.6 samples/s +Step 35860/44000 | Loss: 2.9511 | LR: 4.24e-05 | 1280.7ms/step | 15.6 samples/s +Step 35870/44000 | Loss: 2.9279 | LR: 4.23e-05 | 1280.3ms/step | 15.6 samples/s +Step 35880/44000 | Loss: 2.9447 | LR: 4.22e-05 | 1282.6ms/step | 15.6 samples/s +Step 35890/44000 | Loss: 2.9087 | LR: 4.21e-05 | 1280.2ms/step | 15.6 samples/s +Step 35900/44000 | Loss: 2.9312 | LR: 4.20e-05 | 1278.1ms/step | 15.6 samples/s +Step 35910/44000 | Loss: 2.9145 | LR: 4.19e-05 | 1280.9ms/step | 15.6 samples/s +Step 35920/44000 | Loss: 2.9278 | LR: 4.18e-05 | 1283.3ms/step | 15.6 samples/s +Step 35930/44000 | Loss: 2.9156 | LR: 4.17e-05 | 1282.9ms/step | 15.6 samples/s +Step 35940/44000 | Loss: 2.9177 | LR: 4.16e-05 | 1281.5ms/step | 15.6 samples/s +Step 35950/44000 | Loss: 2.9037 | LR: 4.15e-05 | 1281.6ms/step | 15.6 samples/s +Step 35960/44000 | Loss: 2.9089 | LR: 4.14e-05 | 1282.5ms/step | 15.6 samples/s +Step 35970/44000 | Loss: 2.9007 | LR: 4.13e-05 | 1283.0ms/step | 15.6 samples/s +Step 35980/44000 | Loss: 2.9538 | LR: 4.12e-05 | 1284.4ms/step | 15.6 samples/s +Step 35990/44000 | Loss: 2.9240 | LR: 4.11e-05 | 1285.0ms/step | 15.6 samples/s +Step 36000/44000 | Loss: 2.9246 | LR: 4.10e-05 | 1279.8ms/step | 15.6 samples/s +Validation at step 36000: Loss = 2.9398, Perplexity = 18.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-36000 +Step 36010/44000 | Loss: 2.9396 | LR: 4.09e-05 | 3714.0ms/step | 5.4 samples/s +Step 36020/44000 | Loss: 2.9173 | LR: 4.08e-05 | 1275.9ms/step | 15.7 samples/s +Step 36030/44000 | Loss: 2.9068 | LR: 4.07e-05 | 1283.3ms/step | 15.6 samples/s +Step 36040/44000 | Loss: 2.9132 | LR: 4.06e-05 | 1282.1ms/step | 15.6 samples/s +Step 36050/44000 | Loss: 2.9106 | LR: 4.05e-05 | 1284.7ms/step | 15.6 samples/s +Step 36060/44000 | Loss: 2.9398 | LR: 4.04e-05 | 1287.2ms/step | 15.5 samples/s +Step 36070/44000 | Loss: 2.9294 | LR: 4.03e-05 | 1282.4ms/step | 15.6 samples/s +Step 36080/44000 | Loss: 2.9429 | LR: 4.02e-05 | 1280.1ms/step | 15.6 samples/s +Step 36090/44000 | Loss: 2.9504 | LR: 4.01e-05 | 1284.4ms/step | 15.6 samples/s +Step 36100/44000 | Loss: 2.9290 | LR: 4.00e-05 | 1283.2ms/step | 15.6 samples/s +Step 36110/44000 | Loss: 2.9434 | LR: 3.99e-05 | 1285.1ms/step | 15.6 samples/s +Step 36120/44000 | Loss: 2.9311 | LR: 3.98e-05 | 1286.8ms/step | 15.5 samples/s +Step 36130/44000 | Loss: 2.9327 | LR: 3.97e-05 | 1287.3ms/step | 15.5 samples/s +Step 36140/44000 | Loss: 2.9249 | LR: 3.97e-05 | 1288.2ms/step | 15.5 samples/s +Step 36150/44000 | Loss: 2.9366 | LR: 3.96e-05 | 1286.7ms/step | 15.5 samples/s +Step 36160/44000 | Loss: 2.9280 | LR: 3.95e-05 | 1285.6ms/step | 15.6 samples/s +Step 36170/44000 | Loss: 2.8860 | LR: 3.94e-05 | 1284.9ms/step | 15.6 samples/s +Step 36180/44000 | Loss: 2.9267 | LR: 3.93e-05 | 1283.1ms/step | 15.6 samples/s +Step 36190/44000 | Loss: 2.9237 | LR: 3.92e-05 | 1285.3ms/step | 15.6 samples/s +Step 36200/44000 | Loss: 2.9155 | LR: 3.91e-05 | 1284.9ms/step | 15.6 samples/s +Step 36210/44000 | Loss: 2.9248 | LR: 3.90e-05 | 1288.5ms/step | 15.5 samples/s +Step 36220/44000 | Loss: 2.9198 | LR: 3.89e-05 | 1283.6ms/step | 15.6 samples/s +Step 36230/44000 | Loss: 2.9214 | LR: 3.88e-05 | 1280.9ms/step | 15.6 samples/s +Step 36240/44000 | Loss: 2.9313 | LR: 3.87e-05 | 1281.9ms/step | 15.6 samples/s +Step 36250/44000 | Loss: 2.9462 | LR: 3.86e-05 | 1282.3ms/step | 15.6 samples/s +Validation at step 36250: Loss = 2.9398, Perplexity = 18.91 +Step 36260/44000 | Loss: 2.9366 | LR: 3.85e-05 | 3437.9ms/step | 5.8 samples/s +Step 36270/44000 | Loss: 2.9475 | LR: 3.84e-05 | 1282.9ms/step | 15.6 samples/s +Step 36280/44000 | Loss: 2.9199 | LR: 3.83e-05 | 1282.8ms/step | 15.6 samples/s +Step 36290/44000 | Loss: 2.9347 | LR: 3.82e-05 | 1283.6ms/step | 15.6 samples/s +Step 36300/44000 | Loss: 2.9187 | LR: 3.81e-05 | 1283.6ms/step | 15.6 samples/s +Step 36310/44000 | Loss: 2.9196 | LR: 3.80e-05 | 1282.6ms/step | 15.6 samples/s +Step 36320/44000 | Loss: 2.8971 | LR: 3.79e-05 | 1284.6ms/step | 15.6 samples/s +Step 36330/44000 | Loss: 2.9137 | LR: 3.78e-05 | 1285.4ms/step | 15.6 samples/s +Step 36340/44000 | Loss: 2.9313 | LR: 3.77e-05 | 1286.9ms/step | 15.5 samples/s +Step 36350/44000 | Loss: 2.9013 | LR: 3.76e-05 | 1285.9ms/step | 15.6 samples/s +Step 36360/44000 | Loss: 2.9112 | LR: 3.75e-05 | 1282.8ms/step | 15.6 samples/s +Step 36370/44000 | Loss: 2.9212 | LR: 3.74e-05 | 1280.8ms/step | 15.6 samples/s +Step 36380/44000 | Loss: 2.9195 | LR: 3.73e-05 | 1278.8ms/step | 15.6 samples/s +Step 36390/44000 | Loss: 2.9420 | LR: 3.72e-05 | 1277.6ms/step | 15.7 samples/s +Step 36400/44000 | Loss: 2.9187 | LR: 3.71e-05 | 1278.3ms/step | 15.6 samples/s +Step 36410/44000 | Loss: 2.9286 | LR: 3.70e-05 | 1279.3ms/step | 15.6 samples/s +Step 36420/44000 | Loss: 2.8986 | LR: 3.69e-05 | 1279.4ms/step | 15.6 samples/s +Step 36430/44000 | Loss: 2.9054 | LR: 3.69e-05 | 1279.4ms/step | 15.6 samples/s +Step 36440/44000 | Loss: 2.9153 | LR: 3.68e-05 | 1284.9ms/step | 15.6 samples/s +Step 36450/44000 | Loss: 2.9557 | LR: 3.67e-05 | 1282.2ms/step | 15.6 samples/s +Step 36460/44000 | Loss: 2.9306 | LR: 3.66e-05 | 1282.7ms/step | 15.6 samples/s +Step 36470/44000 | Loss: 2.9048 | LR: 3.65e-05 | 1283.4ms/step | 15.6 samples/s +Step 36480/44000 | Loss: 2.9283 | LR: 3.64e-05 | 1283.9ms/step | 15.6 samples/s +Step 36490/44000 | Loss: 2.9501 | LR: 3.63e-05 | 1282.8ms/step | 15.6 samples/s +Step 36500/44000 | Loss: 2.8997 | LR: 3.62e-05 | 1281.3ms/step | 15.6 samples/s +Validation at step 36500: Loss = 2.9396, Perplexity = 18.91 +Step 36510/44000 | Loss: 2.9384 | LR: 3.61e-05 | 3438.3ms/step | 5.8 samples/s +Step 36520/44000 | Loss: 2.9129 | LR: 3.60e-05 | 1283.2ms/step | 15.6 samples/s +Step 36530/44000 | Loss: 2.9214 | LR: 3.59e-05 | 1285.9ms/step | 15.6 samples/s +Step 36540/44000 | Loss: 2.9088 | LR: 3.58e-05 | 1286.4ms/step | 15.5 samples/s +Step 36550/44000 | Loss: 2.9170 | LR: 3.57e-05 | 1283.6ms/step | 15.6 samples/s +Step 36560/44000 | Loss: 2.9373 | LR: 3.56e-05 | 1284.8ms/step | 15.6 samples/s +Step 36570/44000 | Loss: 2.8984 | LR: 3.55e-05 | 1285.2ms/step | 15.6 samples/s +Step 36580/44000 | Loss: 2.9149 | LR: 3.54e-05 | 1284.4ms/step | 15.6 samples/s +Step 36590/44000 | Loss: 2.9264 | LR: 3.53e-05 | 1282.7ms/step | 15.6 samples/s +Step 36600/44000 | Loss: 2.8989 | LR: 3.53e-05 | 1285.8ms/step | 15.6 samples/s +Step 36610/44000 | Loss: 2.9212 | LR: 3.52e-05 | 1283.6ms/step | 15.6 samples/s +Step 36620/44000 | Loss: 2.9112 | LR: 3.51e-05 | 1281.4ms/step | 15.6 samples/s +Step 36630/44000 | Loss: 2.9258 | LR: 3.50e-05 | 1282.5ms/step | 15.6 samples/s +Step 36640/44000 | Loss: 2.9238 | LR: 3.49e-05 | 1281.8ms/step | 15.6 samples/s +Step 36650/44000 | Loss: 2.9195 | LR: 3.48e-05 | 1281.0ms/step | 15.6 samples/s +Step 36660/44000 | Loss: 2.9124 | LR: 3.47e-05 | 1278.4ms/step | 15.6 samples/s +Step 36670/44000 | Loss: 2.9528 | LR: 3.46e-05 | 1280.3ms/step | 15.6 samples/s +Step 36680/44000 | Loss: 2.9394 | LR: 3.45e-05 | 1280.5ms/step | 15.6 samples/s +Step 36690/44000 | Loss: 2.9163 | LR: 3.44e-05 | 1281.6ms/step | 15.6 samples/s +Step 36700/44000 | Loss: 2.9178 | LR: 3.43e-05 | 1283.8ms/step | 15.6 samples/s +Step 36710/44000 | Loss: 2.9104 | LR: 3.42e-05 | 1282.6ms/step | 15.6 samples/s +Step 36720/44000 | Loss: 2.9226 | LR: 3.41e-05 | 1282.1ms/step | 15.6 samples/s +Step 36730/44000 | Loss: 2.9132 | LR: 3.41e-05 | 1281.0ms/step | 15.6 samples/s +Step 36740/44000 | Loss: 2.9367 | LR: 3.40e-05 | 1279.6ms/step | 15.6 samples/s +Step 36750/44000 | Loss: 2.9066 | LR: 3.39e-05 | 1282.3ms/step | 15.6 samples/s +Validation at step 36750: Loss = 2.9392, Perplexity = 18.90 +Step 36760/44000 | Loss: 2.9147 | LR: 3.38e-05 | 3436.3ms/step | 5.8 samples/s +Step 36770/44000 | Loss: 2.9265 | LR: 3.37e-05 | 1281.3ms/step | 15.6 samples/s +Step 36780/44000 | Loss: 2.9348 | LR: 3.36e-05 | 1282.2ms/step | 15.6 samples/s +Step 36790/44000 | Loss: 2.9335 | LR: 3.35e-05 | 1283.8ms/step | 15.6 samples/s +Step 36800/44000 | Loss: 2.9237 | LR: 3.34e-05 | 1283.0ms/step | 15.6 samples/s +Step 36810/44000 | Loss: 2.9186 | LR: 3.33e-05 | 1283.4ms/step | 15.6 samples/s +Step 36820/44000 | Loss: 2.9198 | LR: 3.32e-05 | 1283.9ms/step | 15.6 samples/s +Step 36830/44000 | Loss: 2.9199 | LR: 3.31e-05 | 1282.6ms/step | 15.6 samples/s +Step 36840/44000 | Loss: 2.9251 | LR: 3.31e-05 | 1283.1ms/step | 15.6 samples/s +Step 36850/44000 | Loss: 2.9257 | LR: 3.30e-05 | 1282.6ms/step | 15.6 samples/s +Step 36860/44000 | Loss: 2.9066 | LR: 3.29e-05 | 1284.9ms/step | 15.6 samples/s +Step 36870/44000 | Loss: 2.9264 | LR: 3.28e-05 | 1278.9ms/step | 15.6 samples/s +Step 36880/44000 | Loss: 2.8996 | LR: 3.27e-05 | 1282.5ms/step | 15.6 samples/s +Step 36890/44000 | Loss: 2.8966 | LR: 3.26e-05 | 1281.2ms/step | 15.6 samples/s +Step 36900/44000 | Loss: 2.9009 | LR: 3.25e-05 | 1282.2ms/step | 15.6 samples/s +Step 36910/44000 | Loss: 2.9181 | LR: 3.24e-05 | 1282.7ms/step | 15.6 samples/s +Step 36920/44000 | Loss: 2.9167 | LR: 3.23e-05 | 1281.4ms/step | 15.6 samples/s +Step 36930/44000 | Loss: 2.8946 | LR: 3.22e-05 | 1280.5ms/step | 15.6 samples/s +Step 36940/44000 | Loss: 2.9345 | LR: 3.22e-05 | 1280.8ms/step | 15.6 samples/s +Step 36950/44000 | Loss: 2.9302 | LR: 3.21e-05 | 1282.4ms/step | 15.6 samples/s +Step 36960/44000 | Loss: 2.9133 | LR: 3.20e-05 | 1284.6ms/step | 15.6 samples/s +Step 36970/44000 | Loss: 2.8951 | LR: 3.19e-05 | 1283.8ms/step | 15.6 samples/s +Step 36980/44000 | Loss: 2.9170 | LR: 3.18e-05 | 1281.6ms/step | 15.6 samples/s +Step 36990/44000 | Loss: 2.9285 | LR: 3.17e-05 | 1285.8ms/step | 15.6 samples/s +Step 37000/44000 | Loss: 2.9392 | LR: 3.16e-05 | 1285.3ms/step | 15.6 samples/s +Validation at step 37000: Loss = 2.9392, Perplexity = 18.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-37000 +Step 37010/44000 | Loss: 2.9356 | LR: 3.15e-05 | 3710.0ms/step | 5.4 samples/s +Step 37020/44000 | Loss: 2.9736 | LR: 3.15e-05 | 1280.7ms/step | 15.6 samples/s +Step 37030/44000 | Loss: 2.9387 | LR: 3.14e-05 | 1281.9ms/step | 15.6 samples/s +Step 37040/44000 | Loss: 2.9365 | LR: 3.13e-05 | 1280.6ms/step | 15.6 samples/s +Step 37050/44000 | Loss: 2.9268 | LR: 3.12e-05 | 1279.9ms/step | 15.6 samples/s +Step 37060/44000 | Loss: 2.8954 | LR: 3.11e-05 | 1281.9ms/step | 15.6 samples/s +Step 37070/44000 | Loss: 2.9269 | LR: 3.10e-05 | 1284.9ms/step | 15.6 samples/s +Step 37080/44000 | Loss: 2.9573 | LR: 3.09e-05 | 1286.2ms/step | 15.5 samples/s +Step 37090/44000 | Loss: 2.9351 | LR: 3.08e-05 | 1283.0ms/step | 15.6 samples/s +Step 37100/44000 | Loss: 2.9198 | LR: 3.07e-05 | 1283.8ms/step | 15.6 samples/s +Step 37110/44000 | Loss: 2.9133 | LR: 3.07e-05 | 1281.3ms/step | 15.6 samples/s +Step 37120/44000 | Loss: 2.9364 | LR: 3.06e-05 | 1281.9ms/step | 15.6 samples/s +Step 37130/44000 | Loss: 2.9281 | LR: 3.05e-05 | 1280.7ms/step | 15.6 samples/s +Step 37140/44000 | Loss: 2.9213 | LR: 3.04e-05 | 1279.4ms/step | 15.6 samples/s +Step 37150/44000 | Loss: 2.8835 | LR: 3.03e-05 | 1282.5ms/step | 15.6 samples/s +Step 37160/44000 | Loss: 2.9332 | LR: 3.02e-05 | 1283.0ms/step | 15.6 samples/s +Step 37170/44000 | Loss: 2.9648 | LR: 3.01e-05 | 1281.9ms/step | 15.6 samples/s +Step 37180/44000 | Loss: 2.9173 | LR: 3.01e-05 | 1281.4ms/step | 15.6 samples/s +Step 37190/44000 | Loss: 2.9207 | LR: 3.00e-05 | 1283.6ms/step | 15.6 samples/s +Step 37200/44000 | Loss: 2.9267 | LR: 2.99e-05 | 1286.4ms/step | 15.5 samples/s +Step 37210/44000 | Loss: 2.9405 | LR: 2.98e-05 | 1277.6ms/step | 15.7 samples/s +Step 37220/44000 | Loss: 2.9211 | LR: 2.97e-05 | 1278.2ms/step | 15.6 samples/s +Step 37230/44000 | Loss: 2.9492 | LR: 2.96e-05 | 1277.9ms/step | 15.7 samples/s +Step 37240/44000 | Loss: 2.9255 | LR: 2.95e-05 | 1278.3ms/step | 15.6 samples/s +Step 37250/44000 | Loss: 2.9181 | LR: 2.95e-05 | 1278.8ms/step | 15.6 samples/s +Validation at step 37250: Loss = 2.9392, Perplexity = 18.90 +Step 37260/44000 | Loss: 2.9313 | LR: 2.94e-05 | 3436.7ms/step | 5.8 samples/s +Step 37270/44000 | Loss: 2.9424 | LR: 2.93e-05 | 1282.8ms/step | 15.6 samples/s +Step 37280/44000 | Loss: 2.9232 | LR: 2.92e-05 | 1284.1ms/step | 15.6 samples/s +Step 37290/44000 | Loss: 2.9312 | LR: 2.91e-05 | 1283.2ms/step | 15.6 samples/s +Step 37300/44000 | Loss: 2.9185 | LR: 2.90e-05 | 1280.0ms/step | 15.6 samples/s +Step 37310/44000 | Loss: 2.9222 | LR: 2.89e-05 | 1279.7ms/step | 15.6 samples/s +Step 37320/44000 | Loss: 2.9258 | LR: 2.89e-05 | 1279.9ms/step | 15.6 samples/s +Step 37330/44000 | Loss: 2.9500 | LR: 2.88e-05 | 1280.9ms/step | 15.6 samples/s +Step 37340/44000 | Loss: 2.9221 | LR: 2.87e-05 | 1283.8ms/step | 15.6 samples/s +Step 37350/44000 | Loss: 2.9416 | LR: 2.86e-05 | 1278.3ms/step | 15.6 samples/s +Step 37360/44000 | Loss: 2.9264 | LR: 2.85e-05 | 1276.0ms/step | 15.7 samples/s +Step 37370/44000 | Loss: 2.9275 | LR: 2.84e-05 | 1281.0ms/step | 15.6 samples/s +Step 37380/44000 | Loss: 2.9101 | LR: 2.84e-05 | 1282.4ms/step | 15.6 samples/s +Step 37390/44000 | Loss: 2.9085 | LR: 2.83e-05 | 1282.0ms/step | 15.6 samples/s +Step 37400/44000 | Loss: 2.9133 | LR: 2.82e-05 | 1284.0ms/step | 15.6 samples/s +Step 37410/44000 | Loss: 2.9096 | LR: 2.81e-05 | 1278.4ms/step | 15.6 samples/s +Step 37420/44000 | Loss: 2.9375 | LR: 2.80e-05 | 1278.6ms/step | 15.6 samples/s +Step 37430/44000 | Loss: 2.8986 | LR: 2.79e-05 | 1279.4ms/step | 15.6 samples/s +Step 37440/44000 | Loss: 2.9109 | LR: 2.78e-05 | 1281.1ms/step | 15.6 samples/s +Step 37450/44000 | Loss: 2.9256 | LR: 2.78e-05 | 1283.8ms/step | 15.6 samples/s +Step 37460/44000 | Loss: 2.9399 | LR: 2.77e-05 | 1284.2ms/step | 15.6 samples/s +Step 37470/44000 | Loss: 2.9235 | LR: 2.76e-05 | 1283.6ms/step | 15.6 samples/s +Step 37480/44000 | Loss: 2.9346 | LR: 2.75e-05 | 1283.5ms/step | 15.6 samples/s +Step 37490/44000 | Loss: 2.9193 | LR: 2.74e-05 | 1286.9ms/step | 15.5 samples/s +Step 37500/44000 | Loss: 2.9239 | LR: 2.74e-05 | 1282.2ms/step | 15.6 samples/s +Validation at step 37500: Loss = 2.9392, Perplexity = 18.90 +Step 37510/44000 | Loss: 2.9443 | LR: 2.73e-05 | 3435.7ms/step | 5.8 samples/s +Step 37520/44000 | Loss: 2.8988 | LR: 2.72e-05 | 1277.0ms/step | 15.7 samples/s +Step 37530/44000 | Loss: 2.9637 | LR: 2.71e-05 | 1279.8ms/step | 15.6 samples/s +Step 37540/44000 | Loss: 2.9332 | LR: 2.70e-05 | 1278.1ms/step | 15.6 samples/s +Step 37550/44000 | Loss: 2.9262 | LR: 2.69e-05 | 1282.8ms/step | 15.6 samples/s +Step 37560/44000 | Loss: 2.9435 | LR: 2.69e-05 | 1280.2ms/step | 15.6 samples/s +Step 37570/44000 | Loss: 2.9227 | LR: 2.68e-05 | 1277.9ms/step | 15.7 samples/s +Step 37580/44000 | Loss: 2.9145 | LR: 2.67e-05 | 1278.2ms/step | 15.6 samples/s +Step 37590/44000 | Loss: 2.9165 | LR: 2.66e-05 | 1282.7ms/step | 15.6 samples/s +Step 37600/44000 | Loss: 2.9212 | LR: 2.65e-05 | 1276.9ms/step | 15.7 samples/s +Step 37610/44000 | Loss: 2.9132 | LR: 2.65e-05 | 1277.1ms/step | 15.7 samples/s +Step 37620/44000 | Loss: 2.9360 | LR: 2.64e-05 | 1278.8ms/step | 15.6 samples/s +Step 37630/44000 | Loss: 2.8903 | LR: 2.63e-05 | 1282.6ms/step | 15.6 samples/s +Step 37640/44000 | Loss: 2.9401 | LR: 2.62e-05 | 1280.8ms/step | 15.6 samples/s +Step 37650/44000 | Loss: 2.9111 | LR: 2.61e-05 | 1279.5ms/step | 15.6 samples/s +Step 37660/44000 | Loss: 2.9480 | LR: 2.60e-05 | 1280.3ms/step | 15.6 samples/s +Step 37670/44000 | Loss: 2.9301 | LR: 2.60e-05 | 1282.0ms/step | 15.6 samples/s +Step 37680/44000 | Loss: 2.9402 | LR: 2.59e-05 | 1284.1ms/step | 15.6 samples/s +Step 37690/44000 | Loss: 2.9164 | LR: 2.58e-05 | 1281.9ms/step | 15.6 samples/s +Step 37700/44000 | Loss: 2.9138 | LR: 2.57e-05 | 1280.8ms/step | 15.6 samples/s +Step 37710/44000 | Loss: 2.9067 | LR: 2.56e-05 | 1279.6ms/step | 15.6 samples/s +Step 37720/44000 | Loss: 2.9413 | LR: 2.56e-05 | 1279.2ms/step | 15.6 samples/s +Step 37730/44000 | Loss: 2.9113 | LR: 2.55e-05 | 1284.5ms/step | 15.6 samples/s +Step 37740/44000 | Loss: 2.9233 | LR: 2.54e-05 | 1279.9ms/step | 15.6 samples/s +Step 37750/44000 | Loss: 2.9081 | LR: 2.53e-05 | 1284.7ms/step | 15.6 samples/s +Validation at step 37750: Loss = 2.9392, Perplexity = 18.90 +Step 37760/44000 | Loss: 2.9293 | LR: 2.52e-05 | 3435.9ms/step | 5.8 samples/s +Step 37770/44000 | Loss: 2.9036 | LR: 2.52e-05 | 1283.1ms/step | 15.6 samples/s +Step 37780/44000 | Loss: 2.9104 | LR: 2.51e-05 | 1282.3ms/step | 15.6 samples/s +Step 37790/44000 | Loss: 2.9443 | LR: 2.50e-05 | 1282.8ms/step | 15.6 samples/s +Step 37800/44000 | Loss: 2.9151 | LR: 2.49e-05 | 1284.2ms/step | 15.6 samples/s +Step 37810/44000 | Loss: 2.8886 | LR: 2.48e-05 | 1279.9ms/step | 15.6 samples/s +Step 37820/44000 | Loss: 2.9466 | LR: 2.48e-05 | 1279.4ms/step | 15.6 samples/s +Step 37830/44000 | Loss: 2.9460 | LR: 2.47e-05 | 1276.9ms/step | 15.7 samples/s +Step 37840/44000 | Loss: 2.9307 | LR: 2.46e-05 | 1283.7ms/step | 15.6 samples/s +Step 37850/44000 | Loss: 2.9118 | LR: 2.45e-05 | 1280.2ms/step | 15.6 samples/s +Step 37860/44000 | Loss: 2.8903 | LR: 2.45e-05 | 1278.2ms/step | 15.6 samples/s +Step 37870/44000 | Loss: 2.9153 | LR: 2.44e-05 | 1280.5ms/step | 15.6 samples/s +Step 37880/44000 | Loss: 2.9127 | LR: 2.43e-05 | 1282.9ms/step | 15.6 samples/s +Step 37890/44000 | Loss: 2.9369 | LR: 2.42e-05 | 1280.9ms/step | 15.6 samples/s +Step 37900/44000 | Loss: 2.9397 | LR: 2.41e-05 | 1281.1ms/step | 15.6 samples/s +Step 37910/44000 | Loss: 2.9220 | LR: 2.41e-05 | 1279.6ms/step | 15.6 samples/s +Step 37920/44000 | Loss: 2.9430 | LR: 2.40e-05 | 1280.0ms/step | 15.6 samples/s +Step 37930/44000 | Loss: 2.9208 | LR: 2.39e-05 | 1280.1ms/step | 15.6 samples/s +Step 37940/44000 | Loss: 2.9315 | LR: 2.38e-05 | 1280.8ms/step | 15.6 samples/s +Step 37950/44000 | Loss: 2.9460 | LR: 2.38e-05 | 1283.2ms/step | 15.6 samples/s +Step 37960/44000 | Loss: 2.9635 | LR: 2.37e-05 | 1285.5ms/step | 15.6 samples/s +Step 37970/44000 | Loss: 2.9206 | LR: 2.36e-05 | 1283.4ms/step | 15.6 samples/s +Step 37980/44000 | Loss: 2.9531 | LR: 2.35e-05 | 1284.5ms/step | 15.6 samples/s +Step 37990/44000 | Loss: 2.9199 | LR: 2.34e-05 | 1282.3ms/step | 15.6 samples/s +Step 38000/44000 | Loss: 2.9244 | LR: 2.34e-05 | 1280.7ms/step | 15.6 samples/s +Validation at step 38000: Loss = 2.9388, Perplexity = 18.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-38000 +Step 38010/44000 | Loss: 2.8763 | LR: 2.33e-05 | 3713.9ms/step | 5.4 samples/s +Step 38020/44000 | Loss: 2.9097 | LR: 2.32e-05 | 1282.3ms/step | 15.6 samples/s +Step 38030/44000 | Loss: 2.8770 | LR: 2.31e-05 | 1284.3ms/step | 15.6 samples/s +Step 38040/44000 | Loss: 2.9481 | LR: 2.31e-05 | 1279.0ms/step | 15.6 samples/s +Step 38050/44000 | Loss: 2.9325 | LR: 2.30e-05 | 1280.5ms/step | 15.6 samples/s +Step 38060/44000 | Loss: 2.9364 | LR: 2.29e-05 | 1281.9ms/step | 15.6 samples/s +Step 38070/44000 | Loss: 2.9097 | LR: 2.28e-05 | 1282.4ms/step | 15.6 samples/s +Step 38080/44000 | Loss: 2.9236 | LR: 2.28e-05 | 1281.3ms/step | 15.6 samples/s +Step 38090/44000 | Loss: 2.9356 | LR: 2.27e-05 | 1283.4ms/step | 15.6 samples/s +Step 38100/44000 | Loss: 2.9177 | LR: 2.26e-05 | 1281.0ms/step | 15.6 samples/s +Step 38110/44000 | Loss: 2.9295 | LR: 2.25e-05 | 1283.7ms/step | 15.6 samples/s +Step 38120/44000 | Loss: 2.9410 | LR: 2.25e-05 | 1283.6ms/step | 15.6 samples/s +Step 38130/44000 | Loss: 2.9170 | LR: 2.24e-05 | 1283.0ms/step | 15.6 samples/s +Step 38140/44000 | Loss: 2.9425 | LR: 2.23e-05 | 1282.4ms/step | 15.6 samples/s +Step 38150/44000 | Loss: 2.9606 | LR: 2.22e-05 | 1284.7ms/step | 15.6 samples/s +Step 38160/44000 | Loss: 2.9647 | LR: 2.22e-05 | 1280.2ms/step | 15.6 samples/s +Step 38170/44000 | Loss: 2.9296 | LR: 2.21e-05 | 1281.9ms/step | 15.6 samples/s +Step 38180/44000 | Loss: 2.9561 | LR: 2.20e-05 | 1279.5ms/step | 15.6 samples/s +Step 38190/44000 | Loss: 2.8993 | LR: 2.19e-05 | 1281.1ms/step | 15.6 samples/s +Step 38200/44000 | Loss: 2.9292 | LR: 2.19e-05 | 1284.0ms/step | 15.6 samples/s +Step 38210/44000 | Loss: 2.9276 | LR: 2.18e-05 | 1284.2ms/step | 15.6 samples/s +Step 38220/44000 | Loss: 2.9462 | LR: 2.17e-05 | 1283.5ms/step | 15.6 samples/s +Step 38230/44000 | Loss: 2.9315 | LR: 2.16e-05 | 1281.9ms/step | 15.6 samples/s +Step 38240/44000 | Loss: 2.9330 | LR: 2.16e-05 | 1281.7ms/step | 15.6 samples/s +Step 38250/44000 | Loss: 2.9321 | LR: 2.15e-05 | 1282.5ms/step | 15.6 samples/s +Validation at step 38250: Loss = 2.9389, Perplexity = 18.90 +Step 38260/44000 | Loss: 2.9057 | LR: 2.14e-05 | 3433.3ms/step | 5.8 samples/s +Step 38270/44000 | Loss: 2.9367 | LR: 2.13e-05 | 1280.9ms/step | 15.6 samples/s +Step 38280/44000 | Loss: 2.8952 | LR: 2.13e-05 | 1284.1ms/step | 15.6 samples/s +Step 38290/44000 | Loss: 2.9042 | LR: 2.12e-05 | 1285.5ms/step | 15.6 samples/s +Step 38300/44000 | Loss: 2.9365 | LR: 2.11e-05 | 1279.7ms/step | 15.6 samples/s +Step 38310/44000 | Loss: 2.9088 | LR: 2.11e-05 | 1277.6ms/step | 15.7 samples/s +Step 38320/44000 | Loss: 2.9248 | LR: 2.10e-05 | 1279.9ms/step | 15.6 samples/s +Step 38330/44000 | Loss: 2.9432 | LR: 2.09e-05 | 1281.6ms/step | 15.6 samples/s +Step 38340/44000 | Loss: 2.8921 | LR: 2.08e-05 | 1285.6ms/step | 15.6 samples/s +Step 38350/44000 | Loss: 2.9216 | LR: 2.08e-05 | 1283.4ms/step | 15.6 samples/s +Step 38360/44000 | Loss: 2.9293 | LR: 2.07e-05 | 1281.8ms/step | 15.6 samples/s +Step 38370/44000 | Loss: 2.9374 | LR: 2.06e-05 | 1282.4ms/step | 15.6 samples/s +Step 38380/44000 | Loss: 2.9403 | LR: 2.05e-05 | 1283.9ms/step | 15.6 samples/s +Step 38390/44000 | Loss: 2.9204 | LR: 2.05e-05 | 1284.6ms/step | 15.6 samples/s +Step 38400/44000 | Loss: 2.9393 | LR: 2.04e-05 | 1283.5ms/step | 15.6 samples/s +Step 38410/44000 | Loss: 2.9222 | LR: 2.03e-05 | 1285.3ms/step | 15.6 samples/s +Step 38420/44000 | Loss: 2.9320 | LR: 2.03e-05 | 1283.5ms/step | 15.6 samples/s +Step 38430/44000 | Loss: 2.9377 | LR: 2.02e-05 | 1283.4ms/step | 15.6 samples/s +Step 38440/44000 | Loss: 2.8906 | LR: 2.01e-05 | 1285.1ms/step | 15.6 samples/s +Step 38450/44000 | Loss: 2.9263 | LR: 2.00e-05 | 1286.8ms/step | 15.5 samples/s +Step 38460/44000 | Loss: 2.9260 | LR: 2.00e-05 | 1286.3ms/step | 15.5 samples/s +Step 38470/44000 | Loss: 2.9213 | LR: 1.99e-05 | 1283.2ms/step | 15.6 samples/s +Step 38480/44000 | Loss: 2.9455 | LR: 1.98e-05 | 1284.1ms/step | 15.6 samples/s +Step 38490/44000 | Loss: 2.9321 | LR: 1.98e-05 | 1285.7ms/step | 15.6 samples/s +Step 38500/44000 | Loss: 2.9337 | LR: 1.97e-05 | 1282.9ms/step | 15.6 samples/s +Validation at step 38500: Loss = 2.9388, Perplexity = 18.89 +Step 38510/44000 | Loss: 2.9259 | LR: 1.96e-05 | 3426.6ms/step | 5.8 samples/s +Step 38520/44000 | Loss: 2.9032 | LR: 1.95e-05 | 1278.1ms/step | 15.6 samples/s +Step 38530/44000 | Loss: 2.9250 | LR: 1.95e-05 | 1280.6ms/step | 15.6 samples/s +Step 38540/44000 | Loss: 2.9362 | LR: 1.94e-05 | 1281.8ms/step | 15.6 samples/s +Step 38550/44000 | Loss: 2.9457 | LR: 1.93e-05 | 1280.9ms/step | 15.6 samples/s +Step 38560/44000 | Loss: 2.9298 | LR: 1.93e-05 | 1282.9ms/step | 15.6 samples/s +Step 38570/44000 | Loss: 2.9203 | LR: 1.92e-05 | 1282.7ms/step | 15.6 samples/s +Step 38580/44000 | Loss: 2.9029 | LR: 1.91e-05 | 1281.9ms/step | 15.6 samples/s +Step 38590/44000 | Loss: 2.9304 | LR: 1.91e-05 | 1282.5ms/step | 15.6 samples/s +Step 38600/44000 | Loss: 2.9083 | LR: 1.90e-05 | 1280.3ms/step | 15.6 samples/s +Step 38610/44000 | Loss: 2.9141 | LR: 1.89e-05 | 1282.9ms/step | 15.6 samples/s +Step 38620/44000 | Loss: 2.9331 | LR: 1.88e-05 | 1283.6ms/step | 15.6 samples/s +Step 38630/44000 | Loss: 2.9097 | LR: 1.88e-05 | 1282.8ms/step | 15.6 samples/s +Step 38640/44000 | Loss: 2.9484 | LR: 1.87e-05 | 1282.2ms/step | 15.6 samples/s +Step 38650/44000 | Loss: 2.9046 | LR: 1.86e-05 | 1283.5ms/step | 15.6 samples/s +Step 38660/44000 | Loss: 2.9142 | LR: 1.86e-05 | 1281.8ms/step | 15.6 samples/s +Step 38670/44000 | Loss: 2.9282 | LR: 1.85e-05 | 1284.4ms/step | 15.6 samples/s +Step 38680/44000 | Loss: 2.9523 | LR: 1.84e-05 | 1282.3ms/step | 15.6 samples/s +Step 38690/44000 | Loss: 2.9263 | LR: 1.84e-05 | 1281.6ms/step | 15.6 samples/s +Step 38700/44000 | Loss: 2.9447 | LR: 1.83e-05 | 1279.6ms/step | 15.6 samples/s +Step 38710/44000 | Loss: 2.9296 | LR: 1.82e-05 | 1279.7ms/step | 15.6 samples/s +Step 38720/44000 | Loss: 2.9026 | LR: 1.82e-05 | 1278.2ms/step | 15.6 samples/s +Step 38730/44000 | Loss: 2.9200 | LR: 1.81e-05 | 1277.9ms/step | 15.7 samples/s +Step 38740/44000 | Loss: 2.9408 | LR: 1.80e-05 | 1282.5ms/step | 15.6 samples/s +Step 38750/44000 | Loss: 2.9219 | LR: 1.80e-05 | 1280.2ms/step | 15.6 samples/s +Validation at step 38750: Loss = 2.9389, Perplexity = 18.89 +Step 38760/44000 | Loss: 2.9319 | LR: 1.79e-05 | 3428.1ms/step | 5.8 samples/s +Step 38770/44000 | Loss: 2.9205 | LR: 1.78e-05 | 1279.9ms/step | 15.6 samples/s +Step 38780/44000 | Loss: 2.9116 | LR: 1.78e-05 | 1281.5ms/step | 15.6 samples/s +Step 38790/44000 | Loss: 2.9446 | LR: 1.77e-05 | 1284.5ms/step | 15.6 samples/s +Step 38800/44000 | Loss: 2.9107 | LR: 1.76e-05 | 1281.4ms/step | 15.6 samples/s +Step 38810/44000 | Loss: 2.8929 | LR: 1.76e-05 | 1284.3ms/step | 15.6 samples/s +Step 38820/44000 | Loss: 2.9299 | LR: 1.75e-05 | 1283.7ms/step | 15.6 samples/s +Step 38830/44000 | Loss: 2.9307 | LR: 1.74e-05 | 1284.5ms/step | 15.6 samples/s +Step 38840/44000 | Loss: 2.8948 | LR: 1.74e-05 | 1276.8ms/step | 15.7 samples/s +Step 38850/44000 | Loss: 2.9325 | LR: 1.73e-05 | 1276.4ms/step | 15.7 samples/s +Step 38860/44000 | Loss: 2.9206 | LR: 1.72e-05 | 1276.0ms/step | 15.7 samples/s +Step 38870/44000 | Loss: 2.9370 | LR: 1.72e-05 | 1282.9ms/step | 15.6 samples/s +Step 38880/44000 | Loss: 2.9371 | LR: 1.71e-05 | 1283.2ms/step | 15.6 samples/s +Step 38890/44000 | Loss: 2.9246 | LR: 1.70e-05 | 1283.5ms/step | 15.6 samples/s +Step 38900/44000 | Loss: 2.9218 | LR: 1.70e-05 | 1280.7ms/step | 15.6 samples/s +Step 38910/44000 | Loss: 2.9444 | LR: 1.69e-05 | 1280.2ms/step | 15.6 samples/s +Step 38920/44000 | Loss: 2.9256 | LR: 1.68e-05 | 1281.5ms/step | 15.6 samples/s +Step 38930/44000 | Loss: 2.9040 | LR: 1.68e-05 | 1281.5ms/step | 15.6 samples/s +Step 38940/44000 | Loss: 2.9109 | LR: 1.67e-05 | 1276.1ms/step | 15.7 samples/s +Step 38950/44000 | Loss: 2.8992 | LR: 1.66e-05 | 1280.9ms/step | 15.6 samples/s +Step 38960/44000 | Loss: 2.9169 | LR: 1.66e-05 | 1280.2ms/step | 15.6 samples/s +Step 38970/44000 | Loss: 2.9298 | LR: 1.65e-05 | 1281.5ms/step | 15.6 samples/s +Step 38980/44000 | Loss: 2.9053 | LR: 1.64e-05 | 1284.8ms/step | 15.6 samples/s +Step 38990/44000 | Loss: 2.9351 | LR: 1.64e-05 | 1282.9ms/step | 15.6 samples/s +Step 39000/44000 | Loss: 2.9228 | LR: 1.63e-05 | 1284.7ms/step | 15.6 samples/s +Validation at step 39000: Loss = 2.9388, Perplexity = 18.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-39000 +Step 39010/44000 | Loss: 2.9600 | LR: 1.62e-05 | 3686.4ms/step | 5.4 samples/s +Step 39020/44000 | Loss: 2.9128 | LR: 1.62e-05 | 1276.9ms/step | 15.7 samples/s +Step 39030/44000 | Loss: 2.9516 | LR: 1.61e-05 | 1280.0ms/step | 15.6 samples/s +Step 39040/44000 | Loss: 2.9056 | LR: 1.61e-05 | 1276.5ms/step | 15.7 samples/s +Step 39050/44000 | Loss: 2.9143 | LR: 1.60e-05 | 1280.4ms/step | 15.6 samples/s +Step 39060/44000 | Loss: 2.9252 | LR: 1.59e-05 | 1281.8ms/step | 15.6 samples/s +Step 39070/44000 | Loss: 2.9021 | LR: 1.59e-05 | 1275.8ms/step | 15.7 samples/s +Step 39080/44000 | Loss: 2.9328 | LR: 1.58e-05 | 1279.9ms/step | 15.6 samples/s +Step 39090/44000 | Loss: 2.8937 | LR: 1.57e-05 | 1280.5ms/step | 15.6 samples/s +Step 39100/44000 | Loss: 2.8991 | LR: 1.57e-05 | 1282.1ms/step | 15.6 samples/s +Step 39110/44000 | Loss: 2.9373 | LR: 1.56e-05 | 1283.8ms/step | 15.6 samples/s +Step 39120/44000 | Loss: 2.9283 | LR: 1.55e-05 | 1282.9ms/step | 15.6 samples/s +Step 39130/44000 | Loss: 2.8983 | LR: 1.55e-05 | 1282.1ms/step | 15.6 samples/s +Step 39140/44000 | Loss: 2.9174 | LR: 1.54e-05 | 1281.2ms/step | 15.6 samples/s +Step 39150/44000 | Loss: 2.9308 | LR: 1.54e-05 | 1281.3ms/step | 15.6 samples/s +Step 39160/44000 | Loss: 2.9578 | LR: 1.53e-05 | 1282.6ms/step | 15.6 samples/s +Step 39170/44000 | Loss: 2.9339 | LR: 1.52e-05 | 1285.0ms/step | 15.6 samples/s +Step 39180/44000 | Loss: 2.9369 | LR: 1.52e-05 | 1283.2ms/step | 15.6 samples/s +Step 39190/44000 | Loss: 2.8780 | LR: 1.51e-05 | 1282.5ms/step | 15.6 samples/s +Step 39200/44000 | Loss: 2.9027 | LR: 1.50e-05 | 1284.1ms/step | 15.6 samples/s +Step 39210/44000 | Loss: 2.8912 | LR: 1.50e-05 | 1281.8ms/step | 15.6 samples/s +Step 39220/44000 | Loss: 2.9300 | LR: 1.49e-05 | 1280.7ms/step | 15.6 samples/s +Step 39230/44000 | Loss: 2.9151 | LR: 1.49e-05 | 1279.1ms/step | 15.6 samples/s +Step 39240/44000 | Loss: 2.9184 | LR: 1.48e-05 | 1283.6ms/step | 15.6 samples/s +Step 39250/44000 | Loss: 2.9161 | LR: 1.47e-05 | 1280.8ms/step | 15.6 samples/s +Validation at step 39250: Loss = 2.9386, Perplexity = 18.89 +Step 39260/44000 | Loss: 2.9146 | LR: 1.47e-05 | 3435.7ms/step | 5.8 samples/s +Step 39270/44000 | Loss: 2.9072 | LR: 1.46e-05 | 1280.4ms/step | 15.6 samples/s +Step 39280/44000 | Loss: 2.9200 | LR: 1.46e-05 | 1277.4ms/step | 15.7 samples/s +Step 39290/44000 | Loss: 2.9180 | LR: 1.45e-05 | 1283.6ms/step | 15.6 samples/s +Step 39300/44000 | Loss: 2.9179 | LR: 1.44e-05 | 1284.3ms/step | 15.6 samples/s +Step 39310/44000 | Loss: 2.9085 | LR: 1.44e-05 | 1282.4ms/step | 15.6 samples/s +Step 39320/44000 | Loss: 2.8865 | LR: 1.43e-05 | 1282.0ms/step | 15.6 samples/s +Step 39330/44000 | Loss: 2.9301 | LR: 1.42e-05 | 1282.2ms/step | 15.6 samples/s +Step 39340/44000 | Loss: 2.9272 | LR: 1.42e-05 | 1283.1ms/step | 15.6 samples/s +Step 39350/44000 | Loss: 2.9192 | LR: 1.41e-05 | 1283.1ms/step | 15.6 samples/s +Step 39360/44000 | Loss: 2.9595 | LR: 1.41e-05 | 1284.2ms/step | 15.6 samples/s +Step 39370/44000 | Loss: 2.9032 | LR: 1.40e-05 | 1285.5ms/step | 15.6 samples/s +Step 39380/44000 | Loss: 2.9416 | LR: 1.39e-05 | 1284.4ms/step | 15.6 samples/s +Step 39390/44000 | Loss: 2.9250 | LR: 1.39e-05 | 1284.0ms/step | 15.6 samples/s +Step 39400/44000 | Loss: 2.9140 | LR: 1.38e-05 | 1284.0ms/step | 15.6 samples/s +Step 39410/44000 | Loss: 2.9232 | LR: 1.38e-05 | 1282.8ms/step | 15.6 samples/s +Step 39420/44000 | Loss: 2.9474 | LR: 1.37e-05 | 1281.5ms/step | 15.6 samples/s +Step 39430/44000 | Loss: 2.9435 | LR: 1.36e-05 | 1281.8ms/step | 15.6 samples/s +Step 39440/44000 | Loss: 2.9107 | LR: 1.36e-05 | 1283.9ms/step | 15.6 samples/s +Step 39450/44000 | Loss: 2.9444 | LR: 1.35e-05 | 1283.6ms/step | 15.6 samples/s +Step 39460/44000 | Loss: 2.9060 | LR: 1.35e-05 | 1284.2ms/step | 15.6 samples/s +Step 39470/44000 | Loss: 2.9110 | LR: 1.34e-05 | 1281.5ms/step | 15.6 samples/s +Step 39480/44000 | Loss: 2.9348 | LR: 1.34e-05 | 1281.2ms/step | 15.6 samples/s +Step 39490/44000 | Loss: 2.9349 | LR: 1.33e-05 | 1283.5ms/step | 15.6 samples/s +Step 39500/44000 | Loss: 2.9173 | LR: 1.32e-05 | 1279.2ms/step | 15.6 samples/s +Validation at step 39500: Loss = 2.9387, Perplexity = 18.89 +Step 39510/44000 | Loss: 2.9015 | LR: 1.32e-05 | 3430.7ms/step | 5.8 samples/s +Step 39520/44000 | Loss: 2.9024 | LR: 1.31e-05 | 1279.6ms/step | 15.6 samples/s +Step 39530/44000 | Loss: 2.9218 | LR: 1.31e-05 | 1280.7ms/step | 15.6 samples/s +Step 39540/44000 | Loss: 2.9255 | LR: 1.30e-05 | 1275.2ms/step | 15.7 samples/s +Step 39550/44000 | Loss: 2.9179 | LR: 1.29e-05 | 1275.1ms/step | 15.7 samples/s +Step 39560/44000 | Loss: 2.9323 | LR: 1.29e-05 | 1281.8ms/step | 15.6 samples/s +Step 39570/44000 | Loss: 2.8782 | LR: 1.28e-05 | 1283.1ms/step | 15.6 samples/s +Step 39580/44000 | Loss: 2.9195 | LR: 1.28e-05 | 1282.1ms/step | 15.6 samples/s +Step 39590/44000 | Loss: 2.9084 | LR: 1.27e-05 | 1282.9ms/step | 15.6 samples/s +Step 39600/44000 | Loss: 2.9155 | LR: 1.27e-05 | 1280.5ms/step | 15.6 samples/s +Step 39610/44000 | Loss: 2.9188 | LR: 1.26e-05 | 1279.9ms/step | 15.6 samples/s +Step 39620/44000 | Loss: 2.9261 | LR: 1.25e-05 | 1280.1ms/step | 15.6 samples/s +Step 39630/44000 | Loss: 2.9262 | LR: 1.25e-05 | 1281.0ms/step | 15.6 samples/s +Step 39640/44000 | Loss: 2.9280 | LR: 1.24e-05 | 1284.2ms/step | 15.6 samples/s +Step 39650/44000 | Loss: 2.9285 | LR: 1.24e-05 | 1285.0ms/step | 15.6 samples/s +Step 39660/44000 | Loss: 2.9143 | LR: 1.23e-05 | 1280.5ms/step | 15.6 samples/s +Step 39670/44000 | Loss: 2.9045 | LR: 1.23e-05 | 1280.6ms/step | 15.6 samples/s +Step 39680/44000 | Loss: 2.9383 | LR: 1.22e-05 | 1282.6ms/step | 15.6 samples/s +Step 39690/44000 | Loss: 2.9371 | LR: 1.22e-05 | 1282.5ms/step | 15.6 samples/s +Step 39700/44000 | Loss: 2.9277 | LR: 1.21e-05 | 1280.4ms/step | 15.6 samples/s +Step 39710/44000 | Loss: 2.9650 | LR: 1.20e-05 | 1284.5ms/step | 15.6 samples/s +Step 39720/44000 | Loss: 2.9339 | LR: 1.20e-05 | 1284.4ms/step | 15.6 samples/s +Step 39730/44000 | Loss: 2.9306 | LR: 1.19e-05 | 1287.1ms/step | 15.5 samples/s +Step 39740/44000 | Loss: 2.9354 | LR: 1.19e-05 | 1285.1ms/step | 15.6 samples/s +Step 39750/44000 | Loss: 2.9506 | LR: 1.18e-05 | 1280.1ms/step | 15.6 samples/s +Validation at step 39750: Loss = 2.9386, Perplexity = 18.89 +Step 39760/44000 | Loss: 2.9548 | LR: 1.18e-05 | 3434.4ms/step | 5.8 samples/s +Step 39770/44000 | Loss: 2.9183 | LR: 1.17e-05 | 1281.7ms/step | 15.6 samples/s +Step 39780/44000 | Loss: 2.9209 | LR: 1.17e-05 | 1290.6ms/step | 15.5 samples/s +Step 39790/44000 | Loss: 2.9071 | LR: 1.16e-05 | 1284.9ms/step | 15.6 samples/s +Step 39800/44000 | Loss: 2.9228 | LR: 1.15e-05 | 1285.7ms/step | 15.6 samples/s +Step 39810/44000 | Loss: 2.9077 | LR: 1.15e-05 | 1283.4ms/step | 15.6 samples/s +Step 39820/44000 | Loss: 2.9197 | LR: 1.14e-05 | 1282.6ms/step | 15.6 samples/s +Step 39830/44000 | Loss: 2.9292 | LR: 1.14e-05 | 1279.9ms/step | 15.6 samples/s +Step 39840/44000 | Loss: 2.9436 | LR: 1.13e-05 | 1281.6ms/step | 15.6 samples/s +Step 39850/44000 | Loss: 2.9352 | LR: 1.13e-05 | 1275.7ms/step | 15.7 samples/s +Step 39860/44000 | Loss: 2.9257 | LR: 1.12e-05 | 1282.6ms/step | 15.6 samples/s +Step 39870/44000 | Loss: 2.9193 | LR: 1.12e-05 | 1282.8ms/step | 15.6 samples/s +Step 39880/44000 | Loss: 2.8970 | LR: 1.11e-05 | 1282.3ms/step | 15.6 samples/s +Step 39890/44000 | Loss: 2.9124 | LR: 1.11e-05 | 1280.4ms/step | 15.6 samples/s +Step 39900/44000 | Loss: 2.9268 | LR: 1.10e-05 | 1286.2ms/step | 15.5 samples/s +Step 39910/44000 | Loss: 2.8957 | LR: 1.10e-05 | 1284.1ms/step | 15.6 samples/s +Step 39920/44000 | Loss: 2.9523 | LR: 1.09e-05 | 1283.9ms/step | 15.6 samples/s +Step 39930/44000 | Loss: 2.9252 | LR: 1.08e-05 | 1282.9ms/step | 15.6 samples/s +Step 39940/44000 | Loss: 2.9472 | LR: 1.08e-05 | 1281.6ms/step | 15.6 samples/s +Step 39950/44000 | Loss: 2.9098 | LR: 1.07e-05 | 1281.7ms/step | 15.6 samples/s +Step 39960/44000 | Loss: 2.9273 | LR: 1.07e-05 | 1284.7ms/step | 15.6 samples/s +Step 39970/44000 | Loss: 2.9240 | LR: 1.06e-05 | 1280.6ms/step | 15.6 samples/s +Step 39980/44000 | Loss: 2.9123 | LR: 1.06e-05 | 1282.3ms/step | 15.6 samples/s +Step 39990/44000 | Loss: 2.9366 | LR: 1.05e-05 | 1282.1ms/step | 15.6 samples/s +Step 40000/44000 | Loss: 2.9071 | LR: 1.05e-05 | 1278.6ms/step | 15.6 samples/s +Validation at step 40000: Loss = 2.9387, Perplexity = 18.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-40000 +Step 40010/44000 | Loss: 2.9269 | LR: 1.04e-05 | 3710.2ms/step | 5.4 samples/s +Step 40020/44000 | Loss: 2.9260 | LR: 1.04e-05 | 1281.1ms/step | 15.6 samples/s +Step 40030/44000 | Loss: 2.9368 | LR: 1.03e-05 | 1284.2ms/step | 15.6 samples/s +Step 40040/44000 | Loss: 2.9171 | LR: 1.03e-05 | 1277.8ms/step | 15.7 samples/s +Step 40050/44000 | Loss: 2.9004 | LR: 1.02e-05 | 1278.1ms/step | 15.6 samples/s +Step 40060/44000 | Loss: 2.9082 | LR: 1.02e-05 | 1280.7ms/step | 15.6 samples/s +Step 40070/44000 | Loss: 2.9303 | LR: 1.01e-05 | 1284.6ms/step | 15.6 samples/s +Step 40080/44000 | Loss: 2.9355 | LR: 1.01e-05 | 1281.3ms/step | 15.6 samples/s +Step 40090/44000 | Loss: 2.9145 | LR: 1.00e-05 | 1279.0ms/step | 15.6 samples/s +Step 40100/44000 | Loss: 2.9232 | LR: 9.96e-06 | 1279.5ms/step | 15.6 samples/s +Step 40110/44000 | Loss: 2.9227 | LR: 9.91e-06 | 1282.2ms/step | 15.6 samples/s +Step 40120/44000 | Loss: 2.9371 | LR: 9.86e-06 | 1282.3ms/step | 15.6 samples/s +Step 40130/44000 | Loss: 2.9313 | LR: 9.81e-06 | 1279.3ms/step | 15.6 samples/s +Step 40140/44000 | Loss: 2.9368 | LR: 9.76e-06 | 1284.7ms/step | 15.6 samples/s +Step 40150/44000 | Loss: 2.8944 | LR: 9.71e-06 | 1289.6ms/step | 15.5 samples/s +Step 40160/44000 | Loss: 2.9031 | LR: 9.66e-06 | 1286.2ms/step | 15.5 samples/s +Step 40170/44000 | Loss: 2.9080 | LR: 9.61e-06 | 1286.2ms/step | 15.5 samples/s +Step 40180/44000 | Loss: 2.9482 | LR: 9.56e-06 | 1285.1ms/step | 15.6 samples/s +Step 40190/44000 | Loss: 2.9228 | LR: 9.51e-06 | 1280.7ms/step | 15.6 samples/s +Step 40200/44000 | Loss: 2.9543 | LR: 9.46e-06 | 1283.2ms/step | 15.6 samples/s +Step 40210/44000 | Loss: 2.9216 | LR: 9.41e-06 | 1282.4ms/step | 15.6 samples/s +Step 40220/44000 | Loss: 2.9406 | LR: 9.36e-06 | 1283.8ms/step | 15.6 samples/s +Step 40230/44000 | Loss: 2.9136 | LR: 9.32e-06 | 1280.3ms/step | 15.6 samples/s +Step 40240/44000 | Loss: 2.9229 | LR: 9.27e-06 | 1283.4ms/step | 15.6 samples/s +Step 40250/44000 | Loss: 2.8871 | LR: 9.22e-06 | 1282.8ms/step | 15.6 samples/s +Validation at step 40250: Loss = 2.9385, Perplexity = 18.89 +Step 40260/44000 | Loss: 2.9154 | LR: 9.17e-06 | 3434.5ms/step | 5.8 samples/s +Step 40270/44000 | Loss: 2.9325 | LR: 9.12e-06 | 1279.0ms/step | 15.6 samples/s +Step 40280/44000 | Loss: 2.9269 | LR: 9.07e-06 | 1281.5ms/step | 15.6 samples/s +Step 40290/44000 | Loss: 2.9023 | LR: 9.02e-06 | 1280.6ms/step | 15.6 samples/s +Step 40300/44000 | Loss: 2.9292 | LR: 8.97e-06 | 1284.6ms/step | 15.6 samples/s +Step 40310/44000 | Loss: 2.9171 | LR: 8.93e-06 | 1283.7ms/step | 15.6 samples/s +Step 40320/44000 | Loss: 2.9456 | LR: 8.88e-06 | 1282.1ms/step | 15.6 samples/s +Step 40330/44000 | Loss: 2.9254 | LR: 8.83e-06 | 1282.4ms/step | 15.6 samples/s +Step 40340/44000 | Loss: 2.9234 | LR: 8.78e-06 | 1284.3ms/step | 15.6 samples/s +Step 40350/44000 | Loss: 2.9148 | LR: 8.74e-06 | 1281.7ms/step | 15.6 samples/s +Step 40360/44000 | Loss: 2.9283 | LR: 8.69e-06 | 1282.2ms/step | 15.6 samples/s +Step 40370/44000 | Loss: 2.9053 | LR: 8.64e-06 | 1280.7ms/step | 15.6 samples/s +Step 40380/44000 | Loss: 2.9251 | LR: 8.59e-06 | 1281.9ms/step | 15.6 samples/s +Step 40390/44000 | Loss: 2.9425 | LR: 8.55e-06 | 1276.2ms/step | 15.7 samples/s +Step 40400/44000 | Loss: 2.9100 | LR: 8.50e-06 | 1280.4ms/step | 15.6 samples/s +Step 40410/44000 | Loss: 2.8968 | LR: 8.45e-06 | 1280.5ms/step | 15.6 samples/s +Step 40420/44000 | Loss: 2.9197 | LR: 8.41e-06 | 1281.5ms/step | 15.6 samples/s +Step 40430/44000 | Loss: 2.9280 | LR: 8.36e-06 | 1279.8ms/step | 15.6 samples/s +Step 40440/44000 | Loss: 2.9058 | LR: 8.31e-06 | 1279.9ms/step | 15.6 samples/s +Step 40450/44000 | Loss: 2.9189 | LR: 8.27e-06 | 1281.9ms/step | 15.6 samples/s +Step 40460/44000 | Loss: 2.8856 | LR: 8.22e-06 | 1282.9ms/step | 15.6 samples/s +Step 40470/44000 | Loss: 2.9402 | LR: 8.17e-06 | 1281.3ms/step | 15.6 samples/s +Step 40480/44000 | Loss: 2.9288 | LR: 8.13e-06 | 1284.2ms/step | 15.6 samples/s +Step 40490/44000 | Loss: 2.9295 | LR: 8.08e-06 | 1282.0ms/step | 15.6 samples/s +Step 40500/44000 | Loss: 2.9227 | LR: 8.04e-06 | 1280.3ms/step | 15.6 samples/s +Validation at step 40500: Loss = 2.9385, Perplexity = 18.89 +Step 40510/44000 | Loss: 2.9106 | LR: 7.99e-06 | 3437.0ms/step | 5.8 samples/s +Step 40520/44000 | Loss: 2.9579 | LR: 7.94e-06 | 1283.7ms/step | 15.6 samples/s +Step 40530/44000 | Loss: 2.9455 | LR: 7.90e-06 | 1281.2ms/step | 15.6 samples/s +Step 40540/44000 | Loss: 2.9386 | LR: 7.85e-06 | 1280.8ms/step | 15.6 samples/s +Step 40550/44000 | Loss: 2.8954 | LR: 7.81e-06 | 1278.6ms/step | 15.6 samples/s +Step 40560/44000 | Loss: 2.8955 | LR: 7.76e-06 | 1281.8ms/step | 15.6 samples/s +Step 40570/44000 | Loss: 2.9314 | LR: 7.72e-06 | 1285.9ms/step | 15.6 samples/s +Step 40580/44000 | Loss: 2.9226 | LR: 7.67e-06 | 1287.0ms/step | 15.5 samples/s +Step 40590/44000 | Loss: 2.8987 | LR: 7.63e-06 | 1284.5ms/step | 15.6 samples/s +Step 40600/44000 | Loss: 2.9274 | LR: 7.59e-06 | 1282.1ms/step | 15.6 samples/s +Step 40610/44000 | Loss: 2.9569 | LR: 7.54e-06 | 1282.0ms/step | 15.6 samples/s +Step 40620/44000 | Loss: 2.9229 | LR: 7.50e-06 | 1280.7ms/step | 15.6 samples/s +Step 40630/44000 | Loss: 2.9375 | LR: 7.45e-06 | 1278.2ms/step | 15.6 samples/s +Step 40640/44000 | Loss: 2.9447 | LR: 7.41e-06 | 1282.7ms/step | 15.6 samples/s +Step 40650/44000 | Loss: 2.9422 | LR: 7.37e-06 | 1280.9ms/step | 15.6 samples/s +Step 40660/44000 | Loss: 2.9321 | LR: 7.32e-06 | 1279.6ms/step | 15.6 samples/s +Step 40670/44000 | Loss: 2.9411 | LR: 7.28e-06 | 1280.1ms/step | 15.6 samples/s +Step 40680/44000 | Loss: 2.9256 | LR: 7.23e-06 | 1282.4ms/step | 15.6 samples/s +Step 40690/44000 | Loss: 2.9359 | LR: 7.19e-06 | 1284.0ms/step | 15.6 samples/s +Step 40700/44000 | Loss: 2.9147 | LR: 7.15e-06 | 1281.7ms/step | 15.6 samples/s +Step 40710/44000 | Loss: 2.9289 | LR: 7.10e-06 | 1281.6ms/step | 15.6 samples/s +Step 40720/44000 | Loss: 2.9374 | LR: 7.06e-06 | 1284.1ms/step | 15.6 samples/s +Step 40730/44000 | Loss: 2.9446 | LR: 7.02e-06 | 1277.8ms/step | 15.7 samples/s +Step 40740/44000 | Loss: 2.9171 | LR: 6.98e-06 | 1281.4ms/step | 15.6 samples/s +Step 40750/44000 | Loss: 2.9135 | LR: 6.93e-06 | 1281.6ms/step | 15.6 samples/s +Validation at step 40750: Loss = 2.9385, Perplexity = 18.89 +Step 40760/44000 | Loss: 2.9408 | LR: 6.89e-06 | 3430.0ms/step | 5.8 samples/s +Step 40770/44000 | Loss: 2.9209 | LR: 6.85e-06 | 1282.0ms/step | 15.6 samples/s +Step 40780/44000 | Loss: 2.9406 | LR: 6.81e-06 | 1283.0ms/step | 15.6 samples/s +Step 40790/44000 | Loss: 2.9446 | LR: 6.77e-06 | 1284.8ms/step | 15.6 samples/s +Step 40800/44000 | Loss: 2.9470 | LR: 6.72e-06 | 1283.2ms/step | 15.6 samples/s +Step 40810/44000 | Loss: 2.9265 | LR: 6.68e-06 | 1283.3ms/step | 15.6 samples/s +Step 40820/44000 | Loss: 2.9056 | LR: 6.64e-06 | 1282.7ms/step | 15.6 samples/s +Step 40830/44000 | Loss: 2.9010 | LR: 6.60e-06 | 1284.1ms/step | 15.6 samples/s +Step 40840/44000 | Loss: 2.9452 | LR: 6.56e-06 | 1285.2ms/step | 15.6 samples/s +Step 40850/44000 | Loss: 2.9325 | LR: 6.52e-06 | 1285.3ms/step | 15.6 samples/s +Step 40860/44000 | Loss: 2.9430 | LR: 6.47e-06 | 1284.7ms/step | 15.6 samples/s +Step 40870/44000 | Loss: 2.9468 | LR: 6.43e-06 | 1282.5ms/step | 15.6 samples/s +Step 40880/44000 | Loss: 2.9187 | LR: 6.39e-06 | 1283.1ms/step | 15.6 samples/s +Step 40890/44000 | Loss: 2.9363 | LR: 6.35e-06 | 1279.6ms/step | 15.6 samples/s +Step 40900/44000 | Loss: 2.9118 | LR: 6.31e-06 | 1282.2ms/step | 15.6 samples/s +Step 40910/44000 | Loss: 2.9819 | LR: 6.27e-06 | 1281.6ms/step | 15.6 samples/s +Step 40920/44000 | Loss: 2.9297 | LR: 6.23e-06 | 1283.2ms/step | 15.6 samples/s +Step 40930/44000 | Loss: 2.9382 | LR: 6.19e-06 | 1282.6ms/step | 15.6 samples/s +Step 40940/44000 | Loss: 2.9417 | LR: 6.15e-06 | 1282.3ms/step | 15.6 samples/s +Step 40950/44000 | Loss: 2.9022 | LR: 6.11e-06 | 1284.3ms/step | 15.6 samples/s +Step 40960/44000 | Loss: 2.8905 | LR: 6.07e-06 | 1284.2ms/step | 15.6 samples/s +Step 40970/44000 | Loss: 2.9193 | LR: 6.03e-06 | 1281.8ms/step | 15.6 samples/s +Step 40980/44000 | Loss: 2.9068 | LR: 5.99e-06 | 1280.6ms/step | 15.6 samples/s +Step 40990/44000 | Loss: 2.9289 | LR: 5.95e-06 | 1282.3ms/step | 15.6 samples/s +Step 41000/44000 | Loss: 2.9340 | LR: 5.91e-06 | 1282.5ms/step | 15.6 samples/s +Validation at step 41000: Loss = 2.9384, Perplexity = 18.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-41000 +Step 41010/44000 | Loss: 2.9413 | LR: 5.87e-06 | 3718.4ms/step | 5.4 samples/s +Step 41020/44000 | Loss: 2.9490 | LR: 5.83e-06 | 1284.3ms/step | 15.6 samples/s +Step 41030/44000 | Loss: 2.9235 | LR: 5.80e-06 | 1280.5ms/step | 15.6 samples/s +Step 41040/44000 | Loss: 2.9230 | LR: 5.76e-06 | 1281.4ms/step | 15.6 samples/s +Step 41050/44000 | Loss: 2.9213 | LR: 5.72e-06 | 1291.5ms/step | 15.5 samples/s +Step 41060/44000 | Loss: 2.8928 | LR: 5.68e-06 | 1286.0ms/step | 15.6 samples/s +Step 41070/44000 | Loss: 2.9194 | LR: 5.64e-06 | 1284.6ms/step | 15.6 samples/s +Step 41080/44000 | Loss: 2.9512 | LR: 5.60e-06 | 1283.2ms/step | 15.6 samples/s +Step 41090/44000 | Loss: 2.9679 | LR: 5.56e-06 | 1283.9ms/step | 15.6 samples/s +Step 41100/44000 | Loss: 2.9455 | LR: 5.53e-06 | 1282.2ms/step | 15.6 samples/s +Step 41110/44000 | Loss: 2.9681 | LR: 5.49e-06 | 1283.3ms/step | 15.6 samples/s +Step 41120/44000 | Loss: 2.9556 | LR: 5.45e-06 | 1285.6ms/step | 15.6 samples/s +Step 41130/44000 | Loss: 2.9250 | LR: 5.41e-06 | 1286.5ms/step | 15.5 samples/s +Step 41140/44000 | Loss: 2.9055 | LR: 5.38e-06 | 1282.7ms/step | 15.6 samples/s +Step 41150/44000 | Loss: 2.9457 | LR: 5.34e-06 | 1285.8ms/step | 15.6 samples/s +Step 41160/44000 | Loss: 2.9184 | LR: 5.30e-06 | 1282.6ms/step | 15.6 samples/s +Step 41170/44000 | Loss: 2.9295 | LR: 5.26e-06 | 1283.5ms/step | 15.6 samples/s +Step 41180/44000 | Loss: 2.9043 | LR: 5.23e-06 | 1280.2ms/step | 15.6 samples/s +Step 41190/44000 | Loss: 2.8882 | LR: 5.19e-06 | 1280.3ms/step | 15.6 samples/s +Step 41200/44000 | Loss: 2.9282 | LR: 5.15e-06 | 1283.5ms/step | 15.6 samples/s +Step 41210/44000 | Loss: 2.9373 | LR: 5.12e-06 | 1284.3ms/step | 15.6 samples/s +Step 41220/44000 | Loss: 2.9451 | LR: 5.08e-06 | 1283.2ms/step | 15.6 samples/s +Step 41230/44000 | Loss: 2.9194 | LR: 5.04e-06 | 1283.8ms/step | 15.6 samples/s +Step 41240/44000 | Loss: 2.9072 | LR: 5.01e-06 | 1283.0ms/step | 15.6 samples/s +Step 41250/44000 | Loss: 2.9244 | LR: 4.97e-06 | 1283.5ms/step | 15.6 samples/s +Validation at step 41250: Loss = 2.9383, Perplexity = 18.88 +Step 41260/44000 | Loss: 2.9518 | LR: 4.94e-06 | 3432.2ms/step | 5.8 samples/s +Step 41270/44000 | Loss: 2.9308 | LR: 4.90e-06 | 1280.5ms/step | 15.6 samples/s +Step 41280/44000 | Loss: 2.9278 | LR: 4.86e-06 | 1284.1ms/step | 15.6 samples/s +Step 41290/44000 | Loss: 2.9159 | LR: 4.83e-06 | 1278.3ms/step | 15.6 samples/s +Step 41300/44000 | Loss: 2.9209 | LR: 4.79e-06 | 1276.4ms/step | 15.7 samples/s +Step 41310/44000 | Loss: 2.9081 | LR: 4.76e-06 | 1276.1ms/step | 15.7 samples/s +Step 41320/44000 | Loss: 2.9336 | LR: 4.72e-06 | 1282.4ms/step | 15.6 samples/s +Step 41330/44000 | Loss: 2.9187 | LR: 4.69e-06 | 1282.2ms/step | 15.6 samples/s +Step 41340/44000 | Loss: 2.9415 | LR: 4.65e-06 | 1280.3ms/step | 15.6 samples/s +Step 41350/44000 | Loss: 2.9049 | LR: 4.62e-06 | 1279.2ms/step | 15.6 samples/s +Step 41360/44000 | Loss: 2.9540 | LR: 4.58e-06 | 1278.8ms/step | 15.6 samples/s +Step 41370/44000 | Loss: 2.9357 | LR: 4.55e-06 | 1279.3ms/step | 15.6 samples/s +Step 41380/44000 | Loss: 2.9293 | LR: 4.51e-06 | 1280.3ms/step | 15.6 samples/s +Step 41390/44000 | Loss: 2.9092 | LR: 4.48e-06 | 1282.7ms/step | 15.6 samples/s +Step 41400/44000 | Loss: 2.8957 | LR: 4.45e-06 | 1285.6ms/step | 15.6 samples/s +Step 41410/44000 | Loss: 2.9017 | LR: 4.41e-06 | 1284.0ms/step | 15.6 samples/s +Step 41420/44000 | Loss: 2.8943 | LR: 4.38e-06 | 1283.2ms/step | 15.6 samples/s +Step 41430/44000 | Loss: 2.9203 | LR: 4.34e-06 | 1281.7ms/step | 15.6 samples/s +Step 41440/44000 | Loss: 2.9599 | LR: 4.31e-06 | 1281.2ms/step | 15.6 samples/s +Step 41450/44000 | Loss: 2.9244 | LR: 4.28e-06 | 1280.0ms/step | 15.6 samples/s +Step 41460/44000 | Loss: 2.9259 | LR: 4.24e-06 | 1282.7ms/step | 15.6 samples/s +Step 41470/44000 | Loss: 2.9126 | LR: 4.21e-06 | 1280.8ms/step | 15.6 samples/s +Step 41480/44000 | Loss: 2.9420 | LR: 4.18e-06 | 1283.0ms/step | 15.6 samples/s +Step 41490/44000 | Loss: 2.9119 | LR: 4.14e-06 | 1283.0ms/step | 15.6 samples/s +Step 41500/44000 | Loss: 2.9263 | LR: 4.11e-06 | 1283.9ms/step | 15.6 samples/s +Validation at step 41500: Loss = 2.9388, Perplexity = 18.89 +Step 41510/44000 | Loss: 2.9224 | LR: 4.08e-06 | 3439.8ms/step | 5.8 samples/s +Step 41520/44000 | Loss: 2.9235 | LR: 4.05e-06 | 1285.5ms/step | 15.6 samples/s +Step 41530/44000 | Loss: 2.9269 | LR: 4.01e-06 | 1287.4ms/step | 15.5 samples/s +Step 41540/44000 | Loss: 2.9495 | LR: 3.98e-06 | 1287.1ms/step | 15.5 samples/s +Step 41550/44000 | Loss: 2.9242 | LR: 3.95e-06 | 1285.2ms/step | 15.6 samples/s +Step 41560/44000 | Loss: 2.9364 | LR: 3.92e-06 | 1289.2ms/step | 15.5 samples/s +Step 41570/44000 | Loss: 2.9547 | LR: 3.88e-06 | 1289.3ms/step | 15.5 samples/s +Step 41580/44000 | Loss: 2.9095 | LR: 3.85e-06 | 1285.1ms/step | 15.6 samples/s +Step 41590/44000 | Loss: 2.9233 | LR: 3.82e-06 | 1287.5ms/step | 15.5 samples/s +Step 41600/44000 | Loss: 2.9359 | LR: 3.79e-06 | 1284.9ms/step | 15.6 samples/s +Step 41610/44000 | Loss: 2.9146 | LR: 3.76e-06 | 1284.8ms/step | 15.6 samples/s +Step 41620/44000 | Loss: 2.8909 | LR: 3.73e-06 | 1283.7ms/step | 15.6 samples/s +Step 41630/44000 | Loss: 2.9488 | LR: 3.70e-06 | 1282.4ms/step | 15.6 samples/s +Step 41640/44000 | Loss: 2.9479 | LR: 3.66e-06 | 1283.6ms/step | 15.6 samples/s +Step 41650/44000 | Loss: 2.9295 | LR: 3.63e-06 | 1281.5ms/step | 15.6 samples/s +Step 41660/44000 | Loss: 2.9483 | LR: 3.60e-06 | 1282.7ms/step | 15.6 samples/s +Step 41670/44000 | Loss: 2.9237 | LR: 3.57e-06 | 1283.7ms/step | 15.6 samples/s +Step 41680/44000 | Loss: 2.8950 | LR: 3.54e-06 | 1285.1ms/step | 15.6 samples/s +Step 41690/44000 | Loss: 2.9109 | LR: 3.51e-06 | 1283.4ms/step | 15.6 samples/s +Step 41700/44000 | Loss: 2.9421 | LR: 3.48e-06 | 1285.0ms/step | 15.6 samples/s +Step 41710/44000 | Loss: 2.9162 | LR: 3.45e-06 | 1281.7ms/step | 15.6 samples/s +Step 41720/44000 | Loss: 2.9200 | LR: 3.42e-06 | 1283.8ms/step | 15.6 samples/s +Step 41730/44000 | Loss: 2.9318 | LR: 3.39e-06 | 1283.1ms/step | 15.6 samples/s +Step 41740/44000 | Loss: 2.9422 | LR: 3.36e-06 | 1281.2ms/step | 15.6 samples/s +Step 41750/44000 | Loss: 2.9189 | LR: 3.33e-06 | 1282.2ms/step | 15.6 samples/s +Validation at step 41750: Loss = 2.9385, Perplexity = 18.89 +Step 41760/44000 | Loss: 2.9510 | LR: 3.30e-06 | 3427.1ms/step | 5.8 samples/s +Step 41770/44000 | Loss: 2.9248 | LR: 3.27e-06 | 1284.5ms/step | 15.6 samples/s +Step 41780/44000 | Loss: 2.9175 | LR: 3.24e-06 | 1279.4ms/step | 15.6 samples/s +Step 41790/44000 | Loss: 2.9602 | LR: 3.21e-06 | 1281.6ms/step | 15.6 samples/s +Step 41800/44000 | Loss: 2.9709 | LR: 3.19e-06 | 1283.2ms/step | 15.6 samples/s +Step 41810/44000 | Loss: 2.9346 | LR: 3.16e-06 | 1275.7ms/step | 15.7 samples/s +Step 41820/44000 | Loss: 2.9112 | LR: 3.13e-06 | 1280.1ms/step | 15.6 samples/s +Step 41830/44000 | Loss: 2.9127 | LR: 3.10e-06 | 1280.4ms/step | 15.6 samples/s +Step 41840/44000 | Loss: 2.8947 | LR: 3.07e-06 | 1282.3ms/step | 15.6 samples/s +Step 41850/44000 | Loss: 2.9004 | LR: 3.04e-06 | 1284.5ms/step | 15.6 samples/s +Step 41860/44000 | Loss: 2.9474 | LR: 3.01e-06 | 1282.6ms/step | 15.6 samples/s +Step 41870/44000 | Loss: 2.9581 | LR: 2.99e-06 | 1281.6ms/step | 15.6 samples/s +Step 41880/44000 | Loss: 2.9357 | LR: 2.96e-06 | 1282.4ms/step | 15.6 samples/s +Step 41890/44000 | Loss: 2.9204 | LR: 2.93e-06 | 1281.9ms/step | 15.6 samples/s +Step 41900/44000 | Loss: 2.9207 | LR: 2.90e-06 | 1283.7ms/step | 15.6 samples/s +Step 41910/44000 | Loss: 2.9147 | LR: 2.88e-06 | 1284.4ms/step | 15.6 samples/s +Step 41920/44000 | Loss: 2.9663 | LR: 2.85e-06 | 1283.3ms/step | 15.6 samples/s +Step 41930/44000 | Loss: 2.9178 | LR: 2.82e-06 | 1281.5ms/step | 15.6 samples/s +Step 41940/44000 | Loss: 2.9261 | LR: 2.79e-06 | 1281.2ms/step | 15.6 samples/s +Step 41950/44000 | Loss: 2.8873 | LR: 2.77e-06 | 1284.9ms/step | 15.6 samples/s +Step 41960/44000 | Loss: 2.9349 | LR: 2.74e-06 | 1280.1ms/step | 15.6 samples/s +Step 41970/44000 | Loss: 2.9298 | LR: 2.71e-06 | 1282.6ms/step | 15.6 samples/s +Step 41980/44000 | Loss: 2.9235 | LR: 2.69e-06 | 1281.5ms/step | 15.6 samples/s +Step 41990/44000 | Loss: 2.9266 | LR: 2.66e-06 | 1283.6ms/step | 15.6 samples/s +Step 42000/44000 | Loss: 2.9262 | LR: 2.63e-06 | 1284.0ms/step | 15.6 samples/s +Validation at step 42000: Loss = 2.9386, Perplexity = 18.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-42000 +Step 42010/44000 | Loss: 2.9342 | LR: 2.61e-06 | 3695.9ms/step | 5.4 samples/s +Step 42020/44000 | Loss: 2.9431 | LR: 2.58e-06 | 1280.7ms/step | 15.6 samples/s +Step 42030/44000 | Loss: 2.9485 | LR: 2.56e-06 | 1282.0ms/step | 15.6 samples/s +Step 42040/44000 | Loss: 2.9464 | LR: 2.53e-06 | 1285.8ms/step | 15.6 samples/s +Step 42050/44000 | Loss: 2.9052 | LR: 2.50e-06 | 1282.6ms/step | 15.6 samples/s +Step 42060/44000 | Loss: 2.9408 | LR: 2.48e-06 | 1281.8ms/step | 15.6 samples/s +Step 42070/44000 | Loss: 2.9435 | LR: 2.45e-06 | 1280.1ms/step | 15.6 samples/s +Step 42080/44000 | Loss: 2.9223 | LR: 2.43e-06 | 1279.4ms/step | 15.6 samples/s +Step 42090/44000 | Loss: 2.9069 | LR: 2.40e-06 | 1283.1ms/step | 15.6 samples/s +Step 42100/44000 | Loss: 2.9292 | LR: 2.38e-06 | 1283.1ms/step | 15.6 samples/s +Step 42110/44000 | Loss: 2.9425 | LR: 2.35e-06 | 1285.7ms/step | 15.6 samples/s +Step 42120/44000 | Loss: 2.9288 | LR: 2.33e-06 | 1283.5ms/step | 15.6 samples/s +Step 42130/44000 | Loss: 2.9411 | LR: 2.30e-06 | 1281.6ms/step | 15.6 samples/s +Step 42140/44000 | Loss: 2.9375 | LR: 2.28e-06 | 1283.2ms/step | 15.6 samples/s +Step 42150/44000 | Loss: 2.9590 | LR: 2.25e-06 | 1282.8ms/step | 15.6 samples/s +Step 42160/44000 | Loss: 2.9248 | LR: 2.23e-06 | 1278.9ms/step | 15.6 samples/s +Step 42170/44000 | Loss: 2.9115 | LR: 2.21e-06 | 1283.7ms/step | 15.6 samples/s +Step 42180/44000 | Loss: 2.9396 | LR: 2.18e-06 | 1286.2ms/step | 15.6 samples/s +Step 42190/44000 | Loss: 2.9245 | LR: 2.16e-06 | 1282.3ms/step | 15.6 samples/s +Step 42200/44000 | Loss: 2.9082 | LR: 2.13e-06 | 1279.9ms/step | 15.6 samples/s +Step 42210/44000 | Loss: 2.9543 | LR: 2.11e-06 | 1280.0ms/step | 15.6 samples/s +Step 42220/44000 | Loss: 2.8944 | LR: 2.09e-06 | 1279.3ms/step | 15.6 samples/s +Step 42230/44000 | Loss: 2.9474 | LR: 2.06e-06 | 1282.4ms/step | 15.6 samples/s +Step 42240/44000 | Loss: 2.9244 | LR: 2.04e-06 | 1284.5ms/step | 15.6 samples/s +Step 42250/44000 | Loss: 2.9407 | LR: 2.02e-06 | 1283.1ms/step | 15.6 samples/s +Validation at step 42250: Loss = 2.9386, Perplexity = 18.89 +Step 42260/44000 | Loss: 2.9307 | LR: 1.99e-06 | 3429.6ms/step | 5.8 samples/s +Step 42270/44000 | Loss: 2.9243 | LR: 1.97e-06 | 1283.5ms/step | 15.6 samples/s +Step 42280/44000 | Loss: 2.9269 | LR: 1.95e-06 | 1282.1ms/step | 15.6 samples/s +Step 42290/44000 | Loss: 2.8963 | LR: 1.93e-06 | 1283.9ms/step | 15.6 samples/s +Step 42300/44000 | Loss: 2.9232 | LR: 1.90e-06 | 1281.9ms/step | 15.6 samples/s +Step 42310/44000 | Loss: 2.8971 | LR: 1.88e-06 | 1283.7ms/step | 15.6 samples/s +Step 42320/44000 | Loss: 2.9430 | LR: 1.86e-06 | 1282.8ms/step | 15.6 samples/s +Step 42330/44000 | Loss: 2.9108 | LR: 1.84e-06 | 1281.0ms/step | 15.6 samples/s +Step 42340/44000 | Loss: 2.9532 | LR: 1.82e-06 | 1281.9ms/step | 15.6 samples/s +Step 42350/44000 | Loss: 2.8844 | LR: 1.79e-06 | 1281.4ms/step | 15.6 samples/s +Step 42360/44000 | Loss: 2.9271 | LR: 1.77e-06 | 1285.6ms/step | 15.6 samples/s +Step 42370/44000 | Loss: 2.9268 | LR: 1.75e-06 | 1289.2ms/step | 15.5 samples/s +Step 42380/44000 | Loss: 2.9601 | LR: 1.73e-06 | 1285.9ms/step | 15.6 samples/s +Step 42390/44000 | Loss: 2.8909 | LR: 1.71e-06 | 1279.7ms/step | 15.6 samples/s +Step 42400/44000 | Loss: 2.9196 | LR: 1.69e-06 | 1280.8ms/step | 15.6 samples/s +Step 42410/44000 | Loss: 2.9236 | LR: 1.67e-06 | 1281.9ms/step | 15.6 samples/s +Step 42420/44000 | Loss: 2.9516 | LR: 1.64e-06 | 1281.5ms/step | 15.6 samples/s +Step 42430/44000 | Loss: 2.9533 | LR: 1.62e-06 | 1280.3ms/step | 15.6 samples/s +Step 42440/44000 | Loss: 2.9449 | LR: 1.60e-06 | 1279.0ms/step | 15.6 samples/s +Step 42450/44000 | Loss: 2.9139 | LR: 1.58e-06 | 1281.1ms/step | 15.6 samples/s +Step 42460/44000 | Loss: 2.9241 | LR: 1.56e-06 | 1283.7ms/step | 15.6 samples/s +Step 42470/44000 | Loss: 2.9589 | LR: 1.54e-06 | 1283.8ms/step | 15.6 samples/s +Step 42480/44000 | Loss: 2.9256 | LR: 1.52e-06 | 1283.7ms/step | 15.6 samples/s +Step 42490/44000 | Loss: 2.9340 | LR: 1.50e-06 | 1286.1ms/step | 15.6 samples/s +Step 42500/44000 | Loss: 2.9019 | LR: 1.48e-06 | 1283.8ms/step | 15.6 samples/s +Validation at step 42500: Loss = 2.9386, Perplexity = 18.89 +Step 42510/44000 | Loss: 2.9433 | LR: 1.46e-06 | 3442.0ms/step | 5.8 samples/s +Step 42520/44000 | Loss: 2.9038 | LR: 1.44e-06 | 1284.8ms/step | 15.6 samples/s +Step 42530/44000 | Loss: 2.9324 | LR: 1.42e-06 | 1282.7ms/step | 15.6 samples/s +Step 42540/44000 | Loss: 2.9193 | LR: 1.40e-06 | 1283.6ms/step | 15.6 samples/s +Step 42550/44000 | Loss: 2.9373 | LR: 1.39e-06 | 1280.1ms/step | 15.6 samples/s +Step 42560/44000 | Loss: 2.9092 | LR: 1.37e-06 | 1280.5ms/step | 15.6 samples/s +Step 42570/44000 | Loss: 2.9255 | LR: 1.35e-06 | 1279.5ms/step | 15.6 samples/s +Step 42580/44000 | Loss: 2.9264 | LR: 1.33e-06 | 1279.4ms/step | 15.6 samples/s +Step 42590/44000 | Loss: 2.9108 | LR: 1.31e-06 | 1283.0ms/step | 15.6 samples/s +Step 42600/44000 | Loss: 2.9047 | LR: 1.29e-06 | 1284.4ms/step | 15.6 samples/s +Step 42610/44000 | Loss: 2.9337 | LR: 1.27e-06 | 1281.0ms/step | 15.6 samples/s +Step 42620/44000 | Loss: 2.9416 | LR: 1.25e-06 | 1283.3ms/step | 15.6 samples/s +Step 42630/44000 | Loss: 2.9115 | LR: 1.24e-06 | 1280.1ms/step | 15.6 samples/s +Step 42640/44000 | Loss: 2.9195 | LR: 1.22e-06 | 1283.0ms/step | 15.6 samples/s +Step 42650/44000 | Loss: 2.9409 | LR: 1.20e-06 | 1281.9ms/step | 15.6 samples/s +Step 42660/44000 | Loss: 2.9139 | LR: 1.18e-06 | 1283.8ms/step | 15.6 samples/s +Step 42670/44000 | Loss: 2.9111 | LR: 1.17e-06 | 1280.6ms/step | 15.6 samples/s +Step 42680/44000 | Loss: 2.9030 | LR: 1.15e-06 | 1281.8ms/step | 15.6 samples/s +Step 42690/44000 | Loss: 2.8968 | LR: 1.13e-06 | 1283.8ms/step | 15.6 samples/s +Step 42700/44000 | Loss: 2.9016 | LR: 1.11e-06 | 1283.0ms/step | 15.6 samples/s +Step 42710/44000 | Loss: 2.9396 | LR: 1.10e-06 | 1284.7ms/step | 15.6 samples/s +Step 42720/44000 | Loss: 2.9309 | LR: 1.08e-06 | 1285.3ms/step | 15.6 samples/s +Step 42730/44000 | Loss: 2.9247 | LR: 1.06e-06 | 1282.4ms/step | 15.6 samples/s +Step 42740/44000 | Loss: 2.8991 | LR: 1.05e-06 | 1281.5ms/step | 15.6 samples/s +Step 42750/44000 | Loss: 2.9086 | LR: 1.03e-06 | 1279.8ms/step | 15.6 samples/s +Validation at step 42750: Loss = 2.9387, Perplexity = 18.89 +Step 42760/44000 | Loss: 2.9238 | LR: 1.01e-06 | 3435.5ms/step | 5.8 samples/s +Step 42770/44000 | Loss: 2.9395 | LR: 9.97e-07 | 1287.2ms/step | 15.5 samples/s +Step 42780/44000 | Loss: 2.9444 | LR: 9.81e-07 | 1285.2ms/step | 15.6 samples/s +Step 42790/44000 | Loss: 2.9167 | LR: 9.65e-07 | 1284.9ms/step | 15.6 samples/s +Step 42800/44000 | Loss: 2.9105 | LR: 9.49e-07 | 1286.4ms/step | 15.5 samples/s +Step 42810/44000 | Loss: 2.9056 | LR: 9.33e-07 | 1288.7ms/step | 15.5 samples/s +Step 42820/44000 | Loss: 2.9614 | LR: 9.18e-07 | 1287.1ms/step | 15.5 samples/s +Step 42830/44000 | Loss: 2.9248 | LR: 9.02e-07 | 1284.5ms/step | 15.6 samples/s +Step 42840/44000 | Loss: 2.9353 | LR: 8.87e-07 | 1281.1ms/step | 15.6 samples/s +Step 42850/44000 | Loss: 2.9344 | LR: 8.72e-07 | 1275.6ms/step | 15.7 samples/s +Step 42860/44000 | Loss: 2.9436 | LR: 8.57e-07 | 1280.1ms/step | 15.6 samples/s +Step 42870/44000 | Loss: 2.9401 | LR: 8.42e-07 | 1280.9ms/step | 15.6 samples/s +Step 42880/44000 | Loss: 2.9285 | LR: 8.27e-07 | 1282.2ms/step | 15.6 samples/s +Step 42890/44000 | Loss: 2.8807 | LR: 8.12e-07 | 1282.1ms/step | 15.6 samples/s +Step 42900/44000 | Loss: 2.9381 | LR: 7.98e-07 | 1285.5ms/step | 15.6 samples/s +Step 42910/44000 | Loss: 2.9292 | LR: 7.83e-07 | 1286.6ms/step | 15.5 samples/s +Step 42920/44000 | Loss: 2.9107 | LR: 7.69e-07 | 1283.7ms/step | 15.6 samples/s +Step 42930/44000 | Loss: 2.9061 | LR: 7.55e-07 | 1285.2ms/step | 15.6 samples/s +Step 42940/44000 | Loss: 2.9172 | LR: 7.41e-07 | 1280.5ms/step | 15.6 samples/s +Step 42950/44000 | Loss: 2.9324 | LR: 7.27e-07 | 1277.9ms/step | 15.7 samples/s +Step 42960/44000 | Loss: 2.9252 | LR: 7.13e-07 | 1281.6ms/step | 15.6 samples/s +Step 42970/44000 | Loss: 2.9190 | LR: 6.99e-07 | 1279.9ms/step | 15.6 samples/s +Step 42980/44000 | Loss: 2.9292 | LR: 6.86e-07 | 1284.3ms/step | 15.6 samples/s +Step 42990/44000 | Loss: 2.9242 | LR: 6.72e-07 | 1282.2ms/step | 15.6 samples/s +Step 43000/44000 | Loss: 2.9169 | LR: 6.59e-07 | 1284.8ms/step | 15.6 samples/s +Validation at step 43000: Loss = 2.9386, Perplexity = 18.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-43000 +Step 43010/44000 | Loss: 2.9238 | LR: 6.46e-07 | 3692.6ms/step | 5.4 samples/s +Step 43020/44000 | Loss: 2.9091 | LR: 6.33e-07 | 1282.6ms/step | 15.6 samples/s +Step 43030/44000 | Loss: 2.9309 | LR: 6.20e-07 | 1283.4ms/step | 15.6 samples/s +Step 43040/44000 | Loss: 2.8934 | LR: 6.08e-07 | 1283.6ms/step | 15.6 samples/s +Step 43050/44000 | Loss: 2.9200 | LR: 5.95e-07 | 1284.4ms/step | 15.6 samples/s +Step 43060/44000 | Loss: 2.9295 | LR: 5.83e-07 | 1282.2ms/step | 15.6 samples/s +Step 43070/44000 | Loss: 2.9443 | LR: 5.70e-07 | 1282.5ms/step | 15.6 samples/s +Step 43080/44000 | Loss: 2.9189 | LR: 5.58e-07 | 1285.6ms/step | 15.6 samples/s +Step 43090/44000 | Loss: 2.9009 | LR: 5.46e-07 | 1279.8ms/step | 15.6 samples/s +Step 43100/44000 | Loss: 2.9146 | LR: 5.34e-07 | 1281.8ms/step | 15.6 samples/s +Step 43110/44000 | Loss: 2.9268 | LR: 5.22e-07 | 1281.6ms/step | 15.6 samples/s +Step 43120/44000 | Loss: 2.9390 | LR: 5.11e-07 | 1279.4ms/step | 15.6 samples/s +Step 43130/44000 | Loss: 2.9422 | LR: 4.99e-07 | 1280.5ms/step | 15.6 samples/s +Step 43140/44000 | Loss: 2.9281 | LR: 4.88e-07 | 1278.5ms/step | 15.6 samples/s +Step 43150/44000 | Loss: 2.9125 | LR: 4.76e-07 | 1284.4ms/step | 15.6 samples/s +Step 43160/44000 | Loss: 2.9111 | LR: 4.65e-07 | 1283.7ms/step | 15.6 samples/s +Step 43170/44000 | Loss: 2.8988 | LR: 4.54e-07 | 1281.7ms/step | 15.6 samples/s +Step 43180/44000 | Loss: 2.9101 | LR: 4.43e-07 | 1280.3ms/step | 15.6 samples/s +Step 43190/44000 | Loss: 2.9052 | LR: 4.33e-07 | 1283.0ms/step | 15.6 samples/s +Step 43200/44000 | Loss: 2.8941 | LR: 4.22e-07 | 1280.2ms/step | 15.6 samples/s +Step 43210/44000 | Loss: 2.9292 | LR: 4.12e-07 | 1281.0ms/step | 15.6 samples/s +Step 43220/44000 | Loss: 2.9166 | LR: 4.01e-07 | 1279.0ms/step | 15.6 samples/s +Step 43230/44000 | Loss: 2.9383 | LR: 3.91e-07 | 1280.6ms/step | 15.6 samples/s +Step 43240/44000 | Loss: 2.9351 | LR: 3.81e-07 | 1280.7ms/step | 15.6 samples/s +Step 43250/44000 | Loss: 2.9399 | LR: 3.71e-07 | 1283.2ms/step | 15.6 samples/s +Validation at step 43250: Loss = 2.9384, Perplexity = 18.89 +Step 43260/44000 | Loss: 2.9319 | LR: 3.61e-07 | 3427.7ms/step | 5.8 samples/s +Step 43270/44000 | Loss: 2.9198 | LR: 3.51e-07 | 1280.2ms/step | 15.6 samples/s +Step 43280/44000 | Loss: 2.9266 | LR: 3.42e-07 | 1279.8ms/step | 15.6 samples/s +Step 43290/44000 | Loss: 2.9107 | LR: 3.32e-07 | 1280.0ms/step | 15.6 samples/s +Step 43300/44000 | Loss: 2.9421 | LR: 3.23e-07 | 1279.8ms/step | 15.6 samples/s +Step 43310/44000 | Loss: 2.9460 | LR: 3.14e-07 | 1283.0ms/step | 15.6 samples/s +Step 43320/44000 | Loss: 2.9517 | LR: 3.05e-07 | 1282.3ms/step | 15.6 samples/s +Step 43330/44000 | Loss: 2.9127 | LR: 2.96e-07 | 1279.0ms/step | 15.6 samples/s +Step 43340/44000 | Loss: 2.9350 | LR: 2.87e-07 | 1282.5ms/step | 15.6 samples/s +Step 43350/44000 | Loss: 2.9096 | LR: 2.79e-07 | 1280.2ms/step | 15.6 samples/s +Step 43360/44000 | Loss: 2.9309 | LR: 2.70e-07 | 1280.2ms/step | 15.6 samples/s +Step 43370/44000 | Loss: 2.9350 | LR: 2.62e-07 | 1280.9ms/step | 15.6 samples/s +Step 43380/44000 | Loss: 2.9284 | LR: 2.53e-07 | 1281.9ms/step | 15.6 samples/s +Step 43390/44000 | Loss: 2.9029 | LR: 2.45e-07 | 1283.0ms/step | 15.6 samples/s +Step 43400/44000 | Loss: 2.9198 | LR: 2.37e-07 | 1282.3ms/step | 15.6 samples/s +Step 43410/44000 | Loss: 2.9188 | LR: 2.30e-07 | 1281.6ms/step | 15.6 samples/s +Step 43420/44000 | Loss: 2.8975 | LR: 2.22e-07 | 1282.0ms/step | 15.6 samples/s +Step 43430/44000 | Loss: 2.9134 | LR: 2.14e-07 | 1280.1ms/step | 15.6 samples/s +Step 43440/44000 | Loss: 2.9610 | LR: 2.07e-07 | 1279.5ms/step | 15.6 samples/s +Step 43450/44000 | Loss: 2.9411 | LR: 1.99e-07 | 1281.1ms/step | 15.6 samples/s +Step 43460/44000 | Loss: 2.9329 | LR: 1.92e-07 | 1281.6ms/step | 15.6 samples/s +Step 43470/44000 | Loss: 2.9324 | LR: 1.85e-07 | 1282.1ms/step | 15.6 samples/s +Step 43480/44000 | Loss: 2.9002 | LR: 1.78e-07 | 1284.9ms/step | 15.6 samples/s +Step 43490/44000 | Loss: 2.9199 | LR: 1.72e-07 | 1282.2ms/step | 15.6 samples/s +Step 43500/44000 | Loss: 2.9594 | LR: 1.65e-07 | 1277.7ms/step | 15.7 samples/s +Validation at step 43500: Loss = 2.9386, Perplexity = 18.89 +Step 43510/44000 | Loss: 2.9235 | LR: 1.58e-07 | 3428.1ms/step | 5.8 samples/s +Step 43520/44000 | Loss: 2.9448 | LR: 1.52e-07 | 1280.8ms/step | 15.6 samples/s +Step 43530/44000 | Loss: 2.9089 | LR: 1.46e-07 | 1282.3ms/step | 15.6 samples/s +Step 43540/44000 | Loss: 2.8966 | LR: 1.40e-07 | 1282.8ms/step | 15.6 samples/s +Step 43550/44000 | Loss: 2.9183 | LR: 1.34e-07 | 1281.3ms/step | 15.6 samples/s +Step 43560/44000 | Loss: 2.9197 | LR: 1.28e-07 | 1279.8ms/step | 15.6 samples/s +Step 43570/44000 | Loss: 2.9180 | LR: 1.22e-07 | 1278.6ms/step | 15.6 samples/s +Step 43580/44000 | Loss: 2.9429 | LR: 1.16e-07 | 1282.5ms/step | 15.6 samples/s +Step 43590/44000 | Loss: 2.9199 | LR: 1.11e-07 | 1283.8ms/step | 15.6 samples/s +Step 43600/44000 | Loss: 2.9310 | LR: 1.06e-07 | 1280.9ms/step | 15.6 samples/s +Step 43610/44000 | Loss: 2.8774 | LR: 1.00e-07 | 1281.6ms/step | 15.6 samples/s +Step 43620/44000 | Loss: 2.9151 | LR: 9.52e-08 | 1281.0ms/step | 15.6 samples/s +Step 43630/44000 | Loss: 2.9080 | LR: 9.03e-08 | 1281.5ms/step | 15.6 samples/s +Step 43640/44000 | Loss: 2.9245 | LR: 8.55e-08 | 1282.7ms/step | 15.6 samples/s +Step 43650/44000 | Loss: 2.9269 | LR: 8.08e-08 | 1279.5ms/step | 15.6 samples/s +Step 43660/44000 | Loss: 2.9284 | LR: 7.62e-08 | 1276.6ms/step | 15.7 samples/s +Step 43670/44000 | Loss: 2.9355 | LR: 7.18e-08 | 1279.2ms/step | 15.6 samples/s +Step 43680/44000 | Loss: 2.9435 | LR: 6.75e-08 | 1274.4ms/step | 15.7 samples/s +Step 43690/44000 | Loss: 2.9045 | LR: 6.34e-08 | 1278.7ms/step | 15.6 samples/s +Step 43700/44000 | Loss: 2.9239 | LR: 5.94e-08 | 1280.8ms/step | 15.6 samples/s +Step 43710/44000 | Loss: 2.9433 | LR: 5.55e-08 | 1282.1ms/step | 15.6 samples/s +Step 43720/44000 | Loss: 2.9101 | LR: 5.17e-08 | 1281.3ms/step | 15.6 samples/s +Step 43730/44000 | Loss: 2.9480 | LR: 4.81e-08 | 1281.5ms/step | 15.6 samples/s +Step 43740/44000 | Loss: 2.9557 | LR: 4.46e-08 | 1279.9ms/step | 15.6 samples/s +Step 43750/44000 | Loss: 2.9269 | LR: 4.12e-08 | 1280.7ms/step | 15.6 samples/s +Validation at step 43750: Loss = 2.9387, Perplexity = 18.89 +Step 43760/44000 | Loss: 2.9453 | LR: 3.80e-08 | 3433.7ms/step | 5.8 samples/s +Step 43770/44000 | Loss: 2.9431 | LR: 3.49e-08 | 1286.3ms/step | 15.5 samples/s +Step 43780/44000 | Loss: 2.9241 | LR: 3.19e-08 | 1283.7ms/step | 15.6 samples/s +Step 43790/44000 | Loss: 2.9135 | LR: 2.91e-08 | 1286.2ms/step | 15.5 samples/s +Step 43800/44000 | Loss: 2.9166 | LR: 2.64e-08 | 1283.0ms/step | 15.6 samples/s +Step 43810/44000 | Loss: 2.9240 | LR: 2.38e-08 | 1285.3ms/step | 15.6 samples/s +Step 43820/44000 | Loss: 2.9163 | LR: 2.14e-08 | 1286.7ms/step | 15.5 samples/s +Step 43830/44000 | Loss: 2.9386 | LR: 1.91e-08 | 1286.1ms/step | 15.6 samples/s +Step 43840/44000 | Loss: 2.9261 | LR: 1.69e-08 | 1286.3ms/step | 15.5 samples/s +Step 43850/44000 | Loss: 2.9171 | LR: 1.48e-08 | 1282.3ms/step | 15.6 samples/s +Step 43860/44000 | Loss: 2.9275 | LR: 1.29e-08 | 1282.7ms/step | 15.6 samples/s +Step 43870/44000 | Loss: 2.9426 | LR: 1.11e-08 | 1281.9ms/step | 15.6 samples/s +Step 43880/44000 | Loss: 2.9587 | LR: 9.50e-09 | 1279.4ms/step | 15.6 samples/s +Step 43890/44000 | Loss: 2.9143 | LR: 7.98e-09 | 1280.1ms/step | 15.6 samples/s +Step 43900/44000 | Loss: 2.9259 | LR: 6.60e-09 | 1282.7ms/step | 15.6 samples/s +Step 43910/44000 | Loss: 2.9282 | LR: 5.34e-09 | 1281.0ms/step | 15.6 samples/s +Step 43920/44000 | Loss: 2.9134 | LR: 4.22e-09 | 1282.3ms/step | 15.6 samples/s +Step 43930/44000 | Loss: 2.9061 | LR: 3.23e-09 | 1281.8ms/step | 15.6 samples/s +Step 43940/44000 | Loss: 2.9129 | LR: 2.37e-09 | 1281.5ms/step | 15.6 samples/s +Step 43950/44000 | Loss: 2.9279 | LR: 1.65e-09 | 1275.6ms/step | 15.7 samples/s +Step 43960/44000 | Loss: 2.9061 | LR: 1.06e-09 | 1276.0ms/step | 15.7 samples/s +Step 43970/44000 | Loss: 2.9356 | LR: 5.94e-10 | 1280.9ms/step | 15.6 samples/s +Step 43980/44000 | Loss: 2.9377 | LR: 2.64e-10 | 1280.1ms/step | 15.6 samples/s +Step 43990/44000 | Loss: 2.9309 | LR: 6.60e-11 | 1280.1ms/step | 15.6 samples/s +Step 44000/44000 | Loss: 2.9725 | LR: 0.00e+00 | 1279.4ms/step | 15.6 samples/s +Validation at step 44000: Loss = 2.9383, Perplexity = 18.88 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled/run_20260707_034554/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss ██▇▇▇▆▆▅▅▅▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ██▇▆▆▆▆▅▅▄▄▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▇▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ▆▅▆▂▇▄▅▃▄▅▇▂▅▅▆▅▄▄▆▆▅▆▄▁▅▅▇▆█▅▄▄▄▄▅▅▄▆▆▅ +wandb: speed/steps_per_sec █████████████████▁██▁███████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.93829 +wandb: eval/perplexity 18.8836 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1279.41339 +wandb: speed/samples_per_sec 15.63216 +wandb: speed/steps_per_sec 0.78161 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m resampled 8k nrb31 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/304r0ilz +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260707_171131-304r0ilz/logs +Training complete!